From e31c8619fc70143b021e05dc6fdcb8833f926cc1 Mon Sep 17 00:00:00 2001 From: aarkue Date: Thu, 20 Aug 2026 14:37:43 +0200 Subject: [PATCH 1/3] Extraction blueprints and cleanup --- .github/workflows/test.linux.yml | 46 - CHANGELOG.md | 22 + Cargo.lock | 876 ++- Cargo.toml | 8 + macros_process_mining/src/lib.rs | 470 +- process_mining/Cargo.toml | 114 +- process_mining/README.md | 2 +- process_mining/benches/common/mod.rs | 17 + process_mining/benches/ocel_import.rs | 62 + process_mining/examples/README.md | 4 - .../examples/ocel_dataset_crosscheck.rs | 1430 +++++ process_mining/examples/ocel_kuzudb_export.rs | 41 - .../examples/ocel_stream_to_duckdb.rs | 23 + .../case_centric/event_timestamp_histogram.rs | 15 +- .../src/bindings/extraction_bindings.rs | 834 +++ .../src/bindings/extraction_dbcon_bindings.rs | 680 +++ process_mining/src/bindings/mod.rs | 1603 +++++- .../src/bindings/path_schema_bindings.rs | 2 +- .../src/bindings/slim_ocel_bindings.rs | 334 +- .../conformance/object_centric/oc_declare.rs | 46 +- .../event_data/case_centric/xes/stream_xes.rs | 2 +- .../event_data/object_centric/appendable.rs | 103 + .../object_centric/extraction/blueprint.rs | 424 ++ .../object_centric/extraction/case_centric.rs | 696 +++ .../object_centric/extraction/catalog.rs | 439 ++ .../object_centric/extraction/compile.rs | 2491 +++++++++ .../extraction/compile/dialect.rs | 350 ++ .../extraction/compile/differential.rs | 2835 ++++++++++ .../object_centric/extraction/compile/emit.rs | 969 ++++ .../extraction/compile/tests.rs | 1276 +++++ .../extraction/dbcon_provider.rs | 714 +++ .../object_centric/extraction/desugar.rs | 208 + .../object_centric/extraction/differential.rs | 418 ++ .../object_centric/extraction/duckdb_sink.rs | 1086 ++++ .../object_centric/extraction/expr.rs | 1067 ++++ .../object_centric/extraction/extract.rs | 364 ++ .../object_centric/extraction/graph.rs | 696 +++ .../object_centric/extraction/mapping_exec.rs | 1116 ++++ .../object_centric/extraction/mod.rs | 107 + .../object_centric/extraction/predicate.rs | 754 +++ .../object_centric/extraction/provider.rs | 211 + .../object_centric/extraction/pushdown.rs | 94 + .../object_centric/extraction/report.rs | 461 ++ .../object_centric/extraction/row.rs | 67 + .../object_centric/extraction/schema.rs | 416 ++ .../object_centric/extraction/sink.rs | 317 ++ .../object_centric/extraction/slim_sink.rs | 298 ++ .../extraction/sqlite_provider.rs | 541 ++ .../object_centric/extraction/tests.rs | 4681 +++++++++++++++++ .../object_centric/extraction/validate.rs | 1819 +++++++ .../object_centric/extraction/value.rs | 511 ++ .../event_data/object_centric/graph_db/mod.rs | 4 - .../object_centric/graph_db/ocel_kuzudb.rs | 442 -- .../src/core/event_data/object_centric/io.rs | 166 +- .../linked_ocel/index_linked_ocel.rs | 66 + .../linked_ocel/slim_linked_ocel.rs | 135 +- .../src/core/event_data/object_centric/mod.rs | 14 +- .../object_centric/ocel_bundle/blueprint.rs | 193 + .../object_centric/ocel_bundle/export.rs | 814 +++ .../object_centric/ocel_bundle/import.rs | 765 +++ .../object_centric/ocel_bundle/meta.rs | 345 ++ .../object_centric/ocel_bundle/mod.rs | 38 + .../object_centric/ocel_bundle/tests.rs | 129 + .../ocel_csv/csv_ocel_export.rs | 35 +- .../ocel_csv/csv_ocel_import.rs | 192 +- .../object_centric/ocel_csv/escaping.rs | 120 + .../event_data/object_centric/ocel_csv/mod.rs | 10 +- .../object_centric/ocel_json/mod.rs | 100 + .../ocel_sql/duckdb/duckdb_ocel_import.rs | 307 +- .../object_centric/ocel_sql/duckdb/mod.rs | 9 + .../ocel_sql/duckdb/schema/mod.rs | 19 + .../ocel_sql/duckdb/schema/reader.rs | 553 ++ .../ocel_sql/duckdb/schema/sink.rs | 534 ++ .../ocel_sql/duckdb/schema/sqlite_source.rs | 54 + .../ocel_sql/duckdb/schema/stream.rs | 467 ++ .../ocel_sql/duckdb/schema/tables.rs | 209 + .../ocel_sql/duckdb/schema/value.rs | 204 + .../ocel_sql/duckdb/schema/views.rs | 306 ++ .../object_centric/ocel_sql/export.rs | 8 +- .../event_data/object_centric/ocel_sql/mod.rs | 265 +- .../ocel_sql/sqlite/sqlite_ocel_import.rs | 292 +- .../event_data/object_centric/ocel_struct.rs | 62 +- process_mining/src/core/io.rs | 20 + process_mining/src/core/mod.rs | 2 + process_mining/src/core/tabular_source.rs | 194 + process_mining/src/lib.rs | 15 +- process_mining/tests/ocel_bundle.rs | 1043 ++++ .../tests/ocel_roundtrip_edge_cases.rs | 355 ++ r4pm/Cargo.toml | 2 +- 89 files changed, 39527 insertions(+), 1121 deletions(-) create mode 100644 process_mining/benches/common/mod.rs create mode 100644 process_mining/benches/ocel_import.rs create mode 100644 process_mining/examples/ocel_dataset_crosscheck.rs delete mode 100644 process_mining/examples/ocel_kuzudb_export.rs create mode 100644 process_mining/examples/ocel_stream_to_duckdb.rs create mode 100644 process_mining/src/bindings/extraction_bindings.rs create mode 100644 process_mining/src/bindings/extraction_dbcon_bindings.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/blueprint.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/case_centric.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/catalog.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/compile.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/compile/dialect.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/compile/differential.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/compile/emit.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/compile/tests.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/dbcon_provider.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/desugar.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/differential.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/duckdb_sink.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/expr.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/extract.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/graph.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/mapping_exec.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/mod.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/predicate.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/provider.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/pushdown.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/report.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/row.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/schema.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/sink.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/slim_sink.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/sqlite_provider.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/tests.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/validate.rs create mode 100644 process_mining/src/core/event_data/object_centric/extraction/value.rs delete mode 100644 process_mining/src/core/event_data/object_centric/graph_db/mod.rs delete mode 100644 process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/blueprint.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/export.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/import.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/meta.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/mod.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_bundle/tests.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_csv/escaping.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/mod.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/reader.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sink.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sqlite_source.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/stream.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/tables.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/value.rs create mode 100644 process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/views.rs create mode 100644 process_mining/src/core/tabular_source.rs create mode 100644 process_mining/tests/ocel_bundle.rs create mode 100644 process_mining/tests/ocel_roundtrip_edge_cases.rs diff --git a/.github/workflows/test.linux.yml b/.github/workflows/test.linux.yml index a29cb1db..d0beb80c 100644 --- a/.github/workflows/test.linux.yml +++ b/.github/workflows/test.linux.yml @@ -22,18 +22,6 @@ jobs: source "$HOME/.cargo/env" - name: Install Graphviz run: sudo apt-get update && sudo apt-get -y install graphviz p7zip-full - - name: Download DuckDB - run: wget -O libduckdb-linux-amd64.zip https://github.com/duckdb/duckdb/releases/download/$DUCKDB_VERSION/libduckdb-linux-amd64.zip - env: - DUCKDB_VERSION: v1.3.2 - - name: Unpacking duckdb - run: 7z x ${{ github.workspace }}/libduckdb-linux-amd64.zip -o${{ github.workspace }}/libduckdb - - name: Download kuzudb - run: wget -O kuzu.zip https://github.com/kuzudb/kuzu/releases/download/$KUZU_VERSION/libkuzu-linux-x86_64.tar.gz - env: - KUZU_VERSION: v0.11.2 - - name: Unpacking kuzudb - run: mkdir ${{ github.workspace }}/libkuzu && tar xzf ${{ github.workspace }}/kuzu.zip -C ${{ github.workspace }}/libkuzu - name: Downloading test files run: wget -O process_mining/test_data/out.zip https://rwth-aachen.sciebo.de/s/4cvtTU3lLOgtxt1/download - name: Unpacking test files @@ -41,51 +29,17 @@ jobs: - name: Build run: source "$HOME/.cargo/env" && cargo build --verbose --all-features working-directory: ./process_mining - env: - DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb - DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb - LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - KUZU_SHARED: 1 - KUZU_LIBRARY_DIR: ${{ github.workspace }}/libkuzu - KUZU_INCLUDE_DIR: ${{ github.workspace }}/libkuzu - name: Clippy working-directory: ./process_mining run: source "$HOME/.cargo/env" && cargo clippy --all-targets --all-features -- -D warnings - env: - DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb - DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb - LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - KUZU_SHARED: 1 - KUZU_LIBRARY_DIR: ${{ github.workspace }}/libkuzu - KUZU_INCLUDE_DIR: ${{ github.workspace }}/libkuzu - name: Check formatting working-directory: ./process_mining run: source "$HOME/.cargo/env" && cargo fmt --all --check - env: - DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb - DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb - LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - KUZU_SHARED: 1 - KUZU_LIBRARY_DIR: ${{ github.workspace }}/libkuzu - KUZU_INCLUDE_DIR: ${{ github.workspace }}/libkuzu - name: Check docs working-directory: ./process_mining run: source "$HOME/.cargo/env" && cargo doc --all-features --no-deps env: - DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb - DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb - LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - KUZU_SHARED: 1 - KUZU_LIBRARY_DIR: ${{ github.workspace }}/libkuzu - KUZU_INCLUDE_DIR: ${{ github.workspace }}/libkuzu RUSTDOCFLAGS: -D warnings - name: Run tests run: source "$HOME/.cargo/env" && cargo test --verbose --all-features working-directory: ./process_mining - env: - DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb - DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb - LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${{ github.workspace }}/libkuzu/:${{ github.workspace }}/libduckdb/ - KUZU_SHARED: 1 - KUZU_LIBRARY_DIR: ${{ github.workspace }}/libkuzu - KUZU_INCLUDE_DIR: ${{ github.workspace }}/libkuzu diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9e2197..96c4573c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,28 @@ # Changelog ## Unreleased +- **Breaking:** Removed `KuzuDB` export (the `kuzudb` feature, the `core::event_data::object_centric::graph_db` module and the `ocel_kuzudb_export` example). Kuzu is no longer maintained upstream. +- Added extraction blueprints (`extraction-blueprint`): A declarative model for building an OCEL from relational data, with a row executor, a SQL-view compiler, a validator, and in-memory and `DuckDB` sinks + - Feature `extraction-dbcon` adds a `RowProvider` over SQLite, CSV and Parquet via `dbcon`, all readable from bytes and buildable for `wasm32` + - Feature `extraction-dbcon-postgres` adds PostgreSQL, which needs `sqlx` and is native-only +- Added the OCEL 2.0 bundled CSV/Parquet format (`ocel-bundle`, plus `ocel-bundle-parquet` for Parquet storage): An `.ocel.zip` archive or a directory with the same layout, read and written through `Importable`/`Exportable` like any other OCEL format + - `Exportable::export_to_path_as` writes to a path in an explicitly named format, for paths that cannot carry one (e.g., a directory) + - Snappy-compressed Parquet is now readable, which is what most other Parquet writers emit by default +- OCEL 2.0 CSV tweaks + - An event attribute column is written under its plain name; the `ea:` prefix is no longer produced on export, and any column that is not `id`, `activity`, `timestamp` or `ot:` is an event attribute. Import still strips a leading `ea:` so older files read unchanged + - An object id or qualifier containing `/`, `#`, `{` or `\` is escaped with a backslash on export and read back on import + - An `ot:` header names the object type exactly, and an event attribute value is kept as written; both were trimmed before + - A value is only read as a number or an instant when its text is the canonical spelling of one; `007`, `+7`, `1e3`, an integer too large for `i64`, and a timestamp with no timezone stay strings + - `strict` rejects an o2o row whose source object has no declared type; the check sat inside a `verbose` branch, so `strict` alone dropped the row silently +- Added the `ocel_dataset_crosscheck` example: walks a directory of dataset folders, each holding the log it started from under `source/` and its re-exports (`.ocel.zip`, `.ocel.csv`, `.json`, `.xml`, `.sqlite`) beside it, reads every re-export back and reports where it disagrees with the source, separating differences in values from ones only in the attribute variant or recorded time +- `ocel_sql` exposes the `DuckDB` consolidated schema: `DuckDbLinkedOCEL`, `stream_ocel_file_to_duckdb` / `_with`, `DuckDbImportOptions`, and `read_consolidated_ocel_from_duckdb_path` / `read_consolidated_slim_ocel_from_duckdb_path` +- `SqlOcelImportOptions` with `import_ocel_sqlite_from_con_with_options` / `import_ocel_sqlite_from_path_with_options` / `import_ocel_duckdb_from_con_with_options` +- An object-type table without its `ocel_changed_field` column is now read as initial state (configurable with `allow_missing_changed_field: false`) +- `StreamImportOCEL` streams a reader or a path into any `AppendableOCEL` and finalizes it; `is_streaming_format` reports which formats it covers +- `TabularSource` holds the bytes of a tabular file (`SQLite`, CSV, Parquet, workbook) in the registry, so a binding can name a dropped file by id where there is no filesystem +- `OCELAttributeType::coalesce`: the narrowest type covering two others, used by CSV type inference +- `OCELEvent.time` accepts the non-RFC3339 timestamp formats the rest of the crate parses +- `AggregatedEventTimestamps::bin_width_ms` (new field) makes the bin width explicit: the spacing of the bin centers, and `0` when there are no bins; before it was implicit and a caller re-deriving it could disagree with the centers (**Breaking**) - Fixed a regression in OC-DECLARE discovery/conformance runtime performance: - Now builds and construct a reverse-E2O index grouped by event type diff --git a/Cargo.lock b/Cargo.lock index 2ed7ca48..0586b220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -235,6 +250,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + [[package]] name = "arrow-ord" version = "58.3.0" @@ -364,6 +393,16 @@ dependencies = [ "debug_unsafe", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -461,6 +500,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "borsh" version = "1.6.1" @@ -491,6 +539,27 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.5.1" @@ -548,6 +617,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -557,6 +632,24 @@ dependencies = [ "serde", ] +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd 0.18.1", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml 0.41.0", + "serde", + "zip 8.6.0", +] + [[package]] name = "cast" version = "0.3.0" @@ -596,6 +689,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -664,7 +768,6 @@ checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", "clap_lex", - "strsim", ] [[package]] @@ -674,22 +777,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] -name = "cmake" -version = "0.1.58" +name = "cmov" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "codepage" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" dependencies = [ - "termcolor", - "unicode-width 0.1.14", + "encoding_rs", ] [[package]] @@ -700,7 +799,7 @@ checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "crossterm", "unicode-segmentation", - "unicode-width 0.2.2", + "unicode-width", ] [[package]] @@ -787,6 +886,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -914,6 +1028,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -936,62 +1059,12 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.138" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3956d60afa98653c5a57f60d7056edd513bfe0307ef6fb06f6167400c3884459" -dependencies = [ - "cc", - "cxxbridge-cmd", - "cxxbridge-flags", - "cxxbridge-macro", - "foldhash 0.1.5", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.138" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4b7522f539fe056f1d6fc8577d8ab731451f6f33a89b1e5912e22b76c553e7" -dependencies = [ - "cc", - "codespan-reporting", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.117", -] - -[[package]] -name = "cxxbridge-cmd" -version = "1.0.138" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f01e92ab4ce9fd4d16e3bb11b158d98cbdcca803c1417aa43130a6526fbf208" -dependencies = [ - "clap", - "codespan-reporting", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.138" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41cbfab344869e70998b388923f7d1266588f56c8ca284abf259b1c1ffc695" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.138" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d82a2f759f0ad3eae43b96604efd42b1d4729a35a6f2dc7bdb797ae25d9284" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", + "cmov", ] [[package]] @@ -1028,6 +1101,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dbcon" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a032a3800982b3848bfcfe8ce674f5c299507deccc1d4697c514be41b4370210" +dependencies = [ + "anyhow", + "bytes", + "calamine", + "chrono", + "csv", + "duckdb", + "futures", + "parquet", + "rusqlite", + "serde", + "serde_json", + "sqlx", + "tokio", +] + [[package]] name = "debug_unsafe" version = "0.1.4" @@ -1076,8 +1170,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1115,6 +1220,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cfcded997a93eb31edd639361fa33fd229a8784e953b37d71035fe3890b7b" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "duckdb" version = "1.10501.0" @@ -1144,6 +1255,18 @@ name = "either" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] [[package]] name = "equivalent" @@ -1161,6 +1284,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "ethnum" version = "1.5.3" @@ -1234,6 +1367,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -1254,6 +1397,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1272,6 +1426,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1339,6 +1508,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -1431,6 +1611,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -1532,6 +1713,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -1582,6 +1765,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.12" @@ -1636,6 +1837,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1859,6 +2069,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "into-attr" version = "0.1.1" @@ -1952,21 +2168,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kuzu" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c435909261e18a703def3991ce5cb13576712a99764999c7c7f6a5d038d7366" -dependencies = [ - "cmake", - "cxx", - "cxx-build", - "rust_decimal", - "rustversion", - "time", - "uuid", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -2048,6 +2249,7 @@ version = "1.10501.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248" dependencies = [ + "cc", "flate2", "pkg-config", "reqwest", @@ -2055,7 +2257,7 @@ dependencies = [ "serde_json", "tar", "vcpkg", - "zip", + "zip 6.0.0", ] [[package]] @@ -2075,15 +2277,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "link-cplusplus" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2142,6 +2335,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "macros_process_mining" version = "0.6.0" @@ -2161,6 +2363,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.2" @@ -2230,6 +2442,23 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "now" version = "0.1.3" @@ -2350,12 +2579,58 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -2394,6 +2669,39 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parquet" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +dependencies = [ + "ahash 0.8.12", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "twox-hash", + "zstd", +] + [[package]] name = "paste" version = "1.0.15" @@ -2446,7 +2754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2556,7 +2864,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b4fed2343961b3eea3db2cee165540c3e1ad9d5782350cc55a9e76cf440148" dependencies = [ - "atoi_simd", + "atoi_simd 0.16.1", "bitflags", "bytemuck", "chrono", @@ -2598,7 +2906,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138785beda4e4a90a025219f09d0d15a671b2be9091513ede58e05db6ad4413f" dependencies = [ - "atoi_simd", + "atoi_simd 0.16.1", "bytemuck", "chrono", "either", @@ -2713,7 +3021,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10388c64b8155122488229a881d1c6f4fdc393bc988e764ab51b182fcb2307e4" dependencies = [ "async-trait", - "atoi_simd", + "atoi_simd 0.16.1", "blake3", "bytes", "chrono", @@ -2910,7 +3218,7 @@ dependencies = [ "rayon", "recursive", "regex", - "sha2", + "sha2 0.10.9", "strum_macros", "version_check", ] @@ -3008,7 +3316,7 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6a3a6e279a7a984a0b83715660f9e880590c6129ec2104396bfa710bcd76dee" dependencies = [ - "atoi_simd", + "atoi_simd 0.16.1", "bytemuck", "chrono", "chrono-tz", @@ -3114,10 +3422,12 @@ dependencies = [ name = "process_mining" version = "0.6.0" dependencies = [ + "base64", + "bytes", "chrono", "criterion", "csv", - "cxx-build", + "dbcon", "dhat", "duckdb", "flate2", @@ -3125,15 +3435,16 @@ dependencies = [ "hashbrown 0.17.1", "inventory", "itertools 0.14.0", - "kuzu", "macros_process_mining", "nalgebra", - "ordered-float", + "ordered-float 5.3.0", + "parquet", "petgraph", "polars", "quick-xml 0.37.5", "rand 0.9.4", "rayon", + "regex", "rusqlite", "rustc-hash 2.1.2", "schemars 1.2.1", @@ -3142,6 +3453,7 @@ dependencies = [ "serde_with", "tempfile", "uuid", + "zip 6.0.0", ] [[package]] @@ -3193,6 +3505,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3306,6 +3628,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3344,6 +3677,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -3643,6 +3982,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3787,12 +4135,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scratch" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" - [[package]] name = "seahash" version = "4.1.0" @@ -3828,6 +4170,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -3937,6 +4285,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3945,7 +4304,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4052,6 +4422,15 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" [[package]] name = "socket2" @@ -4063,6 +4442,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -4084,6 +4472,181 @@ dependencies = [ "log", ] +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64", + "bytes", + "cfg-if", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink 0.11.1", + "indexmap 2.14.0", + "log", + "memchr", + "native-tls", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.117", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.117", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1", + "sha2 0.11.0", + "sqlx-core", + "thiserror", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "rand 0.10.2", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "chrono", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4130,6 +4693,17 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4235,15 +4809,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "2.0.18" @@ -4270,6 +4835,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "time" version = "0.3.49" @@ -4380,6 +4956,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -4475,6 +5062,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4506,6 +5094,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -4518,6 +5118,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -4533,6 +5139,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-reverse" version = "1.0.9" @@ -4548,12 +5160,6 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.2" @@ -4813,6 +5419,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "wide" version = "0.7.33" @@ -5329,6 +5941,20 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "typed-path", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index 09db70e1..98e0cbc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,11 @@ debug = 1 # Full debuginfo on the --all-features test build can take a lot of disk space. [profile.test] debug = "line-tables-only" + +# `bundled` DuckDB compiles its amalgamation into `libduckdb.a`, which every test, example and +# bench binary then links statically. With dev-profile debuginfo that archive is ~1.1 GB and +# parallel links exhaust memory; at `-O0` the linked DuckDB is also far slower than the system +# library used to be. Neither matters for C++ we never step through. +[profile.dev.package.libduckdb-sys] +debug = false +opt-level = 2 diff --git a/macros_process_mining/src/lib.rs b/macros_process_mining/src/lib.rs index d4e19064..f7d5b35b 100644 --- a/macros_process_mining/src/lib.rs +++ b/macros_process_mining/src/lib.rs @@ -6,6 +6,10 @@ use syn::{parse_macro_input, Attribute, FnArg, ItemFn, Lifetime, Pat}; use syn::fold::{self, Fold}; use syn::{AngleBracketedGenericArguments, GenericArgument, Type, TypeReference}; +/// The optional argument added to bindings whose result is stored in the registry, letting the +/// caller pick the id instead of receiving a generated one. +const OUTPUT_ID_ARG: &str = "output_id"; + /// Name of big data types, which are handled over app state instead of being (de-)serialized const BIG_TYPES_NAMES: &[&str] = &[ "EventLog", @@ -13,6 +17,7 @@ const BIG_TYPES_NAMES: &[&str] = &[ "EventLogActivityProjection", "SlimLinkedOCEL", "IndexLinkedOCEL", + "TabularSource", ]; /// Removes/elide lifetimes and other special cases (i.e., certain generics) from types @@ -130,6 +135,7 @@ struct RegisterBindingAttrs { stringify_error: bool, debug_output: bool, custom_name: Option, + returns_handle: bool, } impl RegisterBindingAttrs { @@ -138,9 +144,16 @@ impl RegisterBindingAttrs { self.debug_output = true; } else if meta.path.is_ident("stringify_error") { self.stringify_error = true; + } else if meta.path.is_ident("returns_handle") { + self.returns_handle = true; } else if meta.path.is_ident("name") { let value: syn::LitStr = meta.value()?.parse()?; self.custom_name = Some(value.value()); + } else { + return Err(meta.error( + "unknown #[register_binding] option, expected one of `debug_output`, \ + `stringify_error`, `returns_handle`, `name = \"..\"`", + )); } Ok(()) } @@ -148,28 +161,69 @@ impl RegisterBindingAttrs { struct ArgOptions { default_value: Option, + /// `#[bind(state)]`: not a JSON argument but a read-only [`StateRef`] over the registry + is_state: bool, + /// `#[bind(state_mut)]`: not a JSON argument but a writable [`StateRefMut`] over the + /// registry. Unlike `state`, this forces the write lock for the whole call. + is_state_mut: bool, + /// `#[bind(handle)]`: a reference to a `CustomRegistryValue`, passed as a registry id. + is_handle: bool, } -fn parse_arg_attributes(attrs: &[Attribute]) -> ArgOptions { +fn parse_arg_attributes(attrs: &[Attribute]) -> syn::parse::Result { let mut opts = ArgOptions { default_value: None, + is_state: false, + is_state_mut: false, + is_handle: false, }; for attr in attrs { if attr.path().is_ident("bind") { - let _ = attr.parse_nested_meta(|meta| { - if meta.path.is_ident("default") { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("state") { + opts.is_state = true; + } else if meta.path.is_ident("state_mut") { + opts.is_state_mut = true; + } else if meta.path.is_ident("handle") { + opts.is_handle = true; + } else if meta.path.is_ident("default") { if meta.input.peek(syn::Token![=]) { let expr: syn::Expr = meta.value()?.parse()?; opts.default_value = Some(expr); } else { opts.default_value = Some(syn::parse_quote!(Default::default())); } + } else { + return Err(meta.error( + "unknown #[bind] option, expected one of `state`, `state_mut`, \ + `handle`, `default`, `default = `", + )); } Ok(()) - }); + })?; } } - opts + Ok(opts) +} + +/// One function argument in the required format for codegen +struct ArgInfo { + name: String, + change_from_ref: bool, + ty_without_ref: Type, + opts: ArgOptions, + /// The big type name, for a `&mut` big-type argument + mut_big_type_name: Option, + /// The referenced type and mutability, for a `#[bind(handle)]` argument + handle_elem: Option<(Type, bool)>, +} + +impl ArgInfo { + /// Whether this argument needs the write lock + /// (like a `&mut` big type) + fn is_mut_handle(&self) -> bool { + matches!(self.handle_elem, Some((_, true))) + } } #[proc_macro_attribute] @@ -206,7 +260,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { }) .collect(); - let args_info: Vec<_> = input_fn + let args_info: syn::parse::Result> = input_fn .sig .inputs .iter() @@ -215,16 +269,34 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { let pat = &pat_type.pat; let ty = &pat_type.ty; - let arg_opts = parse_arg_attributes(&pat_type.attrs); + let arg_opts = parse_arg_attributes(&pat_type.attrs)?; - let arg_name = match &**pat { - Pat::Ident(p) => p.ident.to_string(), - _ => panic!("Simple args only"), - }; + let arg_name = + match &**pat { + Pat::Ident(p) => p.ident.to_string(), + _ => return Err(syn::Error::new_spanned( + pat, + "#[register_binding] needs a plain `name: Type` argument, because the \ + name is what a caller sends the argument under.", + )), + }; let ty_no_life = strip_lifetimes(*ty.clone()); let ty_as_str = quote::quote!(#ty_no_life).to_string(); + // A handle is passed as an id and resolved through `FromContext`, like a big type. + let handle_elem = match (&ty_no_life, arg_opts.is_handle) { + (Type::Reference(r), true) => Some((*r.elem.clone(), r.mutability.is_some())), + (_, true) => { + return Err(syn::Error::new_spanned( + ty, + "#[bind(handle)] is only valid on a reference argument \ + (`&T` or `&mut T`), where `T: CustomRegistryValue`.", + )) + } + (_, false) => None, + }; let change_from_ref = matches!(ty_no_life, Type::Reference(_)) + && !arg_opts.is_handle && !(BIG_TYPES_NAMES.iter().any(|tn| ty_as_str.ends_with(tn))); let type_without_ref = match &ty_no_life { Type::Reference(type_reference) if change_from_ref => { @@ -240,49 +312,154 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { } x => x.clone(), }; - let mut_big_type_name = is_mut_big_type_ref(&ty_no_life); - ( - arg_name, - ty_no_life, + let mut_big_type_name = if handle_elem.is_some() { + None + } else { + is_mut_big_type_ref(&ty_no_life) + }; + Ok(ArgInfo { + name: arg_name, change_from_ref, - type_without_ref, - arg_opts, + ty_without_ref: type_without_ref, + opts: arg_opts, mut_big_type_name, - ) + handle_elem, + }) } - _ => panic!("Self not supported"), + FnArg::Receiver(receiver) => Err(syn::Error::new_spanned( + receiver, + "#[register_binding] is for free functions. A method's `self` has no argument \ + name a caller could send it under.", + )), }) .collect(); - let has_any_mut_big_type = args_info + // Strip `#[bind]` before `#input_fn` is emitted on any path, including the error paths below. + for input in &mut input_fn.sig.inputs { + if let FnArg::Typed(pat_type) = input { + pat_type.attrs.retain(|attr| !attr.path().is_ident("bind")); + } + } + + let args_info = match args_info { + Ok(args_info) => args_info, + Err(e) => { + let err = e.to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } + }; + + // Whether the call needs the registry's write lock rather than its read lock: a `&mut` + // big-type argument, a `#[bind(handle)]` on a `&mut` reference, or `#[bind(state_mut)]` + // all resolve through the same write-locked `__state_guard` in the execution block below. + let needs_write_lock = args_info .iter() - .any(|(.., mut_big_type_name)| mut_big_type_name.is_some()); + .any(|a| a.mut_big_type_name.is_some() || a.is_mut_handle() || a.opts.is_state_mut); + + // A `#[bind(state)]` cannot be combined with anything needing the write lock (conflicting + // ownership/mut borrow of the same registry guard). + if needs_write_lock && args_info.iter().any(|a| a.opts.is_state) { + let err = syn::Error::new_spanned( + &input_fn.sig, + "#[bind(state)] cannot be combined with a `&mut` big-type, `#[bind(handle)]`, or \ + `#[bind(state_mut)]` argument. Return the value instead of taking it by `&mut`.", + ) + .to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } - // 1. Extraction Logic (for non-mut-big-type path) - let extractions = args_info.iter().map(|(name, _ty, is_ref, ty_without_ref, opts, _)| { - let maybe_ref = if *is_ref { + // `#[bind(state_mut)]` already gives full mutable access to every item in the registry, so + // it must be the only thing asking for the write lock: combining it with a `&mut` big-type + // or `#[bind(handle)]` argument would borrow the same guard mutably twice. + let state_mut_count = args_info.iter().filter(|a| a.opts.is_state_mut).count(); + if state_mut_count > 1 { + let err = syn::Error::new_spanned( + &input_fn.sig, + "#[bind(state_mut)] may only be used once per binding.", + ) + .to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } + if state_mut_count == 1 + && args_info + .iter() + .any(|a| !a.opts.is_state_mut && (a.mut_big_type_name.is_some() || a.is_mut_handle())) + { + let err = syn::Error::new_spanned( + &input_fn.sig, + "#[bind(state_mut)] cannot be combined with a `&mut` big-type or `#[bind(handle)]` \ + argument; it already has full mutable access to the registry, so reach that item \ + through `state_mut` instead.", + ) + .to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } + + if needs_write_lock { + if let Some(shared) = args_info.iter().find(|a| { + a.mut_big_type_name.is_none() + && !a.is_mut_handle() + && !a.opts.is_state_mut + && (a.handle_elem.is_some() || is_big_type_ref(&a.ty_without_ref)) + }) { + let name = &shared.name; + let err = syn::Error::new_spanned( + &input_fn.sig, + format!( + "#[register_binding] on a function taking a `&mut` big-type, \ + `#[bind(handle)]`, or `#[bind(state_mut)]` argument cannot also take \ + `{name}` by shared reference. Take it by `&mut` too, or take it by value." + ), + ) + .to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } + } + + // 1. Extraction Logic (for the read-locked path only; a `#[bind(state_mut)]` argument + // forces `needs_write_lock`, so this map's default arm never runs for one; it is handled in + // the write-locked `execution_block` below instead.) + let extractions = args_info.iter().map(|a| { + let (name, is_ref, ty_without_ref, opts) = (&a.name, a.change_from_ref, &a.ty_without_ref, &a.opts); + if opts.is_state { + return quote! { ::process_mining::bindings::StateRef::new(state) }; + } + let maybe_ref = if is_ref { quote! {&} } else { quote! {} }; if let Some(default_expr) = &opts.default_value { quote! { - #maybe_ref crate::bindings::extract_param::<#ty_without_ref>(arg_map, #name, state, || Some(#default_expr))? + #maybe_ref ::process_mining::bindings::extract_param::<#ty_without_ref>(arg_map, #name, state, || Some(#default_expr))? } } else { quote! { - #maybe_ref crate::bindings::extract_param::<#ty_without_ref>(arg_map, #name, state, || None)? + #maybe_ref ::process_mining::bindings::extract_param::<#ty_without_ref>(arg_map, #name, state, || None)? } } }); // 2. Schema Logic - let schema_gens = args_info.iter().map(|(name, _ty, _is_ref, ty_without_ref, _, _)| { - if is_big_type_ref(ty_without_ref) { + let schema_gens = args_info.iter().map(|a| { + let (name, ty_without_ref, opts) = (&a.name, &a.ty_without_ref, &a.opts); + if opts.is_state || opts.is_state_mut { + return quote! {}; + } + if let Some((elem, _)) = &a.handle_elem { + quote! { + args_schema.push((#name.to_string(), ::process_mining::__private::serde_json::json!({ + "type": "string", + "title": <#elem as ::process_mining::bindings::CustomRegistryValue>::kind_name(), + "x-registry-ref": <#elem as ::process_mining::bindings::CustomRegistryValue>::kind_name(), + "x-widget": "entity-selector" + }))); + } + } else if is_big_type_ref(ty_without_ref) { let ty_str = quote::quote!(#ty_without_ref).to_string(); let type_name = longest_big_type_match(&ty_str).unwrap(); quote! { - args_schema.push((#name.to_string(), serde_json::json!({ + args_schema.push((#name.to_string(), ::process_mining::__private::serde_json::json!({ "type": "string", "title": #type_name, "x-registry-ref": #type_name, @@ -290,7 +467,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { }))); } } else { - quote! { args_schema.push((#name.to_string(), serde_json::to_value(schemars::schema_for!(#ty_without_ref)).unwrap())); } + quote! { args_schema.push((#name.to_string(), ::process_mining::__private::serde_json::to_value(::process_mining::__private::schemars::schema_for!(#ty_without_ref)).unwrap())); } } }); @@ -303,6 +480,16 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { // Strip lifetimes from return type let mut ret_type = strip_lifetimes(raw_ret_type); + if attrs.returns_handle && attrs.debug_output { + let err = syn::Error::new_spanned( + &input_fn.sig, + "#[register_binding(returns_handle)] cannot be combined with `debug_output`: \ + one stores the value and returns its id, the other formats it away.", + ) + .to_compile_error(); + return TokenStream::from(quote! { #input_fn #err }); + } + // If debug_output is set, the actual return type is String if attrs.debug_output { ret_type = syn::parse_quote!(String); @@ -322,50 +509,83 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { } } + // Handling the two ways a result ends up in the registry instead of being serialized + let returns_registry_handle = attrs.returns_handle || is_big_type(&ret_type).is_some(); + + let output_id_schema = if returns_registry_handle { + quote! { + args_schema.push((#OUTPUT_ID_ARG.to_string(), ::process_mining::__private::serde_json::json!({ + "type": ["string", "null"], + "title": #OUTPUT_ID_ARG, + "description": "Store the result under this id instead of a generated one." + }))); + } + } else { + quote! {} + }; + + let output_id_extraction = if returns_registry_handle { + quote! { + let __output_id = ::process_mining::bindings::extract_param_json::>(arg_map, #OUTPUT_ID_ARG, || Some(None))?; + } + } else { + quote! {} + }; + + let stored_id = quote! { + let id = __output_id + .unwrap_or_else(|| format!("res_{}", ::process_mining::__private::uuid::Uuid::new_v4())); + }; + + // A `#[bind(state)]`/`#[bind(state_mut)]` argument has no schema, so requiring it by name + // would tell a host to demand an argument it cannot describe and the caller cannot supply. let required_arg_names = args_info .iter() - .filter(|(_, _, _, _, opts, _)| opts.default_value.is_none()) - .map(|(name, _, _, _, _, _)| name); + .filter(|a| a.opts.default_value.is_none() && !a.opts.is_state && !a.opts.is_state_mut) + .map(|a| &a.name); // 4. Generate the Execution Logic let extractions: Vec<_> = extractions.collect(); + // Apply error handling if requested, independent of whether the return type is a big type or not. + let error_handling = if attrs.stringify_error && !attrs.debug_output { + quote! { let result = result.map_err(|e| e.to_string())?; } + } else { + quote! {} + }; + let serialization_logic = if attrs.debug_output { quote! { let final_result = format!("{:?}", result); - serde_json::to_vec(&final_result).map_err(|e| e.to_string()) - } - } else if attrs.stringify_error { - quote! { - let ok_result = result.map_err(|e| e.to_string())?; - serde_json::to_vec(&ok_result).map_err(|e| e.to_string()) + ::process_mining::__private::serde_json::to_vec(&final_result).map_err(|e| e.to_string()) } } else { quote! { - serde_json::to_vec(&result).map_err(|e| e.to_string()) + ::process_mining::__private::serde_json::to_vec(&result).map_err(|e| e.to_string()) } }; - let execution_block = if has_any_mut_big_type { + let execution_block = if needs_write_lock { // Mutable big type path: use write lock // 1. Generate JSON extractions for non-mut-big-type params (no state needed) let json_extractions: Vec<_> = args_info .iter() - .filter(|(.., mut_name)| mut_name.is_none()) - .map(|(name, _ty, is_ref, ty_without_ref, opts, _)| { + .filter(|a| a.mut_big_type_name.is_none() && !a.is_mut_handle() && !a.opts.is_state_mut) + .map(|a| { + let (name, is_ref, ty_without_ref, opts) = (&a.name, a.change_from_ref, &a.ty_without_ref, &a.opts); let param_ident = format_ident!("__param_{}", name); - let maybe_ref = if *is_ref { + let maybe_ref = if is_ref { quote! { & } } else { quote! {} }; if let Some(default_expr) = &opts.default_value { quote! { - let #param_ident = #maybe_ref crate::bindings::extract_param_json::<#ty_without_ref>(arg_map, #name, || Some(#default_expr))?; + let #param_ident = #maybe_ref ::process_mining::bindings::extract_param_json::<#ty_without_ref>(arg_map, #name, || Some(#default_expr))?; } } else { quote! { - let #param_ident = #maybe_ref crate::bindings::extract_param_json::<#ty_without_ref>(arg_map, #name, || None)?; + let #param_ident = #maybe_ref ::process_mining::bindings::extract_param_json::<#ty_without_ref>(arg_map, #name, || None)?; } } }) @@ -374,9 +594,28 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { // 2. Generate mutable big type extractions from state let mut_extractions: Vec<_> = args_info .iter() - .filter_map(|(name, _ty, _is_ref, _ty_without_ref, _opts, mut_name)| { - let type_name = mut_name.as_ref()?; + .filter_map(|a| { + let name = &a.name; let param_ident = format_ident!("__param_{}", name); + if a.opts.is_state_mut { + return Some(quote! { + let #param_ident = ::process_mining::bindings::StateRefMut::new(&mut __state_guard); + }); + } + if let Some((elem, true)) = &a.handle_elem { + return Some(quote! { + let #param_ident = { + let __id = arg_map.get(#name).and_then(|v| v.as_str()) + .ok_or_else(|| format!("Missing required argument {}", #name))?; + __state_guard.get_mut(__id) + .ok_or_else(|| format!("Item '{}' not found", __id))? + .as_custom_mut::<#elem>() + .ok_or_else(|| format!("ID '{}' is not a {}", __id, + <#elem as ::process_mining::bindings::CustomRegistryValue>::kind_name()))? + }; + }); + } + let type_name = a.mut_big_type_name.as_ref()?; let variant_ident = format_ident!("{}", type_name); Some(quote! { let #param_ident = { @@ -384,7 +623,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { .ok_or_else(|| format!("Missing required argument {}", #name))?; match __state_guard.get_mut(__id) .ok_or_else(|| format!("Item '{}' not found", __id))? { - crate::bindings::RegistryItem::#variant_ident(inner) => inner, + ::process_mining::bindings::RegistryItem::#variant_ident(inner) => inner, _ => return Err(format!("ID '{}' is not a {}", __id, #type_name)), } }; @@ -395,18 +634,27 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { // 3. Generate call arguments in original order let call_args: Vec<_> = args_info .iter() - .map(|(name, ..)| { - let param_ident = format_ident!("__param_{}", name); + .map(|a| { + let param_ident = format_ident!("__param_{}", a.name); quote! { #param_ident } }) .collect(); - let mut_serialization = if let Some(type_name) = is_big_type(&ret_type) { + // The write guard is still live here, and `std::sync::RwLock` is not reentrant, so this + // has to insert through the guard rather than call `state_lock.add`, which would take the + // lock again and deadlock. + let mut_serialization = if attrs.returns_handle { + quote! { + #stored_id + __state_guard.insert(id.clone(), ::process_mining::bindings::RegistryItem::custom(result)); + ::process_mining::__private::serde_json::to_vec(&id).map_err(|e| e.to_string()) + } + } else if let Some(type_name) = is_big_type(&ret_type) { let variant_ident = format_ident!("{}", type_name); quote! { - let id = format!("res_{}", uuid::Uuid::new_v4()); - __state_guard.insert(id.clone(), crate::bindings::RegistryItem::#variant_ident(result)); - serde_json::to_vec(&id).map_err(|e| e.to_string()) + #stored_id + __state_guard.insert(id.clone(), ::process_mining::bindings::RegistryItem::#variant_ident(result)); + ::process_mining::__private::serde_json::to_vec(&id).map_err(|e| e.to_string()) } } else { serialization_logic.clone() @@ -414,35 +662,59 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { quote! { #(#json_extractions)* - let mut __state_guard = state_lock.items.write().map_err(|e| e.to_string())?; + let mut __state_guard = state_lock.write(); #(#mut_extractions)* let result = #fn_ident( #(#call_args),* ); + #error_handling #mut_serialization } + } else if attrs.returns_handle { + quote! { + let result = { + let state_guard = state_lock.read(); + let state = &*state_guard; + #fn_ident( #(#extractions),* ) + }; + #error_handling + #stored_id + state_lock.add(&id, ::process_mining::bindings::RegistryItem::custom(result)); + ::process_mining::__private::serde_json::to_vec(&id).map_err(|e| e.to_string()) + } } else if let Some(type_name) = is_big_type(&ret_type) { let variant_ident = format_ident!("{}", type_name); quote! { let result = { - let state_guard = state_lock.items.read().map_err(|e| e.to_string())?; + let state_guard = state_lock.read(); let state = &*state_guard; #fn_ident( #(#extractions),* ) }; - let id = format!("res_{}", uuid::Uuid::new_v4()); - state_lock.add(&id, crate::bindings::RegistryItem::#variant_ident(result)); - serde_json::to_vec(&id).map_err(|e| e.to_string()) + #error_handling + #stored_id + state_lock.add(&id, ::process_mining::bindings::RegistryItem::#variant_ident(result)); + ::process_mining::__private::serde_json::to_vec(&id).map_err(|e| e.to_string()) } } else { quote! { - let state_guard = state_lock.items.read().map_err(|e| e.to_string())?; + let state_guard = state_lock.read(); let state = &*state_guard; let result = #fn_ident( #(#extractions),* ); + #error_handling #serialization_logic } }; - let ret_type_schema = if let Some(type_name) = is_big_type(&ret_type) { + let ret_type_schema = if attrs.returns_handle { + quote! { + ::process_mining::__private::serde_json::json!({ + "type": "string", + "title": <#ret_type as ::process_mining::bindings::CustomRegistryValue>::kind_name(), + "x-registry-ref": <#ret_type as ::process_mining::bindings::CustomRegistryValue>::kind_name(), + "x-widget": "entity-selector" + }) + } + } else if let Some(type_name) = is_big_type(&ret_type) { quote! { - serde_json::json!({ + ::process_mining::__private::serde_json::json!({ "type": "string", "title": #type_name, "x-registry-ref": #type_name, @@ -451,15 +723,22 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { } } else { quote! { - serde_json::to_value(schemars::schema_for!(#ret_type)).unwrap() + ::process_mining::__private::serde_json::to_value(::process_mining::__private::schemars::schema_for!(#ret_type)).unwrap() } }; - // Strip #[bind] attributes - for input in &mut input_fn.sig.inputs { - if let FnArg::Typed(pat_type) = input { - pat_type.attrs.retain(|attr| !attr.path().is_ident("bind")); - } + // Shadowing the caller's own argument would silently change what the binding is passed, so + // refuse instead. The function itself is still emitted, to keep its call sites resolving. + if returns_registry_handle && args_info.iter().any(|a| a.name == OUTPUT_ID_ARG) { + let msg = format!( + "#[register_binding] on `{}`: its result is stored in the registry, so the binding \ + already takes an `{}` argument. Rename the function's own argument.", + fn_ident, OUTPUT_ID_ARG + ); + return TokenStream::from(quote! { + #input_fn + ::core::compile_error!(#msg); + }); } let docs_fn_name = format_ident!("{}_docs", fn_ident); @@ -467,17 +746,25 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { let required_args_fn_name = format_ident!("{}_required_args", fn_ident); let return_type_fn_name = format_ident!("{}_return_type", fn_ident); + // `cfg(feature = "bindings")` resolves in the crate being compiled, so a downstream crate with + // no feature by that name would silently register nothing. + let registration_cfg = if std::env::var("CARGO_PKG_NAME").as_deref() == Ok("process_mining") { + quote! { #[cfg(feature = "bindings")] } + } else { + quote! {} + }; + let expanded = quote! { #input_fn - #[cfg(feature = "bindings")] + #registration_cfg const _: () = { - use crate::bindings::{Binding, AppState}; - use serde_json::Value; - use std::sync::RwLock; + use ::process_mining::bindings::{Binding, AppState}; + use ::process_mining::__private::serde_json::Value; fn #wrapper_name(args: &Value, state_lock: &AppState) -> Result, String> { let arg_map = args.as_object().ok_or("Args must be JSON object")?; + #output_id_extraction #execution_block } @@ -488,6 +775,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { fn #args_fn_name() -> Vec<(String, Value)> { let mut args_schema = ::std::vec::Vec::new(); #(#schema_gens)* + #output_id_schema args_schema } @@ -499,7 +787,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { #ret_type_schema } - inventory::submit! { + ::process_mining::__private::inventory::submit! { Binding { id: concat!(module_path!(), "::", stringify!(#fn_ident)), name: #binding_name_str, @@ -529,6 +817,36 @@ pub fn big_types_list(_item: TokenStream) -> TokenStream { TokenStream::from(expanded) } +/// Let a `CustomRegistryValue` implementor be taken by `#[bind(handle)] &T`. +/// +/// Generates the `FromContext` impl that turns the incoming id into a borrow of the stored value. +/// +/// Unlike [`macro@RegistryEntity`], the impl is not behind `#[cfg(feature = "bindings")]`: that +/// cfg is evaluated in the deriving crate's feature namespace, and a downstream crate need not +/// have a feature of that name. +#[proc_macro_derive(CustomRegistryEntity)] +pub fn derive_custom_registry_entity(item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as syn::DeriveInput); + let name = &input.ident; + + let expanded = quote! { + impl<'a> ::process_mining::bindings::FromContext<'a> for &'a #name { + fn from_context(value: &::process_mining::__private::serde_json::Value, state: &'a ::process_mining::bindings::InnerAppState) -> Result { + let id = value.as_str().ok_or("Expected String ID")?; + let item = state.get(id).ok_or_else(|| format!("Item '{}' not found", id))?; + item.as_custom::<#name>().ok_or_else(|| { + format!( + "ID '{}' is not a {}", + id, + <#name as ::process_mining::bindings::CustomRegistryValue>::kind_name() + ) + }) + } + } + }; + TokenStream::from(expanded) +} + #[proc_macro_derive(RegistryEntity)] pub fn derive_registry_entity(item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as syn::DeriveInput); @@ -537,12 +855,12 @@ pub fn derive_registry_entity(item: TokenStream) -> TokenStream { let expanded = quote! { #[cfg(feature = "bindings")] - impl<'a> crate::bindings::FromContext<'a> for &'a #name { - fn from_context(value: &serde_json::Value, state: &'a crate::bindings::InnerAppState) -> Result { + impl<'a> ::process_mining::bindings::FromContext<'a> for &'a #name { + fn from_context(value: &::process_mining::__private::serde_json::Value, state: &'a ::process_mining::bindings::InnerAppState) -> Result { let id = value.as_str().ok_or("Expected String ID")?; let item = state.get(id).ok_or_else(|| format!("Item '{}' not found", id))?; - if let crate::bindings::RegistryItem::#name(inner) = item { + if let ::process_mining::bindings::RegistryItem::#name(inner) = item { Ok(inner) } else { Err(format!("ID '{}' is not a {}", id, #name_str)) diff --git a/process_mining/Cargo.toml b/process_mining/Cargo.toml index 09d88bab..2453d913 100644 --- a/process_mining/Cargo.toml +++ b/process_mining/Cargo.toml @@ -16,11 +16,14 @@ rust-version = "1.88" [dependencies] macros_process_mining = { version = "0.6.0", path = "../macros_process_mining" } chrono = { version = "0.4.40", features = ["serde"] } -duckdb = { version = "1.2.1", optional = true, features = ["chrono"]} +# `bundled` builds DuckDB from source, as `rusqlite` above does for SQLite: no system +# `libduckdb` to install, no version to keep in step with this dependency, and the result is +# statically linked, so nothing has to be findable at run time. Costs a few minutes and ~3 GB +# on a clean build, and needs a C++ compiler. +duckdb = { version = "1.2.1", optional = true, features = ["chrono", "bundled"] } flate2 = "1.1.1" graphviz-rust = { version = "0.9.3", optional = true } itertools = { version = "0.14.0" } -kuzu = {version = "=0.11.2", optional = true} nalgebra = { version = "0.33.2", optional = true } ordered-float = "5.0.0" petgraph = "0.8.1" @@ -28,7 +31,8 @@ polars = { version = "0.51.0", features = ["dtype-slim", "timezones", "partition quick-xml = { version = "0.37.4" } rand = { version = "0.9.1", optional = true } rayon = "1.7.0" -rusqlite = { version = "0.38.0", features = ["bundled","chrono", "serialize"], optional = true } +regex = { version = "1.11", optional = true } +rusqlite = { version = "0.38.0", features = ["bundled","chrono", "serialize", "column_decltype"], optional = true } serde_json = "1.0.105" serde = {version = "1.0.188", features = ["derive"]} serde_with = { version = "3.16.0", features = ["std", "macros", "schemars_1"]} @@ -36,18 +40,32 @@ tempfile = "3" uuid = {version = "1.16.0", features = ["v4", "serde"]} schemars = { version = "1.1.0", features = ["chrono04", "uuid1"]} inventory = { version = "0.3", optional = true } +# Only for the `bytes` form of a registry-reference argument, i.e. how a host with no filesystem +# hands a log over. +base64 = { version = "0.22", optional = true } csv = "1.4.0" +# Backends are selected by this crate's own `extraction-dbcon*` features, since not all of them are +# portable. `dbcon`'s `default` is `[]`, and a `dbcon` with no backend recognises no connection +# string, so at least one must be selected. +dbcon = {version = "0.4.0", optional = true } hashbrown = "0.17.1" rustc-hash = "2.1.2" +zip = { version = "6", default-features = false, features = ["deflate"], optional = true } +# `default-features = false` drops the unused `arrow` integration; the bundled format's column +# types are written through the low-level row-group writer. `zstd` is needed because `parquet` +# defaults to no block compression, which makes an exported container several times larger than the +# same log as CSV. `snap` is for reading: Snappy is what most other parquet writers emit. +parquet = { version = "58", default-features = false, features = [ + "zstd", + "snap", +], optional = true } +# Only to hand `parquet` an in-memory `ChunkReader`; `parquet` already depends on it. +bytes = { version = "1", optional = true } [dev-dependencies] criterion = { version = "0.5.1", features = ["html_reports"] } dhat = "0.3.3" -[build-dependencies] -# Pin cxx-build to match kuzu's cxx; bump this together with the kuzu dependency. -cxx-build = { version = "=1.0.138", optional = true } - [features] # Enables exporting Petri nets as PNG/SVG/... images using graphviz. # Note: graphviz needs to be locally installed to the PATH for the image export functions to work. @@ -58,12 +76,48 @@ graphviz-export = ["dep:graphviz-rust"] ocel-sqlite = ["dep:rusqlite"] ocel-duckdb = ["dep:duckdb"] +# Enables the relational-to-OCEL extraction blueprint model, validation, SQL compiler, extractor +# and sinks. Carries no connector of its own: `ocel-sqlite` brings `SqliteRowProvider`, and +# `extraction-dbcon` brings the `PostgreSQL`/CSV/Parquet one. +extraction-blueprint = ["dep:regex"] + +# Adds `DbconRowProvider`, a `RowProvider` over the file-backed sources `dbcon` reads (SQLite, CSV +# and Parquet), and the extraction bindings that open one. +# +# Excludes PostgreSQL, which is `extraction-dbcon-postgres` below. These three build for +# `wasm32-unknown-unknown` (`dbcon` substitutes `sqlite-wasm-rs` there) and can be read from bytes +# with no filesystem, as a browser needs. Bundling PostgreSQL here would pull `sqlx` and its +# runtime into every build that only wants to read a dropped CSV. +extraction-dbcon = [ + "dep:dbcon", + "extraction-blueprint", + "dbcon/sqlite", + "dbcon/csv", + "dbcon/parquet", + # Pure Rust and byte-readable, so it belongs with the wasm-capable three. + "dbcon/xlsx", +] + +# Adds PostgreSQL to `extraction-dbcon`. Separate because `sqlx` does not build for +# `wasm32-unknown-unknown`, and because a server connection is meaningless from a browser page. +extraction-dbcon-postgres = ["extraction-dbcon", "dbcon/postgres"] + +# Adds DuckDB to `extraction-dbcon`. Separate for the same reason as PostgreSQL: the `duckdb` crate +# builds native code and does not build for `wasm32`. `dbcon` bundles DuckDB just as this crate +# does, so both paths to the shared `duckdb` crate agree on how it is built. +extraction-dbcon-duckdb = ["extraction-dbcon", "dbcon/duckdb"] + +# Enables reading and writing the OCEL 2.0 bundled CSV/Parquet format: a `.ocel.zip` archive or +# a directory with the same layout, with CSV storage. +ocel-bundle = ["dep:zip"] + +# Adds Parquet storage to `ocel-bundle`. Separate because `parquet` is a large dependency and +# a CSV container needs none of it. +ocel-bundle-parquet = ["ocel-bundle", "dep:parquet", "dep:bytes"] + # Enables polars DataFrame conversion from/to event data structs dataframes = ["dep:polars"] -# Enables kuzudb features for OCEL (e.g., exporting OCEL to kuzudb database) -kuzudb = ["dep:kuzu", "dep:cxx-build"] - # Enables event log splitting (+rand dependency) log-splitting = ["dep:rand"] @@ -72,17 +126,30 @@ token-based-replay = ["dep:nalgebra"] # Enable bindings bindings = [ - "dep:inventory" + "dep:inventory", + "dep:base64" ] -all = ["graphviz-export","ocel-sqlite","ocel-duckdb","dataframes","kuzudb", "log-splitting", "token-based-replay", "bindings" ] +all = ["graphviz-export","ocel-sqlite","ocel-duckdb","extraction-blueprint","extraction-dbcon","extraction-dbcon-postgres","extraction-dbcon-duckdb","ocel-bundle","ocel-bundle-parquet","dataframes", "log-splitting", "token-based-replay", "bindings" ] [package.metadata.docs.rs] -all-features = true - -[[example]] -name = "ocel_kuzudb_export" -required-features = ["dataframes", "kuzudb"] +# The same set as `all`, listed explicitly so a feature docs.rs cannot build can be dropped +# individually. +features = [ + "graphviz-export", + "ocel-sqlite", + "ocel-duckdb", + "extraction-blueprint", + "extraction-dbcon", + "extraction-dbcon-postgres", + "extraction-dbcon-duckdb", + "ocel-bundle", + "ocel-bundle-parquet", + "dataframes", + "log-splitting", + "token-based-replay", + "bindings", +] [[example]] name = "ocel_duckdb_export" @@ -109,3 +176,16 @@ harness = false [[bench]] name = "alignments" harness = false + +[[bench]] +name = "ocel_import" +harness = false +required-features = ["ocel-duckdb"] + +[[example]] +name = "ocel_stream_to_duckdb" +required-features = ["ocel-duckdb"] + +[[example]] +name = "ocel_dataset_crosscheck" +required-features = ["ocel-bundle-parquet"] diff --git a/process_mining/README.md b/process_mining/README.md index 48e09353..bb90b11f 100644 --- a/process_mining/README.md +++ b/process_mining/README.md @@ -20,7 +20,7 @@ You can find various usage examples in the [`examples/`](examples/) directory, c - Importing and analyzing XES event logs (`event_log_stats.rs`) - Working with OCEL 2.0 data (`ocel_stats.rs`) - Process discovery (`process_discovery.rs`) -- Exporting to DuckDB/KuzuDB (`ocel_duckdb_export.rs`, `ocel_kuzudb_export.rs`) +- Exporting to `DuckDB` (`ocel_duckdb_export.rs`) To run an example: ```bash diff --git a/process_mining/benches/common/mod.rs b/process_mining/benches/common/mod.rs new file mode 100644 index 00000000..a9029afa --- /dev/null +++ b/process_mining/benches/common/mod.rs @@ -0,0 +1,17 @@ +//! Fixtures shared by the OCEL benchmarks. +//! +//! A directory module rather than `benches/common.rs`, which cargo would take for a bench target. + +// Each bench pulls in the whole module but uses only the part it needs. +#![allow(dead_code)] + +use std::path::PathBuf; + +use process_mining::test_utils::get_test_data_path; + +/// The order-management log in the given serialization (`json`, `xml`, ...). +pub fn order_management(ext: &str) -> PathBuf { + get_test_data_path() + .join("ocel") + .join(format!("order-management.{ext}")) +} diff --git a/process_mining/benches/ocel_import.rs b/process_mining/benches/ocel_import.rs new file mode 100644 index 00000000..c75a1e2f --- /dev/null +++ b/process_mining/benches/ocel_import.rs @@ -0,0 +1,62 @@ +//! Benchmark OCEL 2.0 import speed across importers, per source format: +//! - `ocel_direct`: `OCEL::import_from_path`, materializing the full log in memory +//! - `slim_streaming`: `SlimLinkedOCEL::import_from_path` +//! - `duckdb_*`: `stream_ocel_file_to_duckdb`, on-disk (`duckdb_raw` streaming-append, +//! `duckdb_default` adding compression, cluster-by-key optimize and index building) and +//! in-memory (`duckdb_mem_*`) +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use process_mining::{ + core::event_data::object_centric::{ + linked_ocel::SlimLinkedOCEL, + ocel_sql::{stream_ocel_file_to_duckdb_with, DuckDbImportOptions}, + }, + Importable, OCEL, +}; + +mod common; + +fn bench_format(c: &mut Criterion, ext: &str) { + let src = common::order_management(ext); + // The importer recreates its target, so one path serves every iteration. + let tmp_dir = tempfile::tempdir().expect("create temp dir"); + let out = tmp_dir.path().join(format!("bench_import_{ext}.duckdb")); + let opt_default = DuckDbImportOptions::default(); + let opt_raw = DuckDbImportOptions { + compression: false, + optimize_filesize: false, + }; + + let mut g = c.benchmark_group(format!("ocel_import_{ext}")); + g.sample_size(10); // multi-MB fixtures: keep the sample count low + + g.bench_function("ocel_direct", |b| { + b.iter(|| black_box(OCEL::import_from_path(black_box(&src)).unwrap())) + }); + g.bench_function("slim_streaming", |b| { + b.iter(|| black_box(SlimLinkedOCEL::import_from_path(black_box(&src)).unwrap())) + }); + g.bench_function("duckdb_default", |b| { + b.iter(|| stream_ocel_file_to_duckdb_with(&src, &out, &opt_default).unwrap()) + }); + g.bench_function("duckdb_raw", |b| { + b.iter(|| stream_ocel_file_to_duckdb_with(&src, &out, &opt_raw).unwrap()) + }); + // In-memory DuckDB (":memory:") isolates engine cost from disk I/O; a fresh in-memory + // database per iteration, discarded after. + let mem = std::path::Path::new(":memory:"); + g.bench_function("duckdb_mem_default", |b| { + b.iter(|| stream_ocel_file_to_duckdb_with(&src, mem, &opt_default).unwrap()) + }); + g.bench_function("duckdb_mem_raw", |b| { + b.iter(|| stream_ocel_file_to_duckdb_with(&src, mem, &opt_raw).unwrap()) + }); + g.finish(); +} + +fn bench_import(c: &mut Criterion) { + bench_format(c, "json"); + bench_format(c, "xml"); +} + +criterion_group!(benches, bench_import); +criterion_main!(benches); diff --git a/process_mining/examples/README.md b/process_mining/examples/README.md index 50842a8f..a3da910c 100644 --- a/process_mining/examples/README.md +++ b/process_mining/examples/README.md @@ -36,7 +36,3 @@ This folder contains example usages of the `process_mining` crate. cargo run --example ocel_duckdb_export -- ``` -- **`ocel_kuzudb_export.rs`**: Imports an OCEL and exports it to a KuzuDB graph database. - ```bash - cargo run --example ocel_kuzudb_export -- - ``` diff --git a/process_mining/examples/ocel_dataset_crosscheck.rs b/process_mining/examples/ocel_dataset_crosscheck.rs new file mode 100644 index 00000000..944e595e --- /dev/null +++ b/process_mining/examples/ocel_dataset_crosscheck.rs @@ -0,0 +1,1430 @@ +//! Cross-check every re-exported form of a log against the source it was written from. +//! +//! Walks a directory of dataset folders. Each folder keeps the log it started from under +//! `source/` and the re-exported forms beside it; every re-export is read back and compared +//! against that source. Exits non-zero when any comparison fails. +//! +//! # Layout it expects +//! +//! ```text +//! //source/ the reference, in whatever format it arrived in +//! //.ocel.zip the re-exports, in any of the formats below +//! //.ocel.csv +//! //.json +//! //.xml +//! //.sqlite +//! ``` +//! +//! Dataset folders, their source and their re-exports are all found by scanning, so no dataset +//! has to be named here. A file whose extension names no OCEL format is ignored, a plain `.zip` +//! among them: an archive is read as a bundle only under the `.ocel.zip` name, so a zipped copy +//! of a source log is not mistaken for one. +//! +//! # Usage +//! +//! ```bash +//! cargo run --release --features ocel-bundle-parquet,ocel-sqlite \ +//! --example ocel_dataset_crosscheck -- --root /path/to/ocel2-reexported-all +//! +//! # one dataset, no size ceiling, more example diffs +//! cargo run --release --features ocel-bundle-parquet,ocel-sqlite \ +//! --example ocel_dataset_crosscheck -- --only logistics --max-mb 0 --max-diffs 25 +//! ``` +//! +//! A format whose feature is not enabled is skipped rather than reported as a failed import, so +//! the run is still meaningful without `ocel-sqlite` or `ocel-duckdb`. +//! +//! # What is compared +//! +//! Each log is first checked on its own, for the defects a pairwise diff structurally cannot +//! see: duplicate ids, relations pointing at an object that does not exist, events or objects of +//! an undeclared type, and attributes their type never declares. Each re-export then runs +//! against the source through events, objects, type declarations, E2O and O2O relations, and +//! every attribute observation. +//! +//! Attribute differences are split into three classes: +//! +//! - `VALUE`: the values really differ. +//! - `TYPE`: the values render the same but sit in different [`OCELAttributeValue`] variants, +//! e.g. `Integer(5)` against `Float(5.0)` against `String("5")`. +//! - `TIME`: an object attribute carries the same value at a different point in time. +//! +//! A [`Policy`] decides which of those classes make a check fail, and a pair is held to what +//! both of its formats can carry. Most formats are lossless and tolerate nothing. The flat CSV +//! carries no attribute type information and dates an undated attribute row to the UNIX epoch, +//! so `TYPE` and `TIME` differences are reported for it but tolerated. + +#[cfg(not(feature = "ocel-bundle-parquet"))] +fn main() { + eprintln!( + "This example needs the `ocel-bundle-parquet` feature:\n \ + cargo run --release --features ocel-bundle-parquet --example ocel_dataset_crosscheck" + ); + std::process::exit(2); +} + +#[cfg(feature = "ocel-bundle-parquet")] +fn main() -> std::process::ExitCode { + imp::run() +} + +#[cfg(feature = "ocel-bundle-parquet")] +mod imp { + use std::collections::{BTreeMap, HashMap, HashSet}; + use std::path::{Path, PathBuf}; + use std::process::ExitCode; + use std::time::Instant; + + use process_mining::core::event_data::object_centric::{OCELAttributeValue, OCELType, OCEL}; + use process_mining::core::event_data::timestamp_utils::parse_timestamp; + use process_mining::core::io::Importable; + + // Configuration + + /// Directory holding one folder per dataset. + const DEFAULT_ROOT: &str = "ocel-datasets"; + + /// Subdirectory of a dataset folder holding the log its re-exports were written from. + const SOURCE_DIR: &str = "source"; + + /// Inputs larger than this are skipped unless `--max-mb` says otherwise. `0` means no + /// ceiling. + const DEFAULT_MAX_INPUT_MB: u64 = 100; + + /// How many example differences to print per failing check. + const DEFAULT_MAX_DIFF_SAMPLES: usize = 5; + + /// A format this example can read. The declaration order is also the preference among + /// several candidates for the source of one dataset: the most faithful comes first. + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + enum Format { + Bundle, + Json, + Xml, + Sqlite, + DuckDb, + Csv, + } + + impl Format { + /// The format `path` is named for, or `None` when its extension names none. + /// + /// A bare `.zip` names none on purpose: an archive is the bundled format only under the + /// `.ocel.zip` name, so a zipped copy of a source log sitting next to the log itself is + /// left alone rather than read as a bundle and reported as a broken one. + fn of(path: &Path) -> Option { + let name = path.file_name()?.to_string_lossy().to_lowercase(); + // A compressed log is the same format underneath, and the importer unwraps it. + let name = name.strip_suffix(".gz").unwrap_or(&name); + if name.ends_with(".ocel.zip") || name.ends_with(".ocel") { + Some(Format::Bundle) + } else if name.ends_with(".csv") { + Some(Format::Csv) + } else if name.ends_with(".json") || name.ends_with(".jsonocel") { + Some(Format::Json) + } else if name.ends_with(".xml") || name.ends_with(".xmlocel") { + Some(Format::Xml) + } else if name.ends_with(".sqlite") || name.ends_with(".db") { + Some(Format::Sqlite) + } else if name.ends_with(".duckdb") { + Some(Format::DuckDb) + } else { + None + } + } + + fn label(self) -> &'static str { + match self { + Format::Bundle => "bundle", + Format::Json => "json", + Format::Xml => "xml", + Format::Sqlite => "sqlite", + Format::DuckDb => "duckdb", + Format::Csv => "csv", + } + } + + /// The feature a format needs, when it needs one that this build may not have. + fn needs_feature(self) -> Option<&'static str> { + match self { + Format::Sqlite if !cfg!(feature = "ocel-sqlite") => Some("ocel-sqlite"), + Format::DuckDb if !cfg!(feature = "ocel-duckdb") => Some("ocel-duckdb"), + _ => None, + } + } + + fn policy(self) -> Policy { + match self { + Format::Csv => CSV_TOLERANT, + _ => STRICT, + } + } + } + + /// Which classes of attribute difference make a check fail. + #[derive(Clone, Copy)] + struct Policy { + /// Compare the declared `type` of each attribute in the event/object type definitions. + declared_types: bool, + /// Fail on values that render alike but sit in different variants. + value_types: bool, + /// Fail on object attribute values recorded at a different time. + value_times: bool, + } + + impl Policy { + /// The weaker of two policies, so a pair is only held to what both of its formats carry. + fn and(self, other: Policy) -> Policy { + Policy { + declared_types: self.declared_types && other.declared_types, + value_types: self.value_types && other.value_types, + value_times: self.value_times && other.value_times, + } + } + } + + /// Most formats are lossless, so nothing is tolerated. + const STRICT: Policy = Policy { + declared_types: true, + value_types: true, + value_times: true, + }; + + /// The flat CSV declares no attribute types and infers them from the text, and dates an + /// attribute row with an empty `timestamp` to the UNIX epoch. Both are reported, neither + /// fails the run. + const CSV_TOLERANT: Policy = Policy { + declared_types: false, + value_types: false, + value_times: true, + }; + + // Finding the datasets + + /// One dataset folder: the log under `source/`, and the re-exports beside it. + struct Dataset { + name: String, + /// The reference log, or `None` when `source/` holds nothing this build can read. + source: Option, + /// Why there is no source, for the line that reports the skip. + source_note: String, + derived: Vec, + } + + /// Every subdirectory of `root`, in name order, with its source and its re-exports. + fn discover(root: &Path) -> Result, String> { + let mut folders: Vec = entries(root)?.filter(|p| p.is_dir()).collect(); + folders.sort(); + Ok(folders.iter().map(|dir| dataset_at(dir)).collect()) + } + + fn dataset_at(dir: &Path) -> Dataset { + let name = dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + + let source_dir = dir.join(SOURCE_DIR); + // Sorted by format first, so a folder holding the same log twice is read through the + // most faithful of the two. + let mut candidates: Vec<(Format, PathBuf)> = entries(&source_dir) + .into_iter() + .flatten() + .filter_map(|p| Format::of(&p).map(|f| (f, p))) + .collect(); + candidates.sort(); + + let (source, source_note) = + match candidates.iter().find(|(f, _)| f.needs_feature().is_none()) { + Some((_, path)) => (Some(path.clone()), String::new()), + // A source this build cannot read is a different skip from no source at all, and + // naming the feature makes it actionable. + None => { + let note = match candidates.first() { + Some((f, _)) => format!( + "source is {}, which needs the `{}` feature", + f.label(), + f.needs_feature().unwrap_or_default() + ), + None if source_dir.is_dir() => { + format!("no log in an OCEL format under {SOURCE_DIR}/") + } + None => format!("no {SOURCE_DIR}/ directory"), + }; + (None, note) + } + }; + + let mut derived: Vec = entries(dir) + .into_iter() + .flatten() + .filter(|p| p.file_name().is_some_and(|n| n != SOURCE_DIR)) + .filter(|p| Format::of(p).is_some()) + .collect(); + derived.sort(); + + Dataset { + name, + source, + source_note, + derived, + } + } + + fn entries(dir: &Path) -> Result, String> { + Ok(std::fs::read_dir(dir) + .map_err(|e| format!("cannot read `{}`: {e}", dir.display()))? + .filter_map(Result::ok) + .map(|e| e.path())) + } + + // Command line + + struct Args { + root: PathBuf, + only: Option>, + formats: Option>, + max_bytes: Option, + max_samples: usize, + list: bool, + } + + const HELP: &str = "\ +Cross-check every re-exported form of a log against the source it was written from. + + --root directory with one folder per dataset (default: ocel-datasets) + --only restrict the run to these dataset folder names + --formats restrict the re-exports to these formats + (bundle, json, xml, sqlite, duckdb, csv) + --max-mb skip an input larger than n MB (0 = no limit) + --max-diffs example differences to print per failing check + --list print what was found under --root and exit + -h, --help print this +"; + + fn parse_args() -> Result { + let mut args = Args { + root: PathBuf::from(DEFAULT_ROOT), + only: None, + formats: None, + max_bytes: (DEFAULT_MAX_INPUT_MB > 0).then_some(DEFAULT_MAX_INPUT_MB * 1024 * 1024), + max_samples: DEFAULT_MAX_DIFF_SAMPLES, + list: false, + }; + let mut it = std::env::args().skip(1); + while let Some(arg) = it.next() { + let mut value = || { + it.next() + .ok_or_else(|| format!("`{arg}` needs a value")) + .map_err(|e| e.to_string()) + }; + let list = |v: String| -> Vec { + v.split(',') + .map(|s| s.trim().to_lowercase()) + .filter(|s| !s.is_empty()) + .collect() + }; + match arg.as_str() { + "-h" | "--help" => { + print!("{HELP}"); + std::process::exit(0); + } + "--list" => args.list = true, + // `--datasets` is what this example called the directory before it read one + // folder per dataset. + "--root" | "--datasets" => args.root = PathBuf::from(value()?), + "--only" => args.only = Some(list(value()?)), + "--formats" => args.formats = Some(list(value()?)), + "--max-mb" => { + let mb: u64 = value()?.parse().map_err(|_| "--max-mb needs a number")?; + args.max_bytes = (mb > 0).then(|| mb * 1024 * 1024); + } + "--max-diffs" => { + args.max_samples = + value()?.parse().map_err(|_| "--max-diffs needs a number")?; + } + other => return Err(format!("unknown argument `{other}`\n\n{HELP}")), + } + } + Ok(args) + } + + // Views of a log + + /// `tagged` prefixes the variant, so `Integer(5)`, `Float(5.0)` and `String("5")` render as + /// `i:5`, `f:5` and `s:5`. `plain` drops both the tag and a trailing `.0`, so those three + /// collapse onto one string and only a real difference in the number survives. + fn render(value: &OCELAttributeValue, tagged: bool) -> String { + let (tag, body) = match value { + OCELAttributeValue::Integer(i) => ('i', i.to_string()), + OCELAttributeValue::Float(f) => ('f', number(*f)), + OCELAttributeValue::Boolean(b) => ('b', b.to_string()), + // The same instant is written with different offsets by different formats, which is + // not a difference in the log. + OCELAttributeValue::Time(t) => ('t', t.to_utc().to_rfc3339()), + OCELAttributeValue::String(s) => ( + 's', + // Untagged, a string that is a number renders as one, so text `"5.0"` and a + // float `5.0` agree on the value and differ only in the variant. + if tagged { + s.clone() + } else { + numeric_text(s) + .or_else(|| boolean_text(s)) + .or_else(|| timestamp_text(s)) + .unwrap_or_else(|| s.clone()) + }, + ), + OCELAttributeValue::Null => ('n', String::new()), + }; + if tagged { + format!("{tag}:{body}") + } else { + body + } + } + + /// A float without a trailing `.0`, so `5.0` and `5` render alike. + fn number(f: f64) -> String { + if f.is_finite() && f.fract() == 0.0 { + format!("{f:.0}") + } else { + f.to_string() + } + } + + fn numeric_text(s: &str) -> Option { + let t = s.trim(); + if t.is_empty() { + return None; + } + t.parse::() + .ok() + .map(|i| i.to_string()) + .or_else(|| t.parse::().ok().filter(|f| f.is_finite()).map(number)) + } + + /// Case-folded, so a log holding Python's `False` as text and a log holding a real boolean + /// agree on the value. + fn boolean_text(s: &str) -> Option { + let t = s.trim(); + (t.eq_ignore_ascii_case("true") || t.eq_ignore_ascii_case("false")) + .then(|| t.to_ascii_lowercase()) + } + + /// The UTC instant a string denotes, if it denotes one, so text kept verbatim by one format + /// and parsed into an [`OCELAttributeValue::Time`] by another agree on the value. + /// + /// The shape test in front of the parse is not optional: the parser is tried against every + /// string attribute in the log, and a log of this size has millions. + fn timestamp_text(s: &str) -> Option { + let t = s.trim(); + if !(8..=40).contains(&t.len()) + || !t.starts_with(|c: char| c.is_ascii_digit()) + || !t.contains('-') + { + return None; + } + parse_timestamp(t, None, false) + .ok() + .map(|d| d.to_utc().to_rfc3339()) + } + + /// `id -> type@time`, so a renamed type, a shifted timestamp and a missing event are all one + /// comparison. + fn events(ocel: &OCEL) -> BTreeMap { + ocel.events + .iter() + .map(|e| { + ( + e.id.clone(), + format!("{}@{}", e.event_type, e.time.to_utc().to_rfc3339()), + ) + }) + .collect() + } + + /// `id -> type`. + fn objects(ocel: &OCEL) -> BTreeMap { + ocel.objects + .iter() + .map(|o| (o.id.clone(), o.object_type.clone())) + .collect() + } + + /// `type name -> sorted "attribute:type" declarations`. + fn declared(types: &[OCELType]) -> BTreeMap> { + types + .iter() + .map(|t| { + let mut attrs: Vec = t + .attributes + .iter() + .map(|a| format!("{}:{}", a.name, a.value_type)) + .collect(); + attrs.sort(); + attrs.dedup(); + (t.name.clone(), attrs) + }) + .collect() + } + + /// `type name -> instance count`. + fn per_type<'a>(items: impl Iterator) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for t in items { + *out.entry(t.to_string()).or_default() += 1; + } + out.into_iter().map(|(k, v)| (k, v.to_string())).collect() + } + + fn e2o(ocel: &OCEL) -> Multiset { + let mut m = Multiset::default(); + for e in &ocel.events { + for r in &e.relationships { + m.add(format!("{} -> {} [{}]", e.id, r.object_id, r.qualifier)); + } + } + m + } + + fn o2o(ocel: &OCEL) -> Multiset { + let mut m = Multiset::default(); + for o in &ocel.objects { + for r in &o.relationships { + m.add(format!("{} -> {} [{}]", o.id, r.object_id, r.qualifier)); + } + } + m + } + + /// One attribute observation seen three ways, so a difference can be attributed to the value, + /// to the variant it is stored in, or to the time it was recorded at. + /// + /// `values` is a multiset because the same observation can legitimately repeat. A variant or + /// timestamp difference is only meaningful where both logs agree on what came before it, so + /// the other two are keyed maps and render as one paired line rather than two surplus ones. + #[derive(Default)] + struct AttrViews { + /// `id/name = value`, ignoring variant and time. + values: Multiset, + /// `id/name = value` -> the variants it is stored in. + variants: BTreeMap, + /// `id/name = tagged value` -> the times it was recorded at. + times: BTreeMap, + } + + impl AttrViews { + fn add(&mut self, key: &str, value: &OCELAttributeValue, time: Option) { + let (plain, tagged) = (render(value, false), render(value, true)); + self.values.add(format!("{key} = {plain}")); + self.variants + .entry(format!("{key} = {plain}")) + .or_default() + .0 + .push(tagged[..1].to_string()); + if let Some(time) = time { + self.times + .entry(format!("{key} = {tagged}")) + .or_default() + .0 + .push(time); + } + } + + /// Deduplicated, so that a log recording the same observation twice shows up only in the + /// `values` multiset and does not also colour the variant and time checks. + fn sort(mut self) -> Self { + for v in self.variants.values_mut().chain(self.times.values_mut()) { + v.0.sort(); + v.0.dedup(); + } + self + } + } + + fn event_attrs(ocel: &OCEL) -> AttrViews { + let mut v = AttrViews::default(); + for e in &ocel.events { + for a in &e.attributes { + v.add(&format!("{}/{}", e.id, a.name), &a.value, None); + } + } + v.sort() + } + + fn object_attrs(ocel: &OCEL) -> AttrViews { + let mut v = AttrViews::default(); + for o in &ocel.objects { + for a in &o.attributes { + v.add( + &format!("{}/{}", o.id, a.name), + &a.value, + Some(a.time.to_utc().to_rfc3339()), + ); + } + } + v.sort() + } + + // Diffing + + /// One comparison, carrying enough to read a count without guessing: how large each side is, + /// and which way the difference runs. + #[derive(Default)] + struct Diff { + /// Entries compared on each side. + a_total: u64, + b_total: u64, + /// Entries A has that B does not, and the reverse. + only_a: u64, + only_b: u64, + /// Keys both sides have under a different value. Always 0 for a multiset comparison, + /// which has no notion of a key holding one value. + changed: u64, + samples: Vec, + } + + impl Diff { + fn count(&self) -> u64 { + self.only_a + self.only_b + self.changed + } + + /// `12 of 45 in A / 44 in B: 3 only in A, 2 only in B, 7 changed`, with the empty parts + /// left out. + fn describe(&self) -> String { + let mut parts = Vec::new(); + if self.only_a > 0 { + parts.push(format!("{} only in A", self.only_a)); + } + if self.only_b > 0 { + parts.push(format!("{} only in B", self.only_b)); + } + if self.changed > 0 { + parts.push(format!("{} changed", self.changed)); + } + format!( + "{} of {} in A / {} in B: {}", + self.count(), + self.a_total, + self.b_total, + parts.join(", ") + ) + } + } + + #[derive(Default)] + struct Multiset(BTreeMap); + + impl Multiset { + fn add(&mut self, key: String) { + *self.0.entry(key).or_default() += 1; + } + + fn len(&self) -> u64 { + self.0.values().map(|&n| n.unsigned_abs()).sum() + } + + /// Surplus entries on either side, with up to `max` of them rendered. + fn diff(&self, other: &Self, max: usize) -> Diff { + let mut out = Diff { + a_total: self.len(), + b_total: other.len(), + ..Diff::default() + }; + let (mut left_only, mut right_only) = (Vec::new(), Vec::new()); + for (key, &left) in &self.0 { + let delta = left - other.0.get(key).copied().unwrap_or(0); + if delta != 0 { + let (bucket, side, tally) = if delta > 0 { + (&mut left_only, "A", &mut out.only_a) + } else { + (&mut right_only, "B", &mut out.only_b) + }; + *tally += delta.unsigned_abs(); + if bucket.len() < max { + bucket.push(format!("only in {side} (x{}): {key}", delta.abs())); + } + } + } + for (key, &right) in &other.0 { + if !self.0.contains_key(key) { + out.only_b += right.unsigned_abs(); + if right_only.len() < max { + right_only.push(format!("only in B (x{right}): {key}")); + } + } + } + out.samples = interleave(left_only, right_only, max); + out + } + } + + /// Difference between two keyed maps, split into keys missing from B, keys only in B, and + /// keys present in both with a different value. + fn map_diff( + a: &BTreeMap, + b: &BTreeMap, + max: usize, + ) -> Diff { + let mut out = Diff { + a_total: a.len() as u64, + b_total: b.len() as u64, + ..Diff::default() + }; + let (mut left_only, mut right_only, mut differing) = (Vec::new(), Vec::new(), Vec::new()); + for (key, left) in a { + match b.get(key) { + None => { + out.only_a += 1; + if left_only.len() < max { + left_only.push(format!("only in A: {key} = {left}")); + } + } + Some(right) if right != left => { + out.changed += 1; + if differing.len() < max { + differing.push(format!("differs: {key}: A = {left} | B = {right}")); + } + } + Some(_) => {} + } + } + for (key, right) in b { + if !a.contains_key(key) { + out.only_b += 1; + if right_only.len() < max { + right_only.push(format!("only in B: {key} = {right}")); + } + } + } + differing.truncate(max); + let rest = max.saturating_sub(differing.len()); + differing.extend(interleave(left_only, right_only, rest)); + out.samples = differing; + out + } + + /// Takes from both sides in turn, so a capped sample never shows only one direction of a + /// difference and hides the counterpart that explains it. + fn interleave(a: Vec, b: Vec, max: usize) -> Vec { + let mut out = Vec::with_capacity(max.min(a.len() + b.len())); + let (mut a, mut b) = (a.into_iter(), b.into_iter()); + loop { + let before = out.len(); + for next in [a.next(), b.next()].into_iter().flatten() { + if out.len() < max { + out.push(next); + } + } + if out.len() == before || out.len() >= max { + return out; + } + } + } + + /// Keys the two logs share but disagree on. Keys only one side has are left out: they are + /// already counted by the coarser check that this one refines. + fn shared_diff( + a: &BTreeMap, + b: &BTreeMap, + max: usize, + ) -> Diff { + let mut out = Diff { + a_total: a.len() as u64, + b_total: b.len() as u64, + ..Diff::default() + }; + for (key, left) in a { + if let Some(right) = b.get(key) { + if right != left { + out.changed += 1; + if out.samples.len() < max { + out.samples.push(format!("{key}: A = {left} | B = {right}")); + } + } + } + } + out + } + + /// `Vec` needs a `Display` to go through [`map_diff`]; this wraps it. + #[derive(Default)] + struct Joined(Vec); + + impl PartialEq for Joined { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + impl std::fmt::Display for Joined { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.join(", ")) + } + } + + // Reporting + + enum Status { + Ok, + /// Differences that fail the comparison. + Failed, + /// Differences the policy expects for this pair of formats. + Tolerated, + } + + struct Check { + name: &'static str, + status: Status, + detail: String, + samples: Vec, + } + + impl Check { + fn ok(name: &'static str) -> Self { + Check { + name, + status: Status::Ok, + detail: "match".into(), + samples: Vec::new(), + } + } + + /// A check that counts offenders in one log rather than differences between two. + fn from_count( + name: &'static str, + count: u64, + total: u64, + samples: Vec, + fails: bool, + ) -> Self { + if count == 0 { + return Check::ok(name); + } + Check { + name, + status: if fails { + Status::Failed + } else { + Status::Tolerated + }, + detail: format!("{count} of {total}"), + samples, + } + } + + fn from_diff(name: &'static str, diff: Diff, fails: bool) -> Self { + if diff.count() == 0 { + return Check::ok(name); + } + Check { + name, + status: if fails { + Status::Failed + } else { + Status::Tolerated + }, + detail: diff.describe(), + samples: diff.samples, + } + } + } + + /// Checks that need only one log. + /// + /// A pairwise diff cannot see any of these: two logs can be equally broken and agree + /// perfectly, and a duplicated id collapses into a single map entry before the comparison + /// ever runs. + fn integrity(ocel: &OCEL, max: usize) -> Vec { + let mut checks = Vec::new(); + + let (n, s) = duplicates(ocel.events.iter().map(|e| e.id.as_str()), max); + checks.push(Check::from_count( + "duplicate event ids", + n, + ocel.events.len() as u64, + s, + true, + )); + + let (n, s) = duplicates(ocel.objects.iter().map(|o| o.id.as_str()), max); + checks.push(Check::from_count( + "duplicate object ids", + n, + ocel.objects.len() as u64, + s, + true, + )); + + let object_ids: HashSet<&str> = ocel.objects.iter().map(|o| o.id.as_str()).collect(); + + let mut dangling = 0u64; + let mut total = 0u64; + let mut samples = Vec::new(); + for e in &ocel.events { + for r in &e.relationships { + total += 1; + if !object_ids.contains(r.object_id.as_str()) { + dangling += 1; + if samples.len() < max { + samples.push(format!("{} -> {} (no such object)", e.id, r.object_id)); + } + } + } + } + checks.push(Check::from_count( + "E2O relations pointing at no object", + dangling, + total, + samples, + true, + )); + + let mut dangling = 0u64; + let mut total = 0u64; + let mut samples = Vec::new(); + for o in &ocel.objects { + for r in &o.relationships { + total += 1; + if !object_ids.contains(r.object_id.as_str()) { + dangling += 1; + if samples.len() < max { + samples.push(format!("{} -> {} (no such object)", o.id, r.object_id)); + } + } + } + } + checks.push(Check::from_count( + "O2O relations pointing at no object", + dangling, + total, + samples, + true, + )); + + let declared_event: HashSet<&str> = + ocel.event_types.iter().map(|t| t.name.as_str()).collect(); + let (n, s) = undeclared( + ocel.events + .iter() + .map(|e| (e.id.as_str(), e.event_type.as_str())), + &declared_event, + max, + ); + checks.push(Check::from_count( + "events of an undeclared type", + n, + ocel.events.len() as u64, + s, + true, + )); + + let declared_object: HashSet<&str> = + ocel.object_types.iter().map(|t| t.name.as_str()).collect(); + let (n, s) = undeclared( + ocel.objects + .iter() + .map(|o| (o.id.as_str(), o.object_type.as_str())), + &declared_object, + max, + ); + checks.push(Check::from_count( + "objects of an undeclared type", + n, + ocel.objects.len() as u64, + s, + true, + )); + + let declared = attribute_names(&ocel.event_types); + let mut missing = 0u64; + let mut total = 0u64; + let mut samples = Vec::new(); + for e in &ocel.events { + for a in &e.attributes { + total += 1; + if !declared.contains(&(e.event_type.as_str(), a.name.as_str())) { + missing += 1; + if samples.len() < max { + samples.push(format!( + "{}/{} not declared for event type {}", + e.id, a.name, e.event_type + )); + } + } + } + } + checks.push(Check::from_count( + "event attributes not declared by their type", + missing, + total, + samples, + true, + )); + + let declared = attribute_names(&ocel.object_types); + let mut missing = 0u64; + let mut total = 0u64; + let mut samples = Vec::new(); + for o in &ocel.objects { + for a in &o.attributes { + total += 1; + if !declared.contains(&(o.object_type.as_str(), a.name.as_str())) { + missing += 1; + if samples.len() < max { + samples.push(format!( + "{}/{} not declared for object type {}", + o.id, a.name, o.object_type + )); + } + } + } + } + checks.push(Check::from_count( + "object attributes not declared by their type", + missing, + total, + samples, + true, + )); + + checks + } + + /// Ids that occur more than once, and how many surplus occurrences there are in total. + fn duplicates<'a>(ids: impl Iterator, max: usize) -> (u64, Vec) { + let mut seen: HashMap<&str, u64> = HashMap::new(); + for id in ids { + *seen.entry(id).or_default() += 1; + } + let mut extra = 0u64; + let mut samples = Vec::new(); + let mut repeated: Vec<_> = seen.into_iter().filter(|(_, n)| *n > 1).collect(); + repeated.sort(); + for (id, n) in repeated { + extra += n - 1; + if samples.len() < max { + samples.push(format!("{id} occurs {n} times")); + } + } + (extra, samples) + } + + fn undeclared<'a>( + items: impl Iterator, + declared: &HashSet<&str>, + max: usize, + ) -> (u64, Vec) { + let mut count = 0u64; + let mut samples = Vec::new(); + for (id, type_name) in items { + if !declared.contains(type_name) { + count += 1; + if samples.len() < max { + samples.push(format!("{id} has undeclared type {type_name}")); + } + } + } + (count, samples) + } + + fn attribute_names(types: &[OCELType]) -> HashSet<(&str, &str)> { + types + .iter() + .flat_map(|t| { + t.attributes + .iter() + .map(move |a| (t.name.as_str(), a.name.as_str())) + }) + .collect() + } + + /// Every check for one pair of logs, in the order they are printed. + fn compare(a: &OCEL, b: &OCEL, policy: Policy, max: usize) -> Vec { + let mut checks = Vec::new(); + + let d = map_diff(&events(a), &events(b), max); + checks.push(Check::from_diff("events (id, type, time)", d, true)); + + let d = map_diff(&objects(a), &objects(b), max); + checks.push(Check::from_diff("objects (id, type)", d, true)); + + let d = map_diff( + &per_type(a.events.iter().map(|e| e.event_type.as_str())), + &per_type(b.events.iter().map(|e| e.event_type.as_str())), + max, + ); + checks.push(Check::from_diff("events per event type", d, true)); + + let d = map_diff( + &per_type(a.objects.iter().map(|o| o.object_type.as_str())), + &per_type(b.objects.iter().map(|o| o.object_type.as_str())), + max, + ); + checks.push(Check::from_diff("objects per object type", d, true)); + + // Names always, declarations only where the format carries them. + let (decl_a, decl_b) = (declared(&a.event_types), declared(&b.event_types)); + let d = map_diff( + &decl_a + .keys() + .map(|k| (k.clone(), String::new())) + .collect::>(), + &decl_b + .keys() + .map(|k| (k.clone(), String::new())) + .collect::>(), + max, + ); + checks.push(Check::from_diff("event type names", d, true)); + + let d = map_diff( + &decl_a.into_iter().map(|(k, v)| (k, Joined(v))).collect(), + &decl_b.into_iter().map(|(k, v)| (k, Joined(v))).collect(), + max, + ); + checks.push(Check::from_diff( + "event type attribute declarations", + d, + policy.declared_types, + )); + + let (decl_a, decl_b) = (declared(&a.object_types), declared(&b.object_types)); + let d = map_diff( + &decl_a + .keys() + .map(|k| (k.clone(), String::new())) + .collect::>(), + &decl_b + .keys() + .map(|k| (k.clone(), String::new())) + .collect::>(), + max, + ); + checks.push(Check::from_diff("object type names", d, true)); + + let d = map_diff( + &decl_a.into_iter().map(|(k, v)| (k, Joined(v))).collect(), + &decl_b.into_iter().map(|(k, v)| (k, Joined(v))).collect(), + max, + ); + checks.push(Check::from_diff( + "object type attribute declarations", + d, + policy.declared_types, + )); + + let d = e2o(a).diff(&e2o(b), max); + checks.push(Check::from_diff("E2O relations", d, true)); + + let d = o2o(a).diff(&o2o(b), max); + checks.push(Check::from_diff("O2O relations", d, true)); + + checks.extend(attribute_checks( + "event attribute", + &event_attrs(a), + &event_attrs(b), + policy, + max, + false, + )); + checks.extend(attribute_checks( + "object attribute", + &object_attrs(a), + &object_attrs(b), + policy, + max, + true, + )); + + checks + } + + /// Splits one attribute difference into the value, variant and time classes. + /// + /// The three views nest, since a value difference also shows up in the typed and full views, + /// so each class is the growth from the coarser view below it. + fn attribute_checks( + what: &'static str, + a: &AttrViews, + b: &AttrViews, + policy: Policy, + max: usize, + timed: bool, + ) -> Vec { + let d = a.values.diff(&b.values, max); + let mut out = vec![Check::from_diff(leak(format!("{what} values")), d, true)]; + + let d = shared_diff(&a.variants, &b.variants, max); + out.push(Check::from_diff( + leak(format!("{what} value types (i/f/b/t/s/n)")), + d, + policy.value_types, + )); + + if timed { + let d = shared_diff(&a.times, &b.times, max); + out.push(Check::from_diff( + leak(format!("{what} value timestamps")), + d, + policy.value_times, + )); + } + + out + } + + /// Check names are `&'static str` because most of them are literals; the handful built per + /// attribute kind are leaked once each. + fn leak(s: String) -> &'static str { + Box::leak(s.into_boxed_str()) + } + + // Running + + /// Imports `path`, printing what it read and how long it took. + fn load(label: &str, path: &Path) -> Option { + let started = Instant::now(); + match OCEL::import_from_path(path) { + Ok(ocel) => { + println!( + " {label:<10} {:>9} events {:>9} objects {:>7.1}s {}", + ocel.events.len(), + ocel.objects.len(), + started.elapsed().as_secs_f64(), + path.display() + ); + Some(ocel) + } + Err(e) => { + println!(" {label:<10} IMPORT FAILED: {e}"); + println!(" {}", path.display()); + None + } + } + } + + fn report(title: &str, a: &OCEL, b: &OCEL, policy: Policy, max: usize) -> bool { + print_checks(title, compare(a, b, policy, max)) + } + + fn report_integrity(label: &str, ocel: &OCEL, max: usize) -> bool { + print_checks(&format!("integrity of {label}"), integrity(ocel, max)) + } + + fn print_checks(title: &str, checks: Vec) -> bool { + println!(" {title}"); + let mut failed = false; + for check in &checks { + let (mark, note) = match check.status { + Status::Ok => ("ok ", ""), + Status::Failed => ("FAIL", ""), + Status::Tolerated => ("note", " (expected for this format)"), + }; + if matches!(check.status, Status::Failed) { + failed = true; + } + if matches!(check.status, Status::Ok) { + continue; + } + println!(" [{mark}] {:<42} {}{note}", check.name, check.detail); + for sample in &check.samples { + println!(" {sample}"); + } + } + let ok = checks + .iter() + .filter(|c| matches!(c.status, Status::Ok)) + .count(); + println!(" {ok}/{} checks clean", checks.len()); + failed + } + + fn size_of(path: &Path) -> u64 { + std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) + } + + fn megabytes(bytes: u64) -> u64 { + bytes / (1024 * 1024) + } + + /// Whether `path` is small enough to read, printing why not when it is not. + fn within_ceiling(path: &Path, limit: Option) -> bool { + let (Some(limit), size) = (limit, size_of(path)) else { + return true; + }; + if size <= limit { + return true; + } + println!( + " skipped {} : {} MB, above the {} MB ceiling (--max-mb 0 to lift)", + path.display(), + megabytes(size), + megabytes(limit) + ); + false + } + + /// What one dataset folder did, so the summary can name it without re-deriving anything. + #[derive(Default)] + struct Outcome { + compared: usize, + failed: bool, + /// Re-exports that were not read at all, and why. + skipped: Vec, + } + + fn check_dataset(dataset: &Dataset, args: &Args) -> Outcome { + let mut outcome = Outcome::default(); + + let Some(source_path) = &dataset.source else { + println!(" skipped: {}", dataset.source_note); + outcome.skipped.push("source".into()); + return outcome; + }; + if dataset.derived.is_empty() { + println!(" skipped: no re-exported log beside {SOURCE_DIR}/"); + outcome.skipped.push("re-exports".into()); + return outcome; + } + // Nothing is comparable without the reference, so its ceiling skips the whole folder. + if !within_ceiling(source_path, args.max_bytes) { + outcome.skipped.push("source".into()); + return outcome; + } + + let source_format = Format::of(source_path).expect("a source is chosen by its format"); + let Some(source) = load("source", source_path) else { + outcome.failed = true; + return outcome; + }; + outcome.failed |= report_integrity("source", &source, args.max_samples); + + for path in &dataset.derived { + let format = Format::of(path).expect("a re-export is chosen by its format"); + let label = format.label(); + if let Some(only) = &args.formats { + if !only.iter().any(|f| f == label) { + continue; + } + } + if let Some(feature) = format.needs_feature() { + println!(" skipped {label:<8} : needs the `{feature}` feature"); + outcome.skipped.push(label.to_string()); + continue; + } + if !within_ceiling(path, args.max_bytes) { + outcome.skipped.push(label.to_string()); + continue; + } + // One re-export at a time, so only it and the source are ever held at once. + let Some(derived) = load(label, path) else { + outcome.failed = true; + continue; + }; + outcome.compared += 1; + outcome.failed |= report_integrity(label, &derived, args.max_samples); + outcome.failed |= report( + &format!("A = {label}, B = source ({})", source_format.label()), + &derived, + &source, + format.policy().and(source_format.policy()), + args.max_samples, + ); + } + + outcome + } + + fn print_listing(datasets: &[Dataset]) { + for dataset in datasets { + println!("{}", dataset.name); + match &dataset.source { + Some(p) => println!( + " source {:<8} {} MB {}", + Format::of(p).map(Format::label).unwrap_or(""), + megabytes(size_of(p)), + p.display() + ), + None => println!(" source (none: {})", dataset.source_note), + } + for p in &dataset.derived { + println!( + " re-export {:<8} {} MB {}", + Format::of(p).map(Format::label).unwrap_or(""), + megabytes(size_of(p)), + p.display() + ); + } + } + } + + pub fn run() -> ExitCode { + let args = match parse_args() { + Ok(a) => a, + Err(e) => { + eprintln!("{e}"); + return ExitCode::from(2); + } + }; + + let mut datasets = match discover(&args.root) { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return ExitCode::from(2); + } + }; + if let Some(only) = &args.only { + datasets.retain(|d| only.iter().any(|n| *n == d.name.to_lowercase())); + } + + if args.list { + print_listing(&datasets); + return ExitCode::SUCCESS; + } + if datasets.is_empty() { + eprintln!("no dataset folder under `{}`", args.root.display()); + return ExitCode::from(2); + } + + let mut failed: Vec = Vec::new(); + let mut skipped: Vec = Vec::new(); + let (mut checked, mut comparisons) = (0usize, 0usize); + + for dataset in &datasets { + println!("\n=== {} ===", dataset.name); + let outcome = check_dataset(dataset, &args); + if outcome.compared > 0 { + checked += 1; + comparisons += outcome.compared; + } + if outcome.failed { + failed.push(dataset.name.clone()); + } + if !outcome.skipped.is_empty() { + skipped.push(format!("{} ({})", dataset.name, outcome.skipped.join(", "))); + } + } + + println!("\n=== summary ==="); + println!(" {comparisons} re-export(s) checked across {checked} dataset(s)"); + if !skipped.is_empty() { + println!(" skipped:"); + for s in &skipped { + println!(" {s}"); + } + } + if failed.is_empty() { + println!(" no mismatches"); + ExitCode::SUCCESS + } else { + println!(" mismatches in:"); + for f in &failed { + println!(" {f}"); + } + ExitCode::FAILURE + } + } +} diff --git a/process_mining/examples/ocel_kuzudb_export.rs b/process_mining/examples/ocel_kuzudb_export.rs deleted file mode 100644 index aca47992..00000000 --- a/process_mining/examples/ocel_kuzudb_export.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::{ - env::args, - fs::{create_dir_all, remove_file}, - path::PathBuf, - time::Instant, -}; - -use process_mining::core::event_data::object_centric::{ - graph_db::ocel_kuzudb::export_ocel_to_kuzudb_typed, linked_ocel::IndexLinkedOCEL, -}; -use process_mining::{Importable, OCEL}; -use std::error::Error; - -fn main() -> Result<(), Box> { - let base_path_opt = args().nth(1); - if let Some(base_path) = base_path_opt.map(PathBuf::from) { - let export_path = base_path.join("kuzu"); - create_dir_all(&export_path).expect("Could not create export folder (`kuzu`) base path"); - for p in [ - "order-management.xml", - "ocel2-p2p.xml", - "ContainerLogistics.xml", - "bpic2017-o2o-workflow-qualifier-index.xml", - ] { - println!("== {p} =="); - let now = Instant::now(); - let ocel = OCEL::import_from_path(base_path.join(p))?; - println!("Import OCEL XML took {:?}", now.elapsed()); - let now = Instant::now(); - let locel = IndexLinkedOCEL::from(ocel); - println!("Linking OCEL took {:?}", now.elapsed()); - let now = Instant::now(); - let file_path = export_path.join(format!("{p}.kuzu")); - // Remove file (if it already exists) - let _ = remove_file(&file_path); - export_ocel_to_kuzudb_typed(&file_path, &locel)?; - println!("Kuzu export took {:?}", now.elapsed()); - } - } - Ok(()) -} diff --git a/process_mining/examples/ocel_stream_to_duckdb.rs b/process_mining/examples/ocel_stream_to_duckdb.rs new file mode 100644 index 00000000..105af517 --- /dev/null +++ b/process_mining/examples/ocel_stream_to_duckdb.rs @@ -0,0 +1,23 @@ +//! Stream an OCEL file directly into a DuckDB database (consolidated schema), +//! without materializing the whole log in memory. +//! +//! Run: +//! cargo run --example ocel_stream_to_duckdb --features ocel-duckdb -- +//! +use process_mining::core::event_data::object_centric::ocel_sql::{ + stream_ocel_file_to_duckdb_with, DuckDbImportOptions, +}; + +fn main() { + let mut args = std::env::args().skip(1); + let src = args + .next() + .expect("usage: ocel_stream_to_duckdb "); + let out = args + .next() + .expect("usage: ocel_stream_to_duckdb "); + + let options = DuckDbImportOptions::default(); + stream_ocel_file_to_duckdb_with(&src, &out, &options).expect("streaming import failed"); + println!("Wrote {out}"); +} diff --git a/process_mining/src/analysis/case_centric/event_timestamp_histogram.rs b/process_mining/src/analysis/case_centric/event_timestamp_histogram.rs index 17c5719c..1be60b4e 100644 --- a/process_mining/src/analysis/case_centric/event_timestamp_histogram.rs +++ b/process_mining/src/analysis/case_centric/event_timestamp_histogram.rs @@ -23,6 +23,11 @@ pub struct AggregatedEventTimestamps { pub events_per_timestamp: HashMap>, /// All distinct activity names found in the log. pub activities: Vec, + /// Width of each bin in milliseconds, which is also the spacing of the bin centers. + /// + /// Bins are keyed by their center, so one spans + /// `[center - bin_width_ms / 2, center + bin_width_ms / 2)`. `0` means there are no bins. + pub bin_width_ms: i64, } /// Options for [`get_event_timestamps`]. @@ -88,15 +93,18 @@ pub fn get_event_timestamps( return AggregatedEventTimestamps { events_per_timestamp: HashMap::default(), activities: activities.into_iter().cloned().collect(), + bin_width_ms: 0, }; }; - let bin_size = (max - min) as f64 / num_bins as f64; + // Bin centers are whole milliseconds, so rounding the width up keeps it equal to the center + // spacing even when the span is shorter than the number of bins. + let bin_width_ms = (((max - min) as f64 / num_bins as f64).ceil() as i64).max(1); let date_bins: Vec<_> = (0..num_bins) - .map(|bin_index| (min as f64 + (bin_index as f64 + 0.5) * bin_size).round() as i64) + .map(|bin_index| min + bin_index as i64 * bin_width_ms + bin_width_ms / 2) .collect(); let mut ev_counts: HashMap> = HashMap::new(); for (timestamp, act) in ×tamps_with_act { - let bin_index = (((timestamp - min) as f64 / bin_size).floor() as usize).min(num_bins - 1); + let bin_index = (((timestamp - min) / bin_width_ms) as usize).min(num_bins - 1); *ev_counts .entry(date_bins[bin_index]) .or_default() @@ -106,5 +114,6 @@ pub fn get_event_timestamps( AggregatedEventTimestamps { events_per_timestamp: ev_counts, activities: activities.into_iter().cloned().collect(), + bin_width_ms, } } diff --git a/process_mining/src/bindings/extraction_bindings.rs b/process_mining/src/bindings/extraction_bindings.rs new file mode 100644 index 00000000..1d6deb4c --- /dev/null +++ b/process_mining/src/bindings/extraction_bindings.rs @@ -0,0 +1,834 @@ +//! Binding wrappers for the relational-to-OCEL extraction blueprint subsystem. +//! +//! A [`Blueprint`] describes how to build an OCEL out of relational tables. It carries no +//! connection details and no schema snapshot, so it can be saved, shared and sent to a browser +//! without leaking credentials. Bindings that reach a real data source take the connection strings +//! as a separate argument. +//! +//! [`extraction_validate`] and [`extraction_compile`] are pure: they take a [`Blueprint`] and an +//! already-discovered [`ExtractionCatalog`], open no connection and read no row. The +//! `extraction_*_items` pair reads sources held in the registry as bytes, which needs +//! `ocel-sqlite` but still no connector. +//! +//! `extraction_discover_catalog`, `extraction_column_domain`, `extraction_run` and +//! `extraction_run_to_duckdb` open a real connection through `dbcon` and live in the sibling +//! `extraction_dbcon_bindings` module behind the `extraction-dbcon` feature. + +use macros_process_mining::register_binding; + +#[cfg(feature = "ocel-sqlite")] +use crate::bindings::{RegistryItem, StateRef}; +#[cfg(all(feature = "ocel-sqlite", feature = "extraction-dbcon"))] +use crate::core::event_data::object_centric::extraction::DbconRowProvider; +use crate::core::event_data::object_centric::extraction::{ + compile, validate, Blueprint, CompiledOcel, EmissionShape, ExtractionCatalog, ExtractionReport, + SqlDialect, ValidationError, +}; +#[cfg(feature = "ocel-sqlite")] +use crate::core::event_data::object_centric::extraction::{ + extract, provider::distinct_column_values, provider::preview_rows, Catalog, RowProvider, + SlimOcelSink, SqliteRowProvider, TablePreview, +}; +#[cfg(feature = "ocel-sqlite")] +use crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL; +#[cfg(feature = "ocel-sqlite")] +use crate::core::tabular_source::TabularReader; +#[cfg(feature = "ocel-sqlite")] +use std::collections::HashMap; + +/// Check `blueprint` against `catalog` for problems decidable from the schema alone, such as an +/// unknown source or table, a node graph cycle, or a type-rendering rule the blueprint's own +/// `id_rendering` setting cannot satisfy. +/// +/// An empty result means `blueprint` is safe to pass to [`extraction_run`], +/// [`extraction_run_to_duckdb`] or [`extraction_compile`]. All three call this internally and +/// refuse to run an invalid blueprint, so calling it first is for surfacing errors to a user +/// while they edit, not a required step before the others. +/// +/// The two report a refusal differently, because they fail differently: the running bindings +/// return an error string, while [`extraction_compile`] always returns a [`CompiledOcel`] and +/// puts one entry per validation error in its `errors` array, with no relations emitted. This is +/// also the only place a blueprint's `version` is checked: the bindings deserialize a +/// [`Blueprint`] with plain serde, not `Blueprint::from_json`, so a blueprint from a newer model +/// version is caught here and nowhere else. +#[register_binding] +fn extraction_validate(blueprint: Blueprint, catalog: ExtractionCatalog) -> Vec { + validate(&blueprint, &catalog) +} + +/// Compile `blueprint` into SQL views presenting the OCEL 2.0 surface over `catalog`'s tables, +/// with no connection opened and no row read. +/// +/// `shape` picks the emitted layout: [`EmissionShape::PerType`] (one view per declared event/ +/// object type, the layout external OCEL 2.0 tooling reads) or [`EmissionShape::Consolidated`] +/// (a single wide `events`/`objects`/... layout with the type as a column value). A mapping the +/// emitter cannot reproduce exactly is skipped and recorded in the result rather than failing the +/// whole compile. +#[register_binding] +fn extraction_compile( + blueprint: Blueprint, + catalog: ExtractionCatalog, + shape: EmissionShape, + #[bind(default)] dialect: SqlDialect, +) -> CompiledOcel { + compile(&blueprint, &catalog, dialect, shape) +} + +/// Borrow every source in `sources` as an opened [`RowProvider`], keyed the same way. +/// +/// `sources` maps a blueprint's `source_id` to the registry id of a [`TabularSource`], i.e. the +/// bytes of a file someone dropped. Nothing here touches the filesystem, which is what makes it +/// the only extraction path available on `wasm32`. +/// +/// Each source is opened once and cached on the item, so a second call reuses the open `SQLite` +/// database rather than copying the file in again. +/// +/// Returns the lock guards: the reader is not `Sync`, so it may only be touched while its guard is +/// held. Two `source_id`s naming the same item share one guard, since locking the same mutex twice +/// would deadlock, and items are opened in registry-id order rather than in `sources` iteration +/// order, so two concurrent calls cannot take the same locks in opposite orders. +#[cfg(feature = "ocel-sqlite")] +fn open_sources<'a>( + state: StateRef<'a>, + sources: &HashMap, +) -> Result, OpenedSource<'a>)>, String> { + let mut by_item: Vec<(&str, Vec)> = Vec::new(); + for (source_id, item_id) in sources { + match by_item.iter_mut().find(|(id, _)| *id == item_id.as_str()) { + Some((_, ids)) => ids.push(source_id.clone()), + None => by_item.push((item_id.as_str(), vec![source_id.clone()])), + } + } + by_item.sort_unstable(); + + let mut out: Vec<(Vec, OpenedSource<'a>)> = Vec::with_capacity(by_item.len()); + for (item_id, source_ids) in by_item { + let named = source_ids.join(", "); + let Some(item) = state.get(item_id) else { + return Err(format!("no item '{item_id}' for source '{named}'")); + }; + let RegistryItem::TabularSource(src) = item else { + return Err(format!("item '{item_id}' is not a data source")); + }; + let opened = match src.format() { + "sqlite" | "sqlite3" | "db" => src + .reader(SqliteRowProvider::from_slice) + .map(OpenedSource::Sqlite), + // Bytes, not a path: the route a browser takes, where a dropped file is all there + // is. Feature-gated rather than absent, so a build without the connector still says + // which formats it can manage. + #[cfg(feature = "extraction-dbcon")] + format @ ("csv" | "tsv" | "parquet" | "xlsx") => { + let format = format.to_string(); + let name = named.clone(); + src.reader(move |bytes| { + DbconRowProvider::from_bytes(&name, &format, std::sync::Arc::from(bytes)) + }) + .map(OpenedSource::Dbcon) + } + other => Err(format!("this build cannot read '{other}' from memory")), + }; + out.push(( + source_ids, + opened.map_err(|e| format!("source '{named}': {e}"))?, + )); + } + Ok(out) +} + +/// The entry holding one `source_id`, or an error naming it. +#[cfg(feature = "ocel-sqlite")] +fn opened_for<'a, 'b>( + opened: &'b [(Vec, OpenedSource<'a>)], + source_id: &str, +) -> Result<&'b OpenedSource<'a>, String> { + opened + .iter() + .find(|(ids, _)| ids.iter().any(|id| id == source_id)) + .map(|(_, o)| o) + .ok_or_else(|| format!("no source '{source_id}'")) +} + +/// Merge `from` into `into`, keyed by `source_id` so neither source can clobber the other. +pub(super) fn merge_into(into: &mut ExtractionCatalog, from: ExtractionCatalog) { + into.tables.extend(from.tables); + into.domains.extend(from.domains); + into.previews.extend(from.previews); +} + +/// The message for a run whose report carries per-mapping failures, or `None` for a clean run. +/// +/// A binding returning only a log handle has nowhere to put the [`ExtractionReport`], so a run +/// that dropped every row of a mapping would otherwise look like a clean success. +pub(super) fn report_error_message(report: &ExtractionReport) -> Option { + if report.errors.is_empty() { + return None; + } + let listed = report + .errors + .iter() + .map(ToString::to_string) + .collect::>() + .join(" | "); + let total = report.errors.len() as u64 + report.errors_suppressed; + let mut msg = format!("extraction reported {total} errors: {listed}"); + if report.errors_suppressed > 0 { + msg.push_str(&format!( + " (and {} further errors, not kept)", + report.errors_suppressed + )); + } + Some(msg) +} + +/// Every source's schema, merged into one catalog under its own `source_id`. +#[cfg(feature = "ocel-sqlite")] +fn discover_all(opened: &[(Vec, OpenedSource<'_>)]) -> Result { + let mut catalog = ExtractionCatalog::new(); + for (source_ids, reader) in opened { + for source_id in source_ids { + merge_into(&mut catalog, reader.catalog(source_id)?); + } + } + Ok(catalog) +} + +/// A source opened as whichever reader its format calls for. +#[cfg(feature = "ocel-sqlite")] +enum OpenedSource<'a> { + Sqlite(TabularReader<'a, SqliteRowProvider>), + /// CSV, TSV or Parquet, which need `dbcon`'s readers. `SQLite` does not go here: `dbcon` + /// opens a `SQLite` database by path, and these sources are bytes. + #[cfg(feature = "extraction-dbcon")] + Dbcon(TabularReader<'a, DbconRowProvider>), +} + +#[cfg(feature = "ocel-sqlite")] +impl OpenedSource<'_> { + fn provider(&self) -> &dyn RowProvider { + match self { + Self::Sqlite(r) => r.get(), + #[cfg(feature = "extraction-dbcon")] + Self::Dbcon(r) => r.get(), + } + } + + fn catalog(&self, source_id: &str) -> Result { + match self { + Self::Sqlite(r) => r + .get() + .discover_catalog(source_id) + .map_err(|e| format!("source '{source_id}': {e}")), + #[cfg(feature = "extraction-dbcon")] + Self::Dbcon(r) => Ok(r.get().discover_catalog(source_id)), + } + } +} + +/// The `source_id -> provider` map [`extract`] wants, borrowed from held guards. +#[cfg(feature = "ocel-sqlite")] +fn provider_refs<'a>( + opened: &'a [(Vec, OpenedSource<'a>)], +) -> HashMap { + opened + .iter() + .flat_map(|(source_ids, reader)| { + let provider = reader.provider(); + source_ids.iter().map(move |id| (id.clone(), provider)) + }) + .collect() +} + +/// Discover the schema of every source in `sources`, merged into one [`ExtractionCatalog`]. +/// +/// The in-memory counterpart of `extraction_discover_catalog`: that one takes connection strings +/// and needs a database connector, this one reads files already in the registry and needs only +/// `ocel-sqlite`. `sources` maps each `source_id` a blueprint names to the registry id of an +/// imported source file. +#[cfg(feature = "ocel-sqlite")] +#[register_binding(stringify_error)] +fn extraction_discover_catalog_items( + #[bind(state)] state: StateRef<'_>, + sources: HashMap, +) -> Result { + discover_all(&open_sources(state, &sources)?) +} + +/// Every distinct value of `table.column` in a source held in the registry. +/// +/// The registry counterpart of `extraction_column_domain`: same answer, for a source whose bytes +/// the host already holds rather than one reachable by connection string. Without it a dropped +/// file could be extracted from but never inspected, so the editor could not offer the example +/// values a dynamic type name needs, and on wasm, where every source is byte-held, not at all. +/// +/// `sources` maps source id to registry item id. Every entry other than `source_id` is ignored. +#[cfg(feature = "ocel-sqlite")] +#[register_binding(stringify_error)] +fn extraction_column_domain_items( + #[bind(state)] state: StateRef<'_>, + sources: HashMap, + source_id: String, + table: String, + column: String, +) -> Result, String> { + let opened = open_sources(state, &sources)?; + let provider = opened_for(&opened, &source_id)?.provider(); + distinct_column_values(provider, &table, &column) + .map_err(|e| format!("source '{source_id}': {e}")) +} + +/// The first `limit` rows of `table` in a source held in the registry. +/// +/// See [`extraction_column_domain_items`] for why this exists separately from the connection-string +/// route. Not a substitute for a domain: a preview is incomplete, so it must never be used where +/// the compiler needs a column's full set of values. +/// +/// `sources` maps source id to registry item id. Every entry other than `source_id` is ignored. +#[cfg(feature = "ocel-sqlite")] +#[register_binding(stringify_error)] +fn extraction_table_preview_items( + #[bind(state)] state: StateRef<'_>, + sources: HashMap, + source_id: String, + table: String, + #[bind(default)] limit: Option, +) -> Result { + let opened = open_sources(state, &sources)?; + let source = opened_for(&opened, &source_id)?; + // Columns come from the source's own schema rather than from the caller, so a preview is + // always aligned to something the catalog agrees exists. `TableSchema::columns` is a + // `BTreeMap`, so that order is alphabetical: stable for a table, but not its declared order, + // and `TablePreview::columns` is what says which is which. + let catalog = source.catalog(&source_id)?; + let schema = catalog + .table(&source_id, &table) + .ok_or_else(|| format!("source '{source_id}' has no table '{table}'"))?; + let columns: Vec<&str> = schema.columns.keys().map(String::as_str).collect(); + preview_rows(source.provider(), &table, &columns, limit.unwrap_or(20)) + .map_err(|e| format!("source '{source_id}': {e}")) +} + +/// Run `blueprint` against sources held in the registry, returning the resulting log. +/// +/// Returns a fresh `SlimLinkedOCEL` rather than filling one given by `&mut`, because a binding +/// cannot both take a `&mut` big type and read the registry: the `&mut` borrow of the state guard +/// is live across the call, so lending the same guard out as a [`StateRef`] would not borrow-check. +/// +/// The [`ExtractionReport`] (drop reasons, per-mapping counts) is not returned, because a binding +/// returns either a big type or plain data, never both. A run whose report carries errors +/// therefore fails with them rather than handing back a log that silently lost rows. Validate +/// first with [`extraction_validate`] to catch what a report would otherwise tell you. +#[cfg(feature = "ocel-sqlite")] +#[register_binding(stringify_error)] +fn extraction_run_items( + #[bind(state)] state: StateRef<'_>, + blueprint: Blueprint, + sources: HashMap, + #[bind(default)] catalog: Option, +) -> Result { + let providers = open_sources(state, &sources)?; + let catalog = match catalog { + Some(c) => c, + None => discover_all(&providers)?, + }; + let mut sink = SlimOcelSink::new(); + let report = extract(&blueprint, &catalog, &provider_refs(&providers), &mut sink) + .map_err(|e| e.to_string())?; + if let Some(msg) = report_error_message(&report) { + return Err(msg); + } + Ok(sink.into_ocel()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bindings::{call, list_functions, AppState}; + use crate::core::event_data::object_centric::extraction::{AttributeMapping, FlatEventTable}; + + /// A tiny flat-event-table blueprint reading a table `events(case_id, activity, ts)` from + /// source `db`, built through + /// [`Blueprint::from_flat_event_table`](crate::core::event_data::object_centric::extraction::Blueprint::from_flat_event_table) + /// so it carries only a `source_id` string, never a connection string, matching what a real + /// caller sends over the bindings boundary. + fn flat_blueprint() -> Blueprint { + Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".to_string(), + table: "events".to_string(), + case_id: "case_id".to_string(), + activity: "activity".to_string(), + timestamp: "ts".to_string(), + case_object_type: "Case".to_string(), + case_attributes: Vec::::new(), + event_attributes: Vec::::new(), + }) + } + + fn flat_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + crate::core::event_data::object_centric::extraction::TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ) + } + + /// Registry-held sources answer `extraction_column_domain_items` the same way a connection + /// string answers `extraction_column_domain`. Without this the blueprint editor could extract + /// from a dropped file but never show what is in it, and on `wasm32`, where every source is + /// byte-held, there would be no example values at all. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn a_registry_source_reports_its_column_domain_and_a_preview() { + use crate::core::tabular_source::TabularSource; + + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("shop.sqlite"); + { + let con = rusqlite::Connection::open(&path).expect("open"); + con.execute_batch( + "CREATE TABLE orders (kind TEXT, amount TEXT); + INSERT INTO orders VALUES ('web', '10'), ('phone', '20'), ('web', '30');", + ) + .expect("seed"); + } + let state = AppState::default(); + state.add( + "src1", + RegistryItem::TabularSource(TabularSource::new( + std::fs::read(&path).expect("read"), + "sqlite", + )), + ); + + let domain = list_functions() + .into_iter() + .find(|b| b.name.ends_with("extraction_column_domain_items")) + .expect("domain binding is registered"); + let args = serde_json::json!({ + "sources": { "shop": "src1" }, "source_id": "shop", + "table": "orders", "column": "kind", + }); + let out = call(domain, &args, &state).expect("domain succeeds"); + let mut values: Vec = serde_json::from_slice(&out).expect("deserializes"); + values.sort(); + // Distinct, not one entry per row: 'web' appears twice in the table. + assert_eq!(values, vec!["phone".to_string(), "web".to_string()]); + + let preview = list_functions() + .into_iter() + .find(|b| b.name.ends_with("extraction_table_preview_items")) + .expect("preview binding is registered"); + let args = serde_json::json!({ + "sources": { "shop": "src1" }, "source_id": "shop", "table": "orders", "limit": 2, + }); + let out = call(preview, &args, &state).expect("preview succeeds"); + let preview: TablePreview = serde_json::from_slice(&out).expect("deserializes"); + // Alphabetical, because the schema keys columns in a `BTreeMap`. The rows are aligned to + // exactly that, which is why the header is returned alongside them. + assert_eq!( + preview.columns, + vec!["amount".to_string(), "kind".to_string()] + ); + assert_eq!(preview.rows.len(), 2, "limit is respected"); + assert_eq!( + preview.rows[0], + vec![Some("10".to_string()), Some("web".to_string())] + ); + } + + /// The same two bindings over a CSV, which reaches them through `dbcon` rather than through + /// `SqliteRowProvider`, the arm a dropped `.csv` or `.parquet` takes in a browser. + #[cfg(all(feature = "ocel-sqlite", feature = "extraction-dbcon"))] + #[test] + fn a_registry_csv_reports_its_column_domain_too() { + use crate::core::tabular_source::TabularSource; + + let state = AppState::default(); + state.add( + "src1", + RegistryItem::TabularSource(TabularSource::new( + b"kind,amount\nweb,10\nphone,20\nweb,30\n".to_vec(), + "csv", + )), + ); + let domain = list_functions() + .into_iter() + .find(|b| b.name.ends_with("extraction_column_domain_items")) + .expect("domain binding is registered"); + let args = serde_json::json!({ + "sources": { "rows": "src1" }, "source_id": "rows", + "table": "rows", "column": "kind", + }); + let out = call(domain, &args, &state).expect("domain succeeds"); + let mut values: Vec = serde_json::from_slice(&out).expect("deserializes"); + values.sort(); + assert_eq!(values, vec!["phone".to_string(), "web".to_string()]); + } + + /// The whole in-memory path, through `call`: bytes -> registry -> catalog -> log, with no + /// filesystem and no database connector. This is the only route available on `wasm32`. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn an_extraction_reads_a_source_held_in_the_registry() { + use crate::core::tabular_source::TabularSource; + + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("shop.sqlite"); + { + let con = rusqlite::Connection::open(&path).expect("open"); + con.execute_batch( + "CREATE TABLE orders (order_id TEXT, placed_at TEXT); + INSERT INTO orders VALUES ('o1', '2024-01-02T03:04:05Z'), + ('o2', '2024-01-03T03:04:05Z');", + ) + .expect("seed"); + } + let bytes = std::fs::read(&path).expect("read"); + + let state = AppState::default(); + state.add( + "src1", + RegistryItem::TabularSource(TabularSource::new(bytes, "sqlite")), + ); + + // Discovery finds the table without ever naming a path. + let discover = list_functions() + .into_iter() + .find(|b| b.name.ends_with("extraction_discover_catalog_items")) + .expect("discover binding is registered"); + let args = serde_json::json!({ "sources": { "shop": "src1" } }); + let out = call(discover, &args, &state).expect("discovery succeeds"); + let catalog: serde_json::Value = serde_json::from_slice(&out).expect("deserializes"); + assert!( + catalog["tables"]["shop"]["orders"].is_object(), + "discovered the table: {catalog}" + ); + + // ..and a run over the same source produces a log handle. + let run = list_functions() + .into_iter() + .find(|b| b.name.ends_with("extraction_run_items")) + .expect("run binding is registered"); + let blueprint = serde_json::json!({ + "version": 1, + "nodes": [{ "id": "n1", "op": { "type": "source", "source_id": "shop", "table": "orders" } }], + "mappings": [{ + "type": "single", + "node": "n1", + "target": { + "type": "event", + "event_type": { "type": "constant", "value": "placed" }, + "id": { "type": "column", "column": "order_id" }, + "timestamp": { "type": "value", "source": { "type": "column", "column": "placed_at" } } + } + }] + }); + let args = serde_json::json!({ "blueprint": blueprint, "sources": { "shop": "src1" } }); + let out = call(run, &args, &state).expect("run succeeds"); + let handle: String = serde_json::from_slice(&out).expect("a handle id"); + let items = state.items.read().expect("lock"); + let Some(RegistryItem::SlimLinkedOCEL(log)) = items.get(&handle) else { + panic!("the run should store a log under its returned handle"); + }; + use crate::core::event_data::object_centric::linked_ocel::LinkedOCELAccess; + assert_eq!(log.get_ev_types().count(), 1); + assert_eq!(log.get_all_evs().count(), 2); + } + + /// Two sources discovered together keep their own entries: the merge is keyed by `source_id`, + /// so neither can overwrite the other's tables. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn two_sources_merge_into_one_catalog_without_clobbering_each_other() { + use crate::core::event_data::object_centric::extraction::Catalog; + use crate::core::tabular_source::TabularSource; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut state = AppState::default(); + for (item, table) in [("srcA", "alpha"), ("srcB", "beta")] { + let path = dir.path().join(format!("{item}.sqlite")); + { + let con = rusqlite::Connection::open(&path).expect("open"); + con.execute_batch(&format!("CREATE TABLE {table} (id TEXT);")) + .expect("seed"); + } + state = { + state.add( + item, + RegistryItem::TabularSource(TabularSource::new( + std::fs::read(&path).expect("read"), + "sqlite", + )), + ); + state + }; + } + + let discover = list_functions() + .into_iter() + .find(|b| b.name == "extraction_discover_catalog_items") + .expect("registered"); + let args = serde_json::json!({ "sources": { "a": "srcA", "b": "srcB" } }); + let out = call(discover, &args, &state).expect("discovery succeeds"); + let catalog: ExtractionCatalog = serde_json::from_slice(&out).expect("deserializes"); + assert!(catalog.table("a", "alpha").is_some(), "{catalog:?}"); + assert!(catalog.table("b", "beta").is_some(), "{catalog:?}"); + } + + /// The bytes route end to end for a workbook: registered as a `TabularSource`, opened by + /// `extraction_discover_catalog_items`, and every sheet reported as a table. + #[cfg(all(feature = "ocel-sqlite", feature = "extraction-dbcon"))] + #[test] + fn a_workbook_registered_as_bytes_discovers_one_table_per_sheet() { + use crate::core::io::Importable; + use crate::core::tabular_source::TabularSource; + + assert!( + ::known_import_formats() + .iter() + .any(|f| f.extension == "xlsx"), + "xlsx must be an advertised import format or nothing can register one" + ); + + let state = AppState::default(); + state.add( + "book", + RegistryItem::TabularSource(TabularSource::new(minimal_xlsx(), "xlsx")), + ); + let discover = list_functions() + .into_iter() + .find(|b| b.name == "extraction_discover_catalog_items") + .expect("registered"); + let args = serde_json::json!({ "sources": { "s": "book" } }); + let out = call(discover, &args, &state).expect("the workbook opens from memory"); + let catalog: serde_json::Value = serde_json::from_slice(&out).expect("catalog is JSON"); + let tables = &catalog["tables"]["s"]; + assert!( + tables.get("orders").is_some(), + "expected a table per sheet, got {tables}" + ); + assert!( + tables["orders"]["columns"].get("id").is_some(), + "expected the header row as columns, got {}", + tables["orders"]["columns"] + ); + } + + /// A one-sheet workbook (`orders`, header `id,total`, one row), as the smallest OOXML a + /// reader accepts. Inline base64 so the test needs no filesystem. + #[cfg(all(feature = "ocel-sqlite", feature = "extraction-dbcon"))] + fn minimal_xlsx() -> Vec { + const MINIMAL_XLSX_BASE64: &str = concat!( + "UEsDBBQAAAAIAAAAIQBbma6u5QAAAAsCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbK2RvVLDMBCEX0WjNhOdk4KC", + "sZ0i0AYKXuCQz7HG+hudEszbIzuBggnQUN1Iu3vfalTvJmfFmRKb4Bu5UZXctfXLeyQWRfHcyCHneA/AeiCHrEIk", + "X5Q+JIe5HNMRIuoRjwTbqroDHXwmn9d53iHb+oF6PNksHqdyfaEksizF/mKcWY3EGK3RmIsOZ999o6yvBFWSi4cH", + "E3lVDBJuEmblZ8A191SenUxH4hlTPqArLpgsvIU0voYwqt+X3GgZ+t5o6oI+uRJRHBNhxwNRdlYtUzk0fvU3fzEz", + "LGPzz0W+9n/2gOW72w9QSwMEFAAAAAgAAAAhAEuDozqWAAAABQEAAAsAAABfcmVscy8ucmVsc43PPQ7CMAwF4KtE", + "PkDdMjCgpl1YuiIuEFL3R23iyAlQbk9GihgY/fz0Wa7bza3qQRJn9hqqooS2qS+0mpSDOM0hqtzwUcOUUjghRjuR", + "M7HgQD5vBhZnUh5lxGDsYkbCQ1keUT4N2Juq6zVI11egrq9A/9g8DLOlM9u7I59+nPhqZNnISEnDtuKTZbkxL0VG", + "AZsadw82b1BLAwQUAAAACAAAACEASj8DtJ4AAAD5AAAADwAAAHhsL3dvcmtib29rLnhtbI2PSw6DMAxErxL5AAS6", + "6AKFsOmGY6RgmggSIzv9HL8RlH1X/oz8PGP6T1zVC1kCpQ6aqobemjfxcidaVBGTdOBz3lqtZfQYnVS0YSrKTBxd", + "LiM/tGyMbhKPmOOqL3V91dGFBAeh5X8YNM9hxBuNz4gpHxDG1eViTXzYBKzZP8ivquQidkA8Ff+g9t0wlRSguA2l", + "4WFqQFujzzN9JrNfUEsDBBQAAAAIAAAAIQBtNul0mgAAAAYBAAAaAAAAeGwvX3JlbHMvd29ya2Jvb2sueG1sLnJl", + "bHONzzsOwjAMBuCrRD5A3TIwoKZdWFgRF4hSt6naPBSb1+2JGBCVGJgs/7Y+y23/8Ku6UeY5Bg1NVUPftWdajZSA", + "3ZxYlY3AGpxIOiCydeQNVzFRKJMxZm+ktHnCZOxiJsJdXe8xfxuwNdVp0JBPQwPq8kz0jx3HcbZ0jPbqKciPE3iP", + "eWFHJAU1eSLR8IkY36WpigrYtbj5sHsBUEsDBBQAAAAIAAAAIQBSsWyOsQAAADQBAAAYAAAAeGwvd29ya3NoZWV0", + "cy9zaGVldDEueG1sdZBvDoIwDMWvsuwAFEg00ZQRjTfwBAtMWdwfsjXg8R1gFvzgt/bX9/qaYvu2hk0qRO1dw6ui", + "5K3A2YdXHJQilqYuNnwgGs8AsRuUlbHwo3Jp8vDBSkpteEIcg5L9arIG6rI8gpXacYEru0mSAoOfWUgpiXZLcak4", + "o4ZrZ7RTdwqJ6yiQhO4RSCAsHXRf9fWfmjxJ82uAFJXz6py3VJOoEKb93o2eikPmmx12p0P+ifgAUEsBAhQAFAAA", + "AAgAAAAhAFuZrq7lAAAACwIAABMAAAAAAAAAAAAAAIABAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAAUAAAA", + "CAAAACEAS4OjOpYAAAAFAQAACwAAAAAAAAAAAAAAgAEWAQAAX3JlbHMvLnJlbHNQSwECFAAUAAAACAAAACEASj8D", + "tJ4AAAD5AAAADwAAAAAAAAAAAAAAgAHVAQAAeGwvd29ya2Jvb2sueG1sUEsBAhQAFAAAAAgAAAAhAG026XSaAAAA", + "BgEAABoAAAAAAAAAAAAAAIABoAIAAHhsL19yZWxzL3dvcmtib29rLnhtbC5yZWxzUEsBAhQAFAAAAAgAAAAhAFKx", + "bI6xAAAANAEAABgAAAAAAAAAAAAAAIABcgMAAHhsL3dvcmtzaGVldHMvc2hlZXQxLnhtbFBLBQYAAAAABQAFAEUB", + "AABZBAAAAAA=", + ); + super::super::decode_base64(MINIMAL_XLSX_BASE64).expect("the fixture is valid base64") + } + + /// A source that cannot be opened is reported against the source id that named it, so a + /// caller working with several of them can tell them apart. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn a_source_that_cannot_be_opened_is_named_in_the_error() { + use crate::core::tabular_source::TabularSource; + + let state = AppState::default(); + // Parquet is advertised as a source format but has no in-memory reader yet. + state.add( + "parquet1", + RegistryItem::TabularSource(TabularSource::new(b"PAR1".to_vec(), "parquet")), + ); + state.add( + "junk", + RegistryItem::TabularSource(TabularSource::new(vec![0u8; 4096], "sqlite")), + ); + state.add( + "notasource", + RegistryItem::SlimLinkedOCEL(SlimLinkedOCEL::new()), + ); + + let discover = list_functions() + .into_iter() + .find(|b| b.name == "extraction_discover_catalog_items") + .expect("registered"); + let err_for = |id: &str| { + let args = serde_json::json!({ "sources": { "s": id } }); + call(discover, &args, &state).expect_err("must not open") + }; + assert!( + err_for("missing").contains("no item 'missing'"), + "{}", + err_for("missing") + ); + assert!( + err_for("notasource").contains("is not a data source"), + "{}", + err_for("notasource") + ); + // Whether a Parquet source can be opened at all is what `extraction-dbcon` decides. With + // it, these four bytes are opened and rejected as malformed Parquet; without it, the + // format itself is unreadable. Either way the source is named rather than silently + // yielding an empty catalog. + let parquet = err_for("parquet1"); + assert!(parquet.contains("source 's'"), "{parquet}"); + #[cfg(feature = "extraction-dbcon")] + assert!(parquet.to_lowercase().contains("parquet file"), "{parquet}"); + #[cfg(not(feature = "extraction-dbcon"))] + assert!(parquet.contains("cannot read 'parquet'"), "{parquet}"); + // Bytes that are not a database fail against the source, not against a table name. + let junk = err_for("junk"); + assert!( + junk.contains("source 's'") && junk.contains("not a SQLite database"), + "{junk}" + ); + } + + /// A `#[bind(state)]` argument is not a JSON argument, so it must be absent from both the + /// schema and the required-arguments list. Leaving it in the latter tells every host (the CLI + /// checks exactly this list) to demand an argument it is given no schema for and can never + /// supply, which makes the binding unreachable. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn a_state_argument_is_not_a_required_json_argument() { + for name in ["extraction_discover_catalog_items", "extraction_run_items"] { + let binding = list_functions() + .into_iter() + .find(|b| b.name == name) + .unwrap_or_else(|| panic!("{name} is registered")); + let declared: Vec = (binding.args)().into_iter().map(|(n, _)| n).collect(); + let required = (binding.required_args)(); + assert!( + !declared.contains(&"state".to_string()), + "{name} must not declare a schema for its state argument: {declared:?}" + ); + for req in &required { + assert!( + declared.contains(req), + "{name} requires '{req}' but declares no schema for it: {declared:?}" + ); + } + assert!( + required.contains(&"sources".to_string()), + "{name} still requires its real arguments: {required:?}" + ); + } + } + + #[test] + fn extraction_validate_round_trips_through_the_registry() { + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_validate") + .expect("extraction_validate registered"); + let args = serde_json::json!({ + "blueprint": flat_blueprint(), + "catalog": flat_catalog(), + }); + let state = AppState::default(); + let bytes = call(binding, &args, &state).expect("call succeeds"); + let errors: Vec = + serde_json::from_slice(&bytes).expect("result deserializes"); + assert!( + errors.is_empty(), + "a valid blueprint validates clean: {errors:?}" + ); + } + + #[test] + fn extraction_validate_reports_an_unknown_source() { + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_validate") + .expect("extraction_validate registered"); + let args = serde_json::json!({ + "blueprint": flat_blueprint(), + "catalog": ExtractionCatalog::new(), + }); + let state = AppState::default(); + let bytes = call(binding, &args, &state).expect("call succeeds"); + let errors: Vec = + serde_json::from_slice(&bytes).expect("result deserializes"); + assert!( + !errors.is_empty(), + "an empty catalog cannot satisfy a blueprint reading table 'events'" + ); + } + + #[test] + fn extraction_compile_round_trips_through_the_registry() { + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_compile") + .expect("extraction_compile registered"); + let args = serde_json::json!({ + "blueprint": flat_blueprint(), + "catalog": flat_catalog(), + "shape": "PerType", + }); + let state = AppState::default(); + let bytes = call(binding, &args, &state).expect("call succeeds"); + let compiled: serde_json::Value = + serde_json::from_slice(&bytes).expect("result deserializes"); + let views = compiled + .get("views") + .and_then(|v| v.as_array()) + .expect("a 'views' array"); + assert!(!views.is_empty(), "compiling a valid blueprint emits views"); + } + + #[test] + fn every_extraction_binding_has_non_empty_schemas() { + // The connected bindings (`extraction_run` and friends) are behind `extraction-dbcon` + // and covered by that module's own registry test. + let expected = ["extraction_validate", "extraction_compile"]; + let registered = list_functions(); + for name in expected { + let binding = registered + .iter() + .find(|b| b.name == name) + .unwrap_or_else(|| panic!("{name} is registered")); + assert!( + !(binding.args)().is_empty(), + "{name} should declare at least one argument" + ); + for (arg_name, schema) in (binding.args)() { + assert!( + schema.is_object(), + "{name}'s argument '{arg_name}' should have a non-empty JSON schema" + ); + } + let return_schema = (binding.return_type)(); + assert!( + return_schema.is_object(), + "{name} should have a non-empty return schema" + ); + } + } +} diff --git a/process_mining/src/bindings/extraction_dbcon_bindings.rs b/process_mining/src/bindings/extraction_dbcon_bindings.rs new file mode 100644 index 00000000..dddc00a0 --- /dev/null +++ b/process_mining/src/bindings/extraction_dbcon_bindings.rs @@ -0,0 +1,680 @@ +//! The extraction bindings that open a real connection, behind the `extraction-dbcon` feature. +//! +//! The sibling `extraction_bindings` module holds the pure ones (`extraction_validate`, +//! `extraction_compile`) and the ones reading registry-held `SQLite` bytes, which need no +//! connector. +//! +//! Every function here takes `connections`, a map from the `source_id` a `Blueprint`'s nodes name +//! to a `dbcon` connection string (`postgres://...`, `sqlite:...`, or a `.csv` path), as a +//! separate argument. A blueprint never carries connection details, so the same blueprint can run +//! against staging and then production unedited. +//! +//! [`extraction_run`] cannot return `(SlimLinkedOCEL, ExtractionReport)` directly, since +//! `#[register_binding]` recognises a big type by matching the whole return type's name against a +//! fixed list and a tuple's rendered name never matches. It instead takes an empty +//! `SlimLinkedOCEL` handle (from `locel_new`) as a `&mut` argument and fills it. + +// See `object_centric::mod`: `ExtractionError` is deliberately descriptive. +#![allow(clippy::result_large_err)] + +use std::collections::HashMap; + +use macros_process_mining::register_binding; + +use crate::bindings::extraction_bindings::merge_into; +#[cfg(not(feature = "ocel-sqlite"))] +use crate::bindings::extraction_bindings::report_error_message; +#[cfg(not(feature = "ocel-sqlite"))] +use crate::bindings::{RegistryItem, StateRef}; +use crate::core::event_data::object_centric::extraction::{ + discover_catalog, extract, Blueprint, Catalog, DbconProviderError, DbconRowProvider, + ExtractionCatalog, ExtractionError, ExtractionReport, ExtractionSink, ExtractionTiming, + ProviderError, RowProvider, SlimOcelSink, TablePreview, +}; +use crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL; +#[cfg(not(feature = "ocel-sqlite"))] +use crate::core::tabular_source::TabularReader; + +/// Failure discovering a catalog, connecting to a source, or running an extraction against one. +/// +/// Never crosses a bindings boundary as a typed value: `#[register_binding(stringify_error)]` +/// converts it to a plain `String`, via [`Display`](std::fmt::Display), at the call boundary. +#[derive(Debug)] +enum ExtractionRunError { + /// A blueprint node, or a direct call, named a `source_id` with no entry in `connections`. + UnknownSource(String), + /// A direct call named a table the connected source does not have. + UnknownTable { source_id: String, table: String }, + /// A query against an open connection failed. + Provider(ProviderError), + /// Connecting to a source, or discovering its schema, failed. + Connect(DbconProviderError), + /// The extraction itself failed. + Extract(ExtractionError), + /// Opening the `DuckDB` output file failed. + #[cfg(feature = "ocel-duckdb")] + Sink(crate::core::event_data::object_centric::extraction::SinkError), +} + +impl std::fmt::Display for ExtractionRunError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnknownSource(id) => write!(f, "no connection given for source '{id}'"), + Self::UnknownTable { source_id, table } => { + write!(f, "source '{source_id}' has no table '{table}'") + } + Self::Provider(e) => write!(f, "{e}"), + Self::Connect(e) => write!(f, "{e}"), + Self::Extract(e) => write!(f, "{e}"), + #[cfg(feature = "ocel-duckdb")] + Self::Sink(e) => write!(f, "{e}"), + } + } +} + +impl From for ExtractionRunError { + fn from(e: DbconProviderError) -> Self { + Self::Connect(e) + } +} + +impl From for ExtractionRunError { + fn from(e: ProviderError) -> Self { + Self::Provider(e) + } +} + +impl From for ExtractionRunError { + fn from(e: ExtractionError) -> Self { + Self::Extract(e) + } +} + +/// Discover the schema of every source in `connections` and merge the results into one +/// [`ExtractionCatalog`], keyed by the same source ids. +fn discover_catalog_from_connections( + connections: &HashMap, +) -> Result { + let mut catalog = ExtractionCatalog::new(); + for (source_id, connection_string) in connections { + merge_into( + &mut catalog, + discover_catalog(source_id, connection_string)?, + ); + } + Ok(catalog) +} + +/// Open a fast, non-discovering connection for every entry in `connections`, keyed the same way. +fn open_providers( + connections: &HashMap, +) -> Result, ExtractionRunError> { + let mut providers = HashMap::with_capacity(connections.len()); + for (source_id, connection_string) in connections { + providers.insert( + source_id.clone(), + DbconRowProvider::connect(source_id, connection_string)?, + ); + } + Ok(providers) +} + +/// Discover `connections`' schema, open a provider for each, and run `blueprint` into `sink`. +/// +/// Shared by every connected binding below, so `extraction_run` and `extraction_run_to_duckdb` +/// differ only in which `ExtractionSink` they construct. +fn run_extraction( + blueprint: &Blueprint, + connections: &HashMap, + catalog: Option, + sink: &mut dyn ExtractionSink, +) -> Result { + let started = std::time::Instant::now(); + // Discovery is a fixed cost paid before a single row is read, and an editor that has already + // discovered a catalog to validate against is holding the very thing this would recompute. + // Taking it as an argument turns that into a skipped phase rather than a repeated one. + let catalog = match catalog { + Some(c) => c, + None => discover_catalog_from_connections(connections)?, + }; + let providers = open_providers(connections)?; + let discovery_ms = started.elapsed().as_millis() as u64; + + let extraction_started = std::time::Instant::now(); + let provider_refs: HashMap = providers + .iter() + .map(|(source_id, provider)| (source_id.clone(), provider as &dyn RowProvider)) + .collect(); + let mut report = extract(blueprint, &catalog, &provider_refs, sink)?; + report.timing = Some(ExtractionTiming { + discovery_ms, + extraction_ms: extraction_started.elapsed().as_millis() as u64, + }); + Ok(report) +} + +/// The source kinds *this build's* connector can open, as `dbcon` backend ids (`"csv"`, +/// `"parquet"`, `"xlsx"`, `"sqlite"`, `"duckdb"`, `"postgres"`). Lets a UI offer exactly the +/// kinds the running binary supports instead of hardcoding a per-build list. +/// +/// Asked of `dbcon` itself rather than derived from this crate's `extraction-dbcon*` features: +/// under Cargo feature unification another crate in the graph can enable a `dbcon` backend +/// those features do not name. +#[register_binding] +fn extraction_connection_kinds() -> Vec { + dbcon::enabled_backends() + .iter() + .map(|s| (*s).to_string()) + .collect() +} + +/// Connect to every source in `connections` and discover its schema, returning one merged +/// [`ExtractionCatalog`] a caller can `extraction_validate` or `extraction_compile` a blueprint +/// against, or edit further with `ExtractionCatalog::with_domain` entries from +/// [`extraction_column_domain`]. +/// +/// `connections` maps each source id a blueprint's nodes name to a `dbcon` connection string +/// (`postgres://...`, `postgresql://...`, `sqlite:...`, or a bare path ending in `.csv`). +#[register_binding(stringify_error)] +fn extraction_discover_catalog( + connections: HashMap, +) -> Result { + discover_catalog_from_connections(&connections) +} + +/// The distinct values `table.column` holds in the source named `source_id`, for compiling a +/// dynamic type name (one read from a column rather than written in the blueprint) to SQL. See +/// `ExtractionCatalog::with_domain`. +/// +/// `connections` must contain an entry for `source_id`. Every other entry is ignored. +#[register_binding(stringify_error)] +fn extraction_column_domain( + connections: HashMap, + source_id: String, + table: String, + column: String, +) -> Result, ExtractionRunError> { + let connection_string = connections + .get(&source_id) + .ok_or_else(|| ExtractionRunError::UnknownSource(source_id.clone()))?; + let provider = DbconRowProvider::connect(&source_id, connection_string)?; + Ok(provider.distinct_values(&table, &column)?) +} + +/// The first `limit` rows of `table` in the source named `source_id`, for showing a person what +/// the data looks like while they build a blueprint against it. +/// +/// Not a substitute for [`extraction_column_domain`]: a preview is incomplete, so it must never +/// be used where the compiler needs a column's full domain. +/// +/// `connections` must contain an entry for `source_id`. Every other entry is ignored. +#[register_binding(stringify_error)] +fn extraction_table_preview( + connections: HashMap, + source_id: String, + table: String, + #[bind(default)] limit: Option, +) -> Result { + let connection_string = connections + .get(&source_id) + .ok_or_else(|| ExtractionRunError::UnknownSource(source_id.clone()))?; + let provider = DbconRowProvider::connect(&source_id, connection_string)?; + // Taking the columns from the schema keeps them in the source's declared order. + let catalog = discover_catalog(&source_id, connection_string)?; + let schema = + catalog + .table(&source_id, &table) + .ok_or_else(|| ExtractionRunError::UnknownTable { + source_id: source_id.clone(), + table: table.clone(), + })?; + let columns: Vec<&str> = schema.columns.keys().map(String::as_str).collect(); + Ok(provider.table_preview(&table, &columns, limit.unwrap_or(DEFAULT_PREVIEW_ROWS))?) +} + +const DEFAULT_PREVIEW_ROWS: usize = 5; + +/// Run `blueprint` against `connections`, filling `ocel` in place and returning the +/// `ExtractionReport`. +/// +/// `ocel` must be an empty log (get one from `locel_new`), since this overwrites it wholesale +/// rather than merging into whatever it already held. `connections` maps each source id +/// `blueprint`'s nodes name to a `dbcon` connection string. The blueprint itself carries no +/// connection details, so the same blueprint can run against different connections (a staging +/// database, then production) with no edit. +#[register_binding(stringify_error)] +fn extraction_run( + ocel: &mut SlimLinkedOCEL, + blueprint: Blueprint, + connections: HashMap, + #[bind(default)] catalog: Option, +) -> Result { + let mut sink = SlimOcelSink::new(); + let report = run_extraction(&blueprint, &connections, catalog, &mut sink)?; + *ocel = sink.into_ocel(); + Ok(report) +} + +/// Run `blueprint` against `connections`, streaming straight to a fresh `DuckDB` file at +/// `target_path` (an existing file there is replaced) instead of holding the log in memory. +/// +/// The right choice for a source too large to fit in RAM. See [`extraction_run`] for a log kept +/// as an in-process handle instead. The written file is read back with, for example, +/// `read_ocel_from_duckdb`. +#[cfg(feature = "ocel-duckdb")] +#[register_binding(stringify_error)] +fn extraction_run_to_duckdb( + blueprint: Blueprint, + connections: HashMap, + target_path: impl AsRef, + #[bind(default)] catalog: Option, +) -> Result { + let mut sink = + crate::core::event_data::object_centric::extraction::DuckDbSink::new(target_path) + .map_err(ExtractionRunError::Sink)?; + run_extraction(&blueprint, &connections, catalog, &mut sink) +} + +/// Discover the schema of sources held in the registry as bytes, for the formats `dbcon` reads +/// from memory: CSV, TSV, Parquet and XLSX. +/// +/// Behind `not(ocel-sqlite)`, since `extraction_discover_catalog_items` covers strictly more: +/// this route goes through [`DbconRowProvider::from_bytes`], which cannot read a `SQLite` file at +/// all (`dbcon` opens `SQLite` by path, and a registry item is bytes). +/// +/// `sources` maps each `source_id` a blueprint names to the registry id of an imported file. +#[cfg(not(feature = "ocel-sqlite"))] +#[register_binding(stringify_error)] +fn extraction_discover_catalog_items_dbcon( + #[bind(state)] state: StateRef<'_>, + sources: HashMap, +) -> Result { + let mut catalog = ExtractionCatalog::new(); + for (source_ids, reader) in open_items(state, &sources)? { + for source_id in source_ids { + merge_into(&mut catalog, reader.get().discover_catalog(&source_id)); + } + } + Ok(catalog) +} + +/// Run `blueprint` against registry-held sources, returning the resulting log. +/// +/// Returns a fresh log rather than filling one given by `&mut`, because a binding cannot both take +/// a `&mut` big type and read the registry (see `StateRef`). The `ExtractionReport` has nowhere to +/// go for the same reason, so a run whose report carries errors fails with them. +/// +/// Behind `not(ocel-sqlite)` for the reason [`extraction_discover_catalog_items_dbcon`] gives. +#[cfg(not(feature = "ocel-sqlite"))] +#[register_binding(stringify_error)] +fn extraction_run_items_dbcon( + #[bind(state)] state: StateRef<'_>, + blueprint: Blueprint, + sources: HashMap, + #[bind(default)] catalog: Option, +) -> Result { + let opened = open_items(state, &sources)?; + let catalog = match catalog { + Some(c) => c, + None => { + let mut discovered = ExtractionCatalog::new(); + for (source_ids, reader) in &opened { + for source_id in source_ids { + merge_into(&mut discovered, reader.get().discover_catalog(source_id)); + } + } + discovered + } + }; + let provider_refs: HashMap = opened + .iter() + .flat_map(|(source_ids, reader)| { + let provider = reader.get() as &dyn RowProvider; + source_ids.iter().map(move |id| (id.clone(), provider)) + }) + .collect(); + let mut sink = SlimOcelSink::new(); + let report = + extract(&blueprint, &catalog, &provider_refs, &mut sink).map_err(|e| e.to_string())?; + if let Some(msg) = report_error_message(&report) { + return Err(msg); + } + Ok(sink.into_ocel()) +} + +/// Borrow every source in `sources` as an opened provider, keyed the same way. +/// +/// [`TabularSource::reader`](crate::core::tabular_source::TabularSource::reader) caches the +/// opened source on the item, so a second discovery or run does not reparse the whole file. +/// +/// Returns the lock guards: the reader is not `Sync`, so it may only be touched while its guard is +/// held. Two `source_id`s naming the same item share one guard, since locking the same mutex twice +/// would deadlock, and items are opened in registry-id order rather than in `sources` iteration +/// order, so two concurrent calls cannot take the same locks in opposite orders. +#[cfg(not(feature = "ocel-sqlite"))] +fn open_items<'a>( + state: StateRef<'a>, + sources: &HashMap, +) -> Result, TabularReader<'a, DbconRowProvider>)>, String> { + let mut by_item: Vec<(&str, Vec)> = Vec::new(); + for (source_id, item_id) in sources { + match by_item.iter_mut().find(|(id, _)| *id == item_id.as_str()) { + Some((_, ids)) => ids.push(source_id.clone()), + None => by_item.push((item_id.as_str(), vec![source_id.clone()])), + } + } + by_item.sort_unstable(); + + let mut out = Vec::with_capacity(by_item.len()); + for (item_id, source_ids) in by_item { + let named = source_ids.join(", "); + let Some(item) = state.get(item_id) else { + return Err(format!("no item '{item_id}' for source '{named}'")); + }; + let RegistryItem::TabularSource(src) = item else { + return Err(format!("item '{item_id}' is not a data source")); + }; + let format = src.format().to_string(); + let reader = src + .reader(|bytes| { + DbconRowProvider::from_bytes(&named, &format, std::sync::Arc::from(bytes)) + }) + .map_err(|e| format!("source '{named}': {e}"))?; + out.push((source_ids, reader)); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bindings::{call, list_functions, AppState, RegistryItem}; + #[cfg(feature = "ocel-sqlite")] + use crate::core::event_data::object_centric::extraction::{AttributeMapping, FlatEventTable}; + use crate::core::event_data::object_centric::extraction::{Catalog, TableSchema}; + + /// A tiny flat-event-table blueprint reading a table `events(case_id, activity, ts)` from + /// source `db`, built through `Blueprint::from_flat_event_table` so it carries only a + /// `source_id` string, never a connection string, matching what a real caller sends over the + /// bindings boundary. + #[cfg(feature = "ocel-sqlite")] + fn flat_blueprint() -> Blueprint { + Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".to_string(), + table: "events".to_string(), + case_id: "case_id".to_string(), + activity: "activity".to_string(), + timestamp: "ts".to_string(), + case_object_type: "Case".to_string(), + case_attributes: Vec::::new(), + event_attributes: Vec::::new(), + }) + } + + #[cfg(feature = "ocel-sqlite")] + fn write_fixture_sqlite() -> (tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("fixture.sqlite"); + let con = rusqlite::Connection::open(&path).expect("open sqlite file"); + con.execute_batch("CREATE TABLE events (case_id TEXT, activity TEXT, ts TEXT);") + .expect("create table"); + let rows = [ + ("A", "create", "2020-01-01T00:00:00Z"), + ("A", "close", "2020-01-02T00:00:00Z"), + ("B", "create", "2020-01-01T00:00:00Z"), + ]; + for (case_id, activity, ts) in rows { + con.execute( + "INSERT INTO events (case_id, activity, ts) VALUES (?1, ?2, ?3)", + rusqlite::params![case_id, activity, ts], + ) + .expect("insert row"); + } + drop(con); + (dir, path) + } + + /// Every binding this module contributes is registered, with non-empty schemas. + #[test] + fn every_connected_binding_has_non_empty_schemas() { + let expected = [ + "extraction_discover_catalog", + "extraction_column_domain", + "extraction_run", + #[cfg(feature = "ocel-duckdb")] + "extraction_run_to_duckdb", + ]; + let registered = list_functions(); + for name in expected { + let binding = registered + .iter() + .find(|b| b.name == name) + .unwrap_or_else(|| panic!("{name} is registered")); + assert!( + !(binding.args)().is_empty(), + "{name} should declare at least one argument" + ); + for (arg_name, schema) in (binding.args)() { + assert!( + schema.is_object(), + "{name}'s argument '{arg_name}' should have a non-empty JSON schema" + ); + } + let return_schema = (binding.return_type)(); + assert!( + return_schema.is_object(), + "{name} should have a non-empty return schema" + ); + } + } + + /// Discover, validate and extract a real `SQLite` fixture entirely through the registry, as a + /// Python or `TypeScript` caller does, and check the returned handle resolves to a + /// `SlimLinkedOCEL` with the expected contents. + /// + /// Also pins that a blueprint carries no connection details: `bp`, built by + /// [`flat_blueprint`], is serialized to JSON and back before use and only ever carries the + /// source id `"db"`, with the connection string supplied separately in `connections`. + #[cfg(feature = "ocel-sqlite")] + #[test] + fn discover_validate_and_extract_a_sqlite_fixture_through_the_registry() { + let (_dir, path) = write_fixture_sqlite(); + let connection_string = format!("sqlite:{}", path.display()); + + // The blueprint a caller would hold carries no connection string at all. Round-trip it + // through JSON, as a caller relaying it between processes would. + let bp_json = serde_json::to_value(flat_blueprint()).expect("serialize blueprint"); + assert!( + !serde_json::to_string(&bp_json) + .expect("stringify") + .to_ascii_lowercase() + .contains("sqlite:"), + "a blueprint must carry no connection string, even after a JSON round trip" + ); + + let state = AppState::default(); + let connections = serde_json::json!({ "db": connection_string }); + + let discover = list_functions() + .into_iter() + .find(|b| b.name == "extraction_discover_catalog") + .expect("extraction_discover_catalog registered"); + let catalog_bytes = call( + discover, + &serde_json::json!({ "connections": connections }), + &state, + ) + .expect("discover_catalog succeeds"); + let catalog: ExtractionCatalog = + serde_json::from_slice(&catalog_bytes).expect("catalog deserializes"); + assert!(catalog.table("db", "events").is_some()); + + let validate_fn = list_functions() + .into_iter() + .find(|b| b.name == "extraction_validate") + .expect("extraction_validate registered by process_mining"); + let validate_bytes = call( + validate_fn, + &serde_json::json!({ "blueprint": bp_json, "catalog": catalog }), + &state, + ) + .expect("validate succeeds"); + let errors: serde_json::Value = + serde_json::from_slice(&validate_bytes).expect("errors deserialize"); + assert_eq!( + errors.as_array().map(Vec::len), + Some(0), + "the fixture blueprint should validate: {errors:?}" + ); + + // `locel_new` first, exactly as a real caller would, to get the handle `extraction_run` + // fills in place. + let locel_new = list_functions() + .into_iter() + .find(|b| b.name == "locel_new") + .expect("locel_new registered"); + let handle_bytes = call(locel_new, &serde_json::json!({}), &state).expect("locel_new"); + let handle: String = serde_json::from_slice(&handle_bytes).expect("handle deserializes"); + + let run_binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_run") + .expect("extraction_run registered"); + let run_args = serde_json::json!({ + "ocel": handle, + "blueprint": bp_json, + "connections": connections, + }); + let report_bytes = call(run_binding, &run_args, &state).expect("extraction_run succeeds"); + // `ExtractionReport` derives `Serialize` but not `Deserialize` (some `ExtractionError` + // variants carry `&'static str`, which no deserializer can manufacture), so a caller + // reads this outbound-only value as JSON, not as the Rust struct. + let report: serde_json::Value = + serde_json::from_slice(&report_bytes).expect("report deserializes as JSON"); + let errors = report + .get("errors") + .and_then(|e| e.as_array()) + .expect("an 'errors' array"); + assert!(errors.is_empty(), "no errors expected: {errors:?}"); + let rows_read: u64 = report + .get("per_mapping") + .and_then(|v| v.as_array()) + .expect("a 'per_mapping' array") + .iter() + .map(|m| { + m.get("rows_read") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0) + }) + .sum(); + assert_eq!(rows_read, 3, "the fixture table has 3 rows"); + + let items = state.items.read().unwrap(); + let stored = items.get(&handle).expect("handle resolves in the registry"); + let RegistryItem::SlimLinkedOCEL(locel) = stored else { + panic!("handle resolved to {stored:?}, not a SlimLinkedOCEL"); + }; + assert_eq!(locel.get_evs_of_type("create").count(), 2); + assert_eq!(locel.get_evs_of_type("close").count(), 1); + assert_eq!(locel.get_obs_of_type("Case").count(), 2); + } + + #[cfg(feature = "ocel-sqlite")] + #[test] + fn extraction_column_domain_reports_distinct_values() { + let (_dir, path) = write_fixture_sqlite(); + let connection_string = format!("sqlite:{}", path.display()); + let state = AppState::default(); + + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_column_domain") + .expect("extraction_column_domain registered"); + let args = serde_json::json!({ + "connections": { "db": connection_string }, + "source_id": "db", + "table": "events", + "column": "activity", + }); + let bytes = call(binding, &args, &state).expect("call succeeds"); + let mut values: Vec = serde_json::from_slice(&bytes).expect("deserializes"); + values.sort(); + assert_eq!(values, vec!["close".to_string(), "create".to_string()]); + } + + /// The functional half of `extraction_run_to_duckdb`: called through the registry, it writes a + /// real `DuckDB` file that reads back as the log the fixture describes. + /// `every_connected_binding_has_non_empty_schemas` above checks the binding's schema, which + /// is satisfied by a binding that writes nothing at all. + /// + /// Reads the file back through + /// [`read_consolidated_ocel_from_duckdb_path`](crate::core::event_data::object_centric::ocel_sql::read_consolidated_ocel_from_duckdb_path) + /// rather than opening a `duckdb::Connection` here. + #[cfg(all(feature = "ocel-duckdb", feature = "ocel-sqlite"))] + #[test] + fn extraction_run_to_duckdb_writes_a_readable_file() { + use crate::core::event_data::object_centric::ocel_sql::read_consolidated_ocel_from_duckdb_path; + + let (_dir, path) = write_fixture_sqlite(); + let connection_string = format!("sqlite:{}", path.display()); + let out_dir = tempfile::tempdir().expect("tempdir"); + let out_path = out_dir.path().join("out.duckdb"); + let state = AppState::default(); + + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_run_to_duckdb") + .expect("extraction_run_to_duckdb registered"); + let args = serde_json::json!({ + "blueprint": flat_blueprint(), + "connections": { "db": connection_string }, + "target_path": out_path.to_str().unwrap(), + }); + let bytes = call(binding, &args, &state).expect("call succeeds"); + let report: serde_json::Value = + serde_json::from_slice(&bytes).expect("report deserializes as JSON"); + let errors = report + .get("errors") + .and_then(|e| e.as_array()) + .expect("an 'errors' array"); + assert!(errors.is_empty(), "no errors expected: {errors:?}"); + assert!( + out_path.exists(), + "the DuckDB file should have been written" + ); + + let ocel = read_consolidated_ocel_from_duckdb_path(&out_path).expect("read duckdb back"); + assert_eq!(ocel.events.len(), 3); + assert_eq!(ocel.objects.len(), 2); + } + + /// Pins that `extraction_run`'s `ocel` argument is declared as a `SlimLinkedOCEL` registry + /// reference, not a plain string: the schema shape `resolve_argument` needs to accept the + /// live handle `locel_new` hands back, exercised end-to-end by the registry test above. + #[test] + fn extraction_run_declares_ocel_as_a_registry_reference() { + let binding = list_functions() + .into_iter() + .find(|b| b.name == "extraction_run") + .expect("extraction_run registered"); + let (_, schema) = (binding.args)() + .into_iter() + .find(|(name, _)| name == "ocel") + .expect("an 'ocel' argument"); + assert_eq!( + schema.get("x-registry-ref").and_then(|v| v.as_str()), + Some("SlimLinkedOCEL") + ); + } + + /// The `TableSchema` route is what a caller building a catalog by hand uses, rather than + /// discovering one from a live connection. + #[test] + fn a_catalog_can_be_built_by_hand() { + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("events", [("case_id", "TEXT", false)]), + ); + assert!(catalog.table("db", "events").is_some()); + } +} diff --git a/process_mining/src/bindings/mod.rs b/process_mining/src/bindings/mod.rs index f9d429eb..fce0b113 100644 --- a/process_mining/src/bindings/mod.rs +++ b/process_mining/src/bindings/mod.rs @@ -28,7 +28,8 @@ //! ## Helper Features //! //! - **Auto-Loading**: The `resolve_argument` function can automatically load "Big Types" from -//! file paths if the argument schema indicates a registry reference. +//! file paths, from base64 bytes or from inline JSON if the argument schema indicates a +//! registry reference. `call_resolved` is `call` with that applied to every argument. use crate::core::{ event_data::{ @@ -41,24 +42,62 @@ use crate::core::{ io::ExtensionWithMime, EventLog, }; -use macros_process_mining::register_binding; +pub use macros_process_mining::{register_binding, CustomRegistryEntity, RegistryEntity}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{collections::HashMap, fmt::Display}; use std::{str::FromStr, sync::RwLock}; +/// The formats the object-centric kinds accept on top of what an OCEL reader handles, by way of +/// the case-centric conversion. Empty without the feature that brings it. +fn case_centric_import_formats() -> Vec { + if cfg!(feature = "extraction-blueprint") { + vec![ + ExtensionWithMime::new("xes", "application/xml"), + ExtensionWithMime::new("xes.gz", "application/gzip"), + ] + } else { + Vec::new() + } +} + +/// Whether `item_kind` should read `format` as a case-centric log rather than as an OCEL. +#[cfg(feature = "extraction-blueprint")] +fn reads_as_case_centric(item_kind: &RegistryItemKind, format: &str) -> bool { + matches!( + item_kind, + RegistryItemKind::OCEL | RegistryItemKind::SlimLinkedOCEL + ) && (format.ends_with("xes") || format.ends_with("xes.gz")) +} + +/// Convert an already-parsed case-centric log into whichever object-centric kind was asked for. +#[cfg(feature = "extraction-blueprint")] +fn case_centric_as(item_kind: &RegistryItemKind, log: &EventLog) -> Result { + use crate::core::event_data::object_centric::extraction::{ + event_log_to_ocel, event_log_to_slim_ocel, + }; + match item_kind { + RegistryItemKind::OCEL => event_log_to_ocel(log).map(RegistryItem::OCEL), + _ => event_log_to_slim_ocel(log).map(RegistryItem::SlimLinkedOCEL), + } + .map_err(|e| e.to_string()) +} + /// Manually maintained Registry enum of 'big' types /// /// NOTE: When extending this with a new variant, make sure to also update `BIG_TYPES_NAMES` in the macro crate. #[derive(Debug)] #[allow(clippy::large_enum_variant, missing_docs)] pub enum RegistryItem { + TabularSource(TabularSource), EventLogActivityProjection(EventLogActivityProjection), IndexLinkedOCEL(IndexLinkedOCEL), SlimLinkedOCEL(SlimLinkedOCEL), EventLog(EventLog), OCEL(OCEL), + /// A handle type contributed by a downstream crate, see [`CustomRegistryValue`]. + Custom(Box), } impl From for RegistryItem { @@ -87,30 +126,41 @@ impl From for RegistryItem { } } -#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(missing_docs)] pub enum RegistryItemKind { + TabularSource, EventLogActivityProjection, IndexLinkedOCEL, SlimLinkedOCEL, EventLog, OCEL, + /// A kind contributed by a downstream crate, named by [`CustomRegistryValue::kind_name`]. + /// + /// `&'static str` rather than `String` so the enum stays `Copy`. + Custom(&'static str), } impl Display for RegistryItemKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { + write!(f, "{}", self.name()) + } +} + +impl RegistryItemKind { + /// The name this kind is known by, as used in ids, schemas and `x-registry-ref`. + pub fn name(&self) -> &'static str { + match self { RegistryItemKind::EventLogActivityProjection => "EventLogActivityProjection", RegistryItemKind::IndexLinkedOCEL => "IndexLinkedOCEL", RegistryItemKind::SlimLinkedOCEL => "SlimLinkedOCEL", RegistryItemKind::EventLog => "EventLog", RegistryItemKind::OCEL => "OCEL", - }; - write!(f, "{}", s) + RegistryItemKind::TabularSource => "TabularSource", + RegistryItemKind::Custom(name) => name, + } } -} -impl RegistryItemKind { /// Get all kinds of `RegistryItemKind` pub fn all_kinds() -> &'static [Self] { &[ @@ -119,9 +169,22 @@ impl RegistryItemKind { RegistryItemKind::EventLogActivityProjection, RegistryItemKind::SlimLinkedOCEL, RegistryItemKind::IndexLinkedOCEL, + RegistryItemKind::TabularSource, ] } + /// Get all kinds, including the ones downstream crates registered. + /// + /// [`RegistryItemKind::all_kinds`] stays limited to the built-ins, so a host that only knows + /// those keeps seeing exactly those. + pub fn all_registered_kinds() -> Vec { + Self::all_kinds() + .iter() + .copied() + .chain(custom_kinds().into_iter().map(|c| Self::Custom(c.name))) + .collect() + } + /// Get known import formats pub fn known_import_formats(&self) -> Vec { match self { @@ -130,8 +193,15 @@ impl RegistryItemKind { } RegistryItemKind::IndexLinkedOCEL => IndexLinkedOCEL::known_import_formats(), RegistryItemKind::EventLog => EventLog::known_import_formats(), - RegistryItemKind::OCEL => OCEL::known_import_formats(), - RegistryItemKind::SlimLinkedOCEL => OCEL::known_import_formats(), + RegistryItemKind::OCEL | RegistryItemKind::SlimLinkedOCEL => { + let mut formats = OCEL::known_import_formats(); + formats.extend(case_centric_import_formats()); + formats + } + RegistryItemKind::TabularSource => TabularSource::known_import_formats(), + RegistryItemKind::Custom(name) => custom_kind(name) + .map(|c| (c.import_formats)()) + .unwrap_or_default(), } } /// Get known export formats @@ -144,10 +214,32 @@ impl RegistryItemKind { RegistryItemKind::EventLog => EventLog::known_export_formats(), RegistryItemKind::OCEL => OCEL::known_export_formats(), RegistryItemKind::SlimLinkedOCEL => OCEL::known_export_formats(), + // A source is read, never written back out. + RegistryItemKind::TabularSource => Vec::new(), + RegistryItemKind::Custom(name) => custom_kind(name) + .map(|c| (c.export_formats)()) + .unwrap_or_default(), } } } +// Hand-written rather than derived because `Custom(&'static str)` has no derivable `Deserialize`, +// and because the derive would give it the externally tagged `{"Custom": "X"}` form while every +// other variant is a bare string. Going through `Display`/`FromStr` keeps the wire format a +// string for all kinds, byte-for-byte what the derive produced for the built-in six. +impl Serialize for RegistryItemKind { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.name()) + } +} + +impl<'de> Deserialize<'de> for RegistryItemKind { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + s.parse().map_err(serde::de::Error::custom) + } +} + impl std::str::FromStr for RegistryItemKind { type Err = String; @@ -158,14 +250,42 @@ impl std::str::FromStr for RegistryItemKind { "EventLog" => Ok(RegistryItemKind::EventLog), "OCEL" => Ok(RegistryItemKind::OCEL), "SlimLinkedOCEL" => Ok(RegistryItemKind::SlimLinkedOCEL), - _ => Err(format!("Unknown RegistryItemKind: {}", s)), + "TabularSource" => Ok(RegistryItemKind::TabularSource), + // Only registered custom names resolve, so a typo still reports as unknown. + _ => custom_kind(s) + .map(|c| RegistryItemKind::Custom(c.name)) + .ok_or_else(|| format!("Unknown RegistryItemKind: {}", s)), } } } use crate::core::io::{Exportable, Importable}; +use crate::core::tabular_source::TabularSource; impl RegistryItem { + /// Wrap a downstream handle type as a registry item. + /// + /// There is no `From` impl for this: it would overlap the concrete `From` impls above. + pub fn custom(value: impl CustomRegistryValue) -> Self { + RegistryItem::Custom(Box::new(value)) + } + + /// Borrow the item as the custom type `T`, or `None` if it is not one. + pub fn as_custom(&self) -> Option<&T> { + match self { + RegistryItem::Custom(v) => (&**v as &dyn std::any::Any).downcast_ref::(), + _ => None, + } + } + + /// Mutably borrow the item as the custom type `T`, or `None` if it is not one. + pub fn as_custom_mut(&mut self) -> Option<&mut T> { + match self { + RegistryItem::Custom(v) => (&mut **v as &mut dyn std::any::Any).downcast_mut::(), + _ => None, + } + } + /// Convert the registry item to a JSON value /// /// For "Big Types", this performs a full serialization. @@ -183,13 +303,30 @@ impl RegistryItem { RegistryItem::EventLogActivityProjection(proj) => { serde_json::to_value(proj).map_err(|e| e.to_string()) } + // Serialising a whole database into JSON would be a mistake, not a courtesy. + RegistryItem::TabularSource(src) => Ok(serde_json::json!({ + "format": src.format(), + "bytes": src.bytes().len(), + })), + RegistryItem::Custom(v) => v.to_value(), } } /// Try to load a registry item from a file path based on the expected type name + /// + /// Takes the same `xes`/`xes.gz` route as [`RegistryItem::load_from_bytes`], on the format + /// inferred from the path. pub fn load_from_path(item_kind: &RegistryItemKind, path: &str) -> Result { let path = std::path::Path::new(path); + #[cfg(feature = "extraction-blueprint")] + if crate::core::io::infer_format_from_path(path) + .is_some_and(|format| reads_as_case_centric(item_kind, &format)) + { + let log = EventLog::import_from_path(path).map_err(|e| e.to_string())?; + return case_centric_as(item_kind, &log); + } + match item_kind { RegistryItemKind::EventLog => Ok(RegistryItem::EventLog( EventLog::import_from_path(path).map_err(|e| e.to_string())?, @@ -209,15 +346,33 @@ impl RegistryItem { .map_err(|e| e.to_string())?, )) } + RegistryItemKind::TabularSource => Ok(RegistryItem::TabularSource( + TabularSource::import_from_path(path).map_err(|e| e.to_string())?, + )), + RegistryItemKind::Custom(name) => { + let info = custom_kind(name).ok_or_else(|| unregistered_kind_msg(name))?; + (info.from_path)(path) + } } } /// Try to load a registry item from bytes based on the expected type name and format + /// + /// The object-centric kinds also accept `xes`/`xes.gz`, which no OCEL reader handles: the + /// bytes are parsed as a case-centric [`EventLog`] and converted with + /// [`write_event_log_to_sink`](crate::core::event_data::object_centric::extraction::write_event_log_to_sink), + /// one object per case. Only with `extraction-blueprint`, which is what brings the sink the + /// conversion is written against. pub fn load_from_bytes( item_kind: &RegistryItemKind, data: &[u8], format: &str, ) -> Result { + #[cfg(feature = "extraction-blueprint")] + if reads_as_case_centric(item_kind, format) { + let log = EventLog::import_from_bytes(data, format).map_err(|e| e.to_string())?; + return case_centric_as(item_kind, &log); + } match item_kind { RegistryItemKind::EventLog => Ok(RegistryItem::EventLog( EventLog::import_from_bytes(data, format).map_err(|e| e.to_string())?, @@ -239,6 +394,48 @@ impl RegistryItem { .map_err(|e| e.to_string())?, )) } + RegistryItemKind::TabularSource => Ok(RegistryItem::TabularSource( + TabularSource::import_from_bytes(data, format).map_err(|e| e.to_string())?, + )), + RegistryItemKind::Custom(name) => { + let info = custom_kind(name).ok_or_else(|| unregistered_kind_msg(name))?; + (info.from_bytes)(data, format) + } + } + } + + /// Try to build a registry item from the JSON form of the given kind. + /// + /// The inverse of [`RegistryItem::to_value`] rather than a second JSON dialect: what that + /// method writes for a kind is what this method reads back for it. + /// + /// # Errors + /// Returns the deserializer's message, or an explanation for a kind that has no JSON form. + pub fn from_json_value(item_kind: &RegistryItemKind, value: &Value) -> Result { + match item_kind { + RegistryItemKind::EventLog => serde_json::from_value(value.clone()) + .map(RegistryItem::EventLog) + .map_err(|e| e.to_string()), + RegistryItemKind::OCEL => serde_json::from_value(value.clone()) + .map(RegistryItem::OCEL) + .map_err(|e| e.to_string()), + RegistryItemKind::IndexLinkedOCEL => serde_json::from_value(value.clone()) + .map(RegistryItem::IndexLinkedOCEL) + .map_err(|e| e.to_string()), + RegistryItemKind::EventLogActivityProjection => serde_json::from_value(value.clone()) + .map(RegistryItem::EventLogActivityProjection) + .map_err(|e| e.to_string()), + // Mirrors `to_value`, which hands out the constructed OCEL: the linked form itself is + // not `Deserialize`, so the OCEL is the JSON form of both directions. + RegistryItemKind::SlimLinkedOCEL => serde_json::from_value::(value.clone()) + .map(|ocel| RegistryItem::SlimLinkedOCEL(SlimLinkedOCEL::from_ocel(ocel))) + .map_err(|e| e.to_string()), + // `to_value` only reports a source's format and size, which is nothing to rebuild from. + RegistryItemKind::TabularSource => Err("a data source has no JSON form".to_string()), + RegistryItemKind::Custom(name) => { + let info = custom_kind(name).ok_or_else(|| unregistered_kind_msg(name))?; + (info.from_value)(value) + } } } @@ -252,22 +449,71 @@ impl RegistryItem { RegistryItem::EventLog(_) => RegistryItemKind::EventLog, RegistryItem::OCEL(_) => RegistryItemKind::OCEL, RegistryItem::SlimLinkedOCEL(_) => RegistryItemKind::SlimLinkedOCEL, + RegistryItem::TabularSource(_) => RegistryItemKind::TabularSource, + RegistryItem::Custom(v) => RegistryItemKind::Custom(v.kind()), } } - /// Export the registry item to a file path + /// Export the registry item to a file path, in the format the path names. + /// + /// Only the inference happens here, since each kind reads a path by its own rule (`.csv` + /// means `ocel.csv` for an OCEL, a directory means the bundled format). The writing itself is + /// [`RegistryItem::export_to_path_as`]. pub fn export_to_path(&self, path: impl AsRef) -> Result<(), String> { + let path = path.as_ref(); + let inferred = match self { + RegistryItem::EventLog(_) => ::infer_format(path), + RegistryItem::OCEL(_) => ::infer_format(path), + RegistryItem::IndexLinkedOCEL(_) => ::infer_format(path), + RegistryItem::SlimLinkedOCEL(_) => ::infer_format(path), + RegistryItem::EventLogActivityProjection(_) => { + ::infer_format(path) + } + // Neither kind has an inference rule of its own. + RegistryItem::TabularSource(_) | RegistryItem::Custom(_) => { + crate::core::io::infer_format_from_path(path) + } + }; + let format = inferred + .ok_or_else(|| format!("Cannot infer format from path {}", path.to_string_lossy()))?; + self.export_to_path_as(path, &format) + } + + /// Export the registry item to a file path in an explicitly named format. + /// + /// Unlike [`RegistryItem::export_to_path`], the format is given rather than read off the + /// path: a directory carries no extension, and the OCEL 2.0 bundled format's uncompressed + /// form is a directory. This is also the route that avoids materialising the whole export in + /// memory, which [`RegistryItem::export_to_bytes`] cannot. + /// + /// # Errors + /// Returns the underlying exporter's message, or an explanation for a kind that has no + /// file representation. + pub fn export_to_path_as( + &self, + path: impl AsRef, + format: &str, + ) -> Result<(), String> { let path = path.as_ref(); match self { - RegistryItem::EventLog(x) => x.export_to_path(path).map_err(|e| e.to_string()), - RegistryItem::OCEL(x) => x.export_to_path(path).map_err(|e| e.to_string()), - RegistryItem::IndexLinkedOCEL(x) => x.export_to_path(path).map_err(|e| e.to_string()), + RegistryItem::EventLog(x) => x + .export_to_path_as(path, format, ()) + .map_err(|e| e.to_string()), + RegistryItem::OCEL(x) => x + .export_to_path_as(path, format, ()) + .map_err(|e| e.to_string()), + RegistryItem::IndexLinkedOCEL(x) => x + .export_to_path_as(path, format, ()) + .map_err(|e| e.to_string()), RegistryItem::SlimLinkedOCEL(x) => x - .construct_ocel() - .export_to_path(path) + .export_to_path_as(path, format, ()) + .map_err(|e| e.to_string()), + RegistryItem::EventLogActivityProjection(x) => x + .export_to_path_as(path, format, ()) .map_err(|e| e.to_string()), - RegistryItem::EventLogActivityProjection(x) => { - x.export_to_path(path).map_err(|e| e.to_string()) + RegistryItem::TabularSource(_) => Err("a data source cannot be exported".to_string()), + RegistryItem::Custom(v) => { + std::fs::write(path, v.export_to_bytes(format)?).map_err(|e| e.to_string()) } } } @@ -292,6 +538,10 @@ impl RegistryItem { RegistryItem::EventLogActivityProjection(x) => x .export_to_writer(&mut bytes, format) .map_err(|e| e.to_string())?, + RegistryItem::TabularSource(_) => { + return Err("a data source cannot be exported".to_string()) + } + RegistryItem::Custom(x) => return x.export_to_bytes(format), }; Ok(bytes) } @@ -319,8 +569,311 @@ impl RegistryItem { } } +/// The name of a custom value's kind, reachable through `dyn CustomRegistryValue`. +/// +/// Blanket-implemented from [`CustomRegistryValue::kind_name`]. There is nothing to write by hand. +/// It exists because `kind_name` is `where Self: Sized`, as an associated function has to be, or +/// [`CustomRegistryValue`] would not be dyn-compatible and could not be boxed into +/// [`RegistryItem::Custom`]. +pub trait CustomRegistryKind { + /// Name this value's kind is known by. + fn kind(&self) -> &'static str; +} + +impl CustomRegistryKind for T { + fn kind(&self) -> &'static str { + T::kind_name() + } +} + +/// A registry handle type owned by a downstream crate. +/// +/// Implement this, then a `#[bind(handle)]` argument or a `#[register_binding(returns_handle)]` +/// return of that type crosses the binding boundary as a registry id instead of being serialized +/// as JSON, the same treatment the built-in big types get but without an entry in the macro +/// crate's list of them. Derive [`macro@CustomRegistryEntity`] to accept it as a `#[bind(handle)]` +/// argument. +/// +/// Registering the type with [`crate::register_custom_registry_kind!`] is optional and adds what +/// needs a name rather than a Rust type: [`RegistryItemKind::from_str`](std::str::FromStr::from_str) +/// resolution, [`RegistryItem::load_from_path`] / [`RegistryItem::load_from_bytes`], and the +/// format lists. Unregistered, a value still stores, resolves by id and exports. +/// +/// `Send + Sync` are load-bearing: the registry lives in a `static` in the wasm hosts, so +/// [`AppState`] has to stay `Sync`. +pub trait CustomRegistryValue: + CustomRegistryKind + std::any::Any + Send + Sync + std::fmt::Debug +{ + /// Name this kind is known by in ids, in `x-registry-ref` and in [`RegistryItemKind::Custom`]. + fn kind_name() -> &'static str + where + Self: Sized; + + /// JSON projection of the value, as [`RegistryItem::to_value`] gives for a built-in kind. + fn to_value(&self) -> Result; + + /// Rebuild the value from the JSON form [`CustomRegistryValue::to_value`] produces. + fn from_value(_value: &Value) -> Result + where + Self: Sized, + { + Err(format!("{} cannot be read from JSON", Self::kind_name())) + } + + /// Read the value from bytes in the named format. + fn from_bytes(_bytes: &[u8], format: &str) -> Result + where + Self: Sized, + { + Err(format!( + "{} cannot be read from '{}' bytes", + Self::kind_name(), + format + )) + } + + /// Read the value from a file, defaulting to [`CustomRegistryValue::from_bytes`] with the + /// format inferred from the extension. + fn from_path(path: &std::path::Path) -> Result + where + Self: Sized, + { + let format = crate::core::io::infer_format_from_path(path) + .ok_or_else(|| format!("Cannot infer format from path {}", path.to_string_lossy()))?; + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + Self::from_bytes(&bytes, &format) + } + + /// Write the value out in the named format. + fn export_to_bytes(&self, format: &str) -> Result, String> { + Err(format!( + "{} cannot be exported as '{}'", + self.kind(), + format + )) + } + + /// Formats [`CustomRegistryValue::from_bytes`] accepts. + fn known_import_formats() -> Vec + where + Self: Sized, + { + Vec::new() + } + + /// Formats [`CustomRegistryValue::export_to_bytes`] produces. + fn known_export_formats() -> Vec + where + Self: Sized, + { + Vec::new() + } +} + +/// A custom kind registered by a downstream crate, so its name resolves back to its loaders. +/// +/// Submitted by [`crate::register_custom_registry_kind!`], not constructed by hand. +#[derive(Debug)] +pub struct CustomKindInfo { + /// The kind name, matching [`CustomRegistryValue::kind_name`]. + pub name: &'static str, + /// [`CustomRegistryValue::from_path`], wrapped into a [`RegistryItem`]. + pub from_path: fn(&std::path::Path) -> Result, + /// [`CustomRegistryValue::from_bytes`], wrapped into a [`RegistryItem`]. + pub from_bytes: fn(&[u8], &str) -> Result, + /// [`CustomRegistryValue::from_value`], wrapped into a [`RegistryItem`]. + pub from_value: fn(&Value) -> Result, + /// [`CustomRegistryValue::known_import_formats`]. + pub import_formats: fn() -> Vec, + /// [`CustomRegistryValue::known_export_formats`]. + pub export_formats: fn() -> Vec, +} +inventory::collect!(CustomKindInfo); + +/// The registered custom kind called `name`, if there is one. +pub fn custom_kind(name: &str) -> Option<&'static CustomKindInfo> { + inventory::iter:: + .into_iter() + .find(|c| c.name == name) +} + +/// All custom kinds registered by downstream crates. +pub fn custom_kinds() -> Vec<&'static CustomKindInfo> { + inventory::iter::.into_iter().collect() +} + +fn unregistered_kind_msg(name: &str) -> String { + format!( + "'{}' is not a registered custom kind (see register_custom_registry_kind!)", + name + ) +} + +#[doc(hidden)] +pub fn __custom_from_path( + path: &std::path::Path, +) -> Result { + T::from_path(path).map(RegistryItem::custom) +} +#[doc(hidden)] +pub fn __custom_from_bytes( + bytes: &[u8], + format: &str, +) -> Result { + T::from_bytes(bytes, format).map(RegistryItem::custom) +} +#[doc(hidden)] +pub fn __custom_from_value(value: &Value) -> Result { + T::from_value(value).map(RegistryItem::custom) +} +#[doc(hidden)] +pub fn __custom_import_formats() -> Vec { + T::known_import_formats() +} +#[doc(hidden)] +pub fn __custom_export_formats() -> Vec { + T::known_export_formats() +} + +/// Register a [`CustomRegistryValue`] implementor so its kind name resolves to its loaders. +/// +/// The name defaults to the type's own tokens, so pass it explicitly for a path or a generic +/// instantiation: +/// +/// ```ignore +/// register_custom_registry_kind!(MyHandle); +/// register_custom_registry_kind!(some::module::MyHandle, "MyHandle"); +/// ``` +/// +/// The name given here has to match [`CustomRegistryValue::kind_name`], which is what +/// [`RegistryItem::kind`] reports. +#[macro_export] +macro_rules! register_custom_registry_kind { + ($t:ty) => { + $crate::register_custom_registry_kind!($t, ::core::stringify!($t)); + }; + ($t:ty, $name:expr) => { + $crate::__private::inventory::submit! { + $crate::bindings::CustomKindInfo { + name: $name, + from_path: $crate::bindings::__custom_from_path::<$t>, + from_bytes: $crate::bindings::__custom_from_bytes::<$t>, + from_value: $crate::bindings::__custom_from_value::<$t>, + import_formats: $crate::bindings::__custom_import_formats::<$t>, + export_formats: $crate::bindings::__custom_export_formats::<$t>, + } + } + }; +} + /// Inner App State pub type InnerAppState = HashMap; + +/// Read-only access to the registry from inside a binding, requested with `#[bind(state)]`. +/// +/// For a binding that must look up items it is given the ids of, such as an extraction naming one +/// source per id. Every other argument arrives as JSON, and a big type arrives already resolved. +/// +/// Not `&AppState`: that owns the `RwLock`, and the caller already holds a guard when the body +/// runs. `std::sync::RwLock` is not reentrant, so locking again would deadlock. +/// +/// Read-only, and only on bindings with no `&mut` big-type argument. See the macro's own check. +#[derive(Debug, Clone, Copy)] +pub struct StateRef<'a> { + items: &'a InnerAppState, +} + +impl<'a> StateRef<'a> { + /// Wrap the already-locked registry. Called by `#[register_binding]`, not by hand. + #[must_use] + pub fn new(items: &'a InnerAppState) -> Self { + Self { items } + } + + /// The item stored under `id`, if any. + #[must_use] + pub fn get(&self, id: &str) -> Option<&'a RegistryItem> { + self.items.get(id) + } + + /// Whether `id` names a stored item. + #[must_use] + pub fn contains(&self, id: &str) -> bool { + self.items.contains_key(id) + } +} + +/// Writable access to the whole registry, requested with `#[bind(state_mut)]`. +/// +/// For a binding that manages the registry itself rather than one resolved item, e.g. evicting +/// items to free memory. A `#[bind(handle)]` or `&mut` big-type +/// argument is still the right tool for "mutate the one item named by this parameter"; reach for +/// `state_mut` only when the set of items to touch is not known until the body runs. +/// +/// Not `&mut AppState`, for the same reason [`StateRef`] is not `&AppState`: the caller already +/// holds the write guard, and `std::sync::RwLock` is not reentrant. +/// +/// The macro's own check makes this the only argument that can ask for the write lock on a +/// binding that takes it. See [`StateRef`]'s docs for why a shared reference to the same registry +/// cannot coexist with it. +#[derive(Debug)] +pub struct StateRefMut<'a> { + items: &'a mut InnerAppState, +} + +impl<'a> StateRefMut<'a> { + /// Wrap the already-locked registry. Called by `#[register_binding]`, not by hand. + #[must_use] + pub fn new(items: &'a mut InnerAppState) -> Self { + Self { items } + } + + /// The item stored under `id`, if any. + #[must_use] + pub fn get(&self, id: &str) -> Option<&RegistryItem> { + self.items.get(id) + } + + /// The item stored under `id`, mutably, if any. + #[must_use] + pub fn get_mut(&mut self, id: &str) -> Option<&mut RegistryItem> { + self.items.get_mut(id) + } + + /// Whether `id` names a stored item. + #[must_use] + pub fn contains(&self, id: &str) -> bool { + self.items.contains_key(id) + } + + /// Drop the item stored under `id`, returning it if there was one. + pub fn remove(&mut self, id: &str) -> Option { + self.items.remove(id) + } + + /// Drop every item in the registry. + pub fn clear(&mut self) { + self.items.clear(); + } + + /// How many items are currently stored. + #[must_use] + pub fn len(&self) -> usize { + self.items.len() + } + + /// Whether the registry currently holds no items. + #[must_use] + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// The id of every stored item, in no particular order. + pub fn ids(&self) -> impl Iterator { + self.items.keys() + } +} + /// State that can store 'big' types #[derive(Debug, Default)] pub struct AppState { @@ -328,13 +881,34 @@ pub struct AppState { pub items: RwLock, } impl AppState { + /// The registry for reading, recovering from a poisoned lock. + /// + /// A binding body that panics leaves the lock poisoned, but the registry is a plain map no + /// half-finished insert can leave inconsistent, so refusing every later call would be worse. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, InnerAppState> { + self.items + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// The registry for writing. Recovers from a poisoned lock, see [`AppState::read`]. + pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, InnerAppState> { + self.items + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + /// Add the passed registry item pub fn add(&self, id: impl Into, item: impl Into) { - self.items.write().unwrap().insert(id.into(), item.into()); + self.write().insert(id.into(), item.into()); + } + /// Drop the item stored under `id`, returning it if there was one. + pub fn remove(&self, id: &str) -> Option { + self.write().remove(id) } /// Check if the state contains the passed key pub fn contains_key(&self, id: &str) -> bool { - self.items.read().unwrap().contains_key(id) + self.read().contains_key(id) } } @@ -467,56 +1041,192 @@ where } } +/// The forms a registry-reference argument can take, see [`resolve_argument`]. +enum HandleArg<'a> { + /// A bare string: an id if the registry knows it, a file path otherwise. + Bare(&'a str), + Id(&'a str), + Path(&'a str), + Bytes { + b64: &'a str, + format: &'a str, + }, + Inline(&'a Value), +} + +/// Decide which form a registry-reference argument is in, or `None` for a value that is none of +/// them and is left alone. +/// +/// An object is a wrapper only when its key set is exactly one of the recognised ones and the +/// recognised keys hold strings. Both halves matter: a real OCEL or `EventLog` object carries +/// dozens of keys and so can never be mistaken for a wrapper, and a domain object that happens to +/// be `{"bytes": 5}` is read as itself rather than reported as a malformed wrapper. +fn classify_handle_arg(value: &Value) -> Option> { + match value { + Value::String(s) => Some(HandleArg::Bare(s)), + Value::Object(map) => { + let string_field = |k: &str| map.get(k).and_then(Value::as_str); + match map.len() { + 1 => { + if let Some(id) = string_field("id") { + Some(HandleArg::Id(id)) + } else if let Some(path) = string_field("path") { + Some(HandleArg::Path(path)) + } else if let Some(inner) = map.get("inline") { + Some(HandleArg::Inline(inner)) + } else { + Some(HandleArg::Inline(value)) + } + } + 2 => match (string_field("bytes"), string_field("format")) { + (Some(b64), Some(format)) => Some(HandleArg::Bytes { b64, format }), + _ => Some(HandleArg::Inline(value)), + }, + _ => Some(HandleArg::Inline(value)), + } + } + Value::Array(_) => Some(HandleArg::Inline(value)), + // Notably `null`, which has to reach `extract_param` untouched for `#[bind(default)]` + // to still apply. + _ => None, + } +} + +/// Accepts both the padded and the unpadded encoding: which of the two a host emits is not +/// something the caller should have to know. +fn decode_base64(s: &str) -> Result, String> { + use base64::Engine; + const ENGINE: base64::engine::GeneralPurpose = base64::engine::GeneralPurpose::new( + &base64::alphabet::STANDARD, + base64::engine::GeneralPurposeConfig::new() + .with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent), + ); + ENGINE.decode(s).map_err(|e| e.to_string()) +} + +/// Resolve a string that names a stored item, returning `None` if the registry does not know it. +/// +/// Converts on a kind mismatch, storing the result under `{id}_as_{arg_ref}`. +fn resolve_stored_id(id: &str, arg_ref: &str, state: &AppState) -> Result, String> { + // An id of the right kind inserts nothing, and taking the write lock for it would serialise + // every argument of every call against all other writers. + { + let items = state.read(); + match items.get(id) { + None => return Ok(None), + Some(item) if item.kind().to_string() == arg_ref => { + return Ok(Some(Value::String(id.to_string()))) + } + Some(_) => {} + } + } + + let mut items = state.write(); + let Some(item) = items.get(id) else { + return Ok(None); + }; + if item.kind().to_string() == arg_ref { + return Ok(Some(Value::String(id.to_string()))); + } + + // Try conversion + let target_kind = RegistryItemKind::from_str(arg_ref)?; + match item.convert(target_kind) { + Ok(converted) => { + let new_id = format!("{}_as_{}", id, arg_ref); + items.insert(new_id.clone(), converted); + Ok(Some(Value::String(new_id))) + } + Err(e) => Err(format!( + "Type mismatch for ID '{}': expected {}, found {}. Conversion failed: {}", + id, + arg_ref, + item.kind(), + e + )), + } +} + /// Resolve an argument value based on its schema and the current state. /// /// This function handles: -/// 1. Loading "Big Types" from file paths if the schema indicates a registry reference. +/// 1. Materialising "Big Types" if the schema indicates a registry reference, from any of the +/// forms below. Everything but the id itself is stored, and the new id is what comes back, so +/// a caller always ends up with a plain string id. /// 2. Loading JSON objects from files if the value is a path ending in `.json`. /// 3. Parsing JSON strings if the value is a string but the schema expects an object/array. +/// +/// The registry-reference forms are: +/// +/// | value | meaning | +/// |---|---| +/// | `"log1"` | a stored id, or else a file path | +/// | `{"id": "log1"}` | a stored id only, never the filesystem | +/// | `{"path": "/a/b.xes.gz"}` | a file path only, never a stored id | +/// | `{"bytes": "", "format": "xes.gz"}` | the bytes themselves, for a host with no filesystem | +/// | `{"inline": }` | the JSON form of the item, as [`RegistryItem::to_value`] writes it | +/// | any other object or array | the JSON form, unwrapped | +/// +/// The two id forms look up an item that already exists, so a mismatched kind is converted where +/// a conversion exists. The other forms have no kind to mismatch: the path, the bytes and the JSON +/// are read as the referenced kind directly. pub fn resolve_argument( arg_name: &str, value: Value, schema: &Value, state: &AppState, +) -> Result { + resolve_argument_tracked(arg_name, value, schema, state, &mut Vec::new()) +} + +/// [`resolve_argument`], pushing every id it stores an item under onto `minted`. +/// +/// Those ids never reach the caller, so only the requester of the resolution can drop them again. +fn resolve_argument_tracked( + arg_name: &str, + value: Value, + schema: &Value, + state: &AppState, + minted: &mut Vec, ) -> Result { let schema_obj = schema.as_object().ok_or("Invalid schema")?; // Case 1: Registry Reference if let Some(arg_ref) = schema_obj.get("x-registry-ref").and_then(|r| r.as_str()) { - // If the value is already a string ID that exists in the registry, use it. - if let Some(id) = value.as_str() { - let mut items = state.items.write().map_err(|e| e.to_string())?; - if let Some(item) = items.get(id) { - if item.kind().to_string() == arg_ref { - return Ok(value); - } - - // Try conversion - use std::str::FromStr; - let target_kind = RegistryItemKind::from_str(arg_ref)?; - match item.convert(target_kind) { - Ok(converted) => { - let new_id = format!("{}_as_{}", id, arg_ref); - items.insert(new_id.clone(), converted); - return Ok(serde_json::Value::String(new_id)); - } - Err(e) => { - return Err(format!( - "Type mismatch for ID '{}': expected {}, found {}. Conversion failed: {}", - id, - arg_ref, - item.kind(), - e - )) + if let Some(handle_arg) = classify_handle_arg(&value) { + let invalid = |e: String| format!("Invalid Argument: {}\n{}", arg_name, e); + // A kind that no downstream crate registered still resolves by id, so this is looked + // up only where a loader is actually needed. + let target_kind = || RegistryItemKind::from_str(arg_ref).map_err(invalid); + let item = match handle_arg { + HandleArg::Bare(id) => { + if let Some(resolved) = resolve_stored_id(id, arg_ref, state)? { + return Ok(resolved); } + // Otherwise, try to load it from file + RegistryItem::load_from_path(&RegistryItemKind::from_str(arg_ref)?, id)? } - } - drop(items); - - // Otherwise, try to load it from file - let item = RegistryItem::load_from_path(&RegistryItemKind::from_str(arg_ref)?, id)?; + HandleArg::Id(id) => { + return resolve_stored_id(id, arg_ref, state)?.ok_or_else(|| { + invalid(format!("No {} is stored under the ID '{}'", arg_ref, id)) + }) + } + HandleArg::Path(path) => { + RegistryItem::load_from_path(&target_kind()?, path).map_err(invalid)? + } + HandleArg::Bytes { b64, format } => { + let bytes = decode_base64(b64) + .map_err(|e| invalid(format!("'bytes' is not valid base64: {}", e)))?; + RegistryItem::load_from_bytes(&target_kind()?, &bytes, format) + .map_err(invalid)? + } + HandleArg::Inline(inner) => { + RegistryItem::from_json_value(&target_kind()?, inner).map_err(invalid)? + } + }; let stored_name = format!("A{}_{}", arg_name, uuid::Uuid::new_v4()); state.add(&stored_name, item); + minted.push(stored_name.clone()); return Ok(serde_json::Value::String(stored_name)); } } @@ -551,8 +1261,68 @@ pub fn resolve_argument( /// Call the specified function with the passed arguments. /// Returns the result pre-serialized as UTF-8 JSON bytes. +/// +/// A panic in the binding body is caught and reported as an error, because hosts reach this +/// across an FFI or wasm boundary, where an unwind is undefined behaviour. pub fn call(binding: &Binding, args: &Value, state: &AppState) -> Result, String> { - (binding.handler)(args, state) + let called = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + (binding.handler)(args, state) + })); + called.unwrap_or_else(|payload| { + let what = payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "panicked".to_string()); + Err(format!("{} panicked: {}", binding.name, what)) + }) +} + +/// [`call`], with every argument first put through [`resolve_argument`]. +/// +/// This accepts a registry reference given as anything other than a stored id: a path, base64 +/// bytes, or the item inline. Arguments the binding does not declare are passed through +/// untouched. +/// +/// Separate from [`call`] rather than folded into it: resolution reads files named by a plain +/// string argument, so a host exposing bindings to something it does not trust keeps the +/// choice. +pub fn call_resolved(binding: &Binding, args: &Value, state: &AppState) -> Result, String> { + let Some(passed) = args.as_object() else { + return call(binding, args, state); + }; + let schemas = (binding.args)(); + let mut resolved = serde_json::Map::with_capacity(passed.len()); + let mut minted: Vec = Vec::new(); + for (name, value) in passed { + let value = match schemas.iter().find(|(n, _)| n == name) { + Some((_, schema)) => { + match resolve_argument_tracked(name, value.clone(), schema, state, &mut minted) { + Ok(value) => value, + Err(e) => { + drop_minted(state, &minted); + return Err(e); + } + } + } + None => value.clone(), + }; + resolved.insert(name.clone(), value); + } + let result = call(binding, &Value::Object(resolved), state); + drop_minted(state, &minted); + result +} + +/// Drop the registry entries [`call_resolved`] stored for the duration of one call. +fn drop_minted(state: &AppState, minted: &[String]) { + if minted.is_empty() { + return; + } + let mut items = state.write(); + for id in minted { + items.remove(id); + } } /// Get a list of all functions available through bindings @@ -572,6 +1342,10 @@ pub fn get_fn_binding(id: &str) -> Option<&'static Binding> { inventory::iter::.into_iter().find(|b| b.id == id) } +#[cfg(feature = "extraction-blueprint")] +mod extraction_bindings; +#[cfg(feature = "extraction-dbcon")] +mod extraction_dbcon_bindings; mod path_schema_bindings; mod slim_ocel_bindings; @@ -685,4 +1459,727 @@ mod tests { "Mismatch in number of types between RegistryItem and macros_process_mining" ); } + + /// A handle type of the shape a downstream crate would define: not in `BIG_TYPES_NAMES`, not + /// `Deserialize`, reached only through the registry. + #[derive(Debug, Clone, PartialEq, CustomRegistryEntity)] + struct DummyHandle { + label: String, + hits: usize, + } + + impl CustomRegistryValue for DummyHandle { + fn kind_name() -> &'static str { + "DummyHandle" + } + fn to_value(&self) -> Result { + Ok(serde_json::json!({ "label": self.label, "hits": self.hits })) + } + fn from_value(value: &Value) -> Result { + Ok(DummyHandle { + label: value["label"] + .as_str() + .ok_or("DummyHandle needs a string 'label'")? + .to_string(), + hits: value["hits"].as_u64().unwrap_or(0) as usize, + }) + } + fn from_bytes(bytes: &[u8], format: &str) -> Result { + if format != "txt" { + return Err(format!("DummyHandle cannot be read from '{}'", format)); + } + Ok(DummyHandle { + label: String::from_utf8_lossy(bytes).to_string(), + hits: 0, + }) + } + fn export_to_bytes(&self, format: &str) -> Result, String> { + if format != "txt" { + return Err(format!("DummyHandle cannot be written as '{}'", format)); + } + Ok(self.label.as_bytes().to_vec()) + } + fn known_import_formats() -> Vec { + vec![ExtensionWithMime::new("txt", "text/plain")] + } + fn known_export_formats() -> Vec { + vec![ExtensionWithMime::new("txt", "text/plain")] + } + } + + crate::register_custom_registry_kind!(DummyHandle); + + /// The label of a stored handle, taken by shared reference. + #[register_binding] + fn dummy_label(#[bind(handle)] h: &DummyHandle) -> String { + h.label.clone() + } + + /// Bump a stored handle's counter in place. + #[register_binding] + fn dummy_bump(#[bind(handle)] h: &mut DummyHandle) -> usize { + h.hits += 1; + h.hits + } + + /// Build a new handle and hand back its registry id. + #[register_binding(returns_handle)] + fn dummy_new(label: String) -> DummyHandle { + DummyHandle { label, hits: 0 } + } + + /// Bump a stored handle and hand back a copy of it: a handle result stored through the write + /// guard, which is a different insert site than [`dummy_new`]'s. + #[register_binding(returns_handle)] + fn dummy_fork(#[bind(handle)] h: &mut DummyHandle) -> DummyHandle { + h.hits += 1; + h.clone() + } + + /// A big-type result stored through the write guard, the last of the four insert sites. + #[register_binding] + fn dummy_log_of(#[bind(handle)] h: &mut DummyHandle) -> EventLog { + h.hits += 1; + EventLog::default() + } + + /// Drop every item in the registry: the `#[bind(state_mut)]` escape hatch, for a binding + /// that manages the registry itself (evicting items under memory pressure, say) rather than + /// mutating one item named by an argument. + #[register_binding] + fn dummy_clear_all(#[bind(state_mut)] mut state: StateRefMut<'_>) -> usize { + let n = state.len(); + state.clear(); + n + } + + fn binding_named(name: &str) -> &'static Binding { + list_functions() + .into_iter() + .find(|b| b.name == name) + .unwrap_or_else(|| panic!("no binding named {}", name)) + } + + #[test] + fn custom_registry_kind_is_a_string_everywhere() { + let custom = RegistryItemKind::Custom("DummyHandle"); + assert_eq!(custom.to_string(), "DummyHandle"); + assert_eq!("DummyHandle".parse::().unwrap(), custom); + assert_eq!( + serde_json::to_value(custom).unwrap(), + serde_json::json!("DummyHandle") + ); + assert_eq!( + serde_json::from_value::(serde_json::json!("DummyHandle")).unwrap(), + custom + ); + // The built-in six keep the exact wire form the derive gave them. + assert_eq!( + serde_json::to_value(RegistryItemKind::OCEL).unwrap(), + serde_json::json!("OCEL") + ); + // An unregistered name is still an error, not a `Custom`. + assert!("NoSuchKind".parse::().is_err()); + assert!(RegistryItemKind::all_registered_kinds().contains(&custom)); + assert_eq!(RegistryItemKind::all_kinds().len(), 6); + assert_eq!(custom.known_import_formats().len(), 1); + assert_eq!(custom.known_export_formats().len(), 1); + } + + #[test] + fn custom_registry_item_import_and_export() { + let kind: RegistryItemKind = "DummyHandle".parse().unwrap(); + let item = RegistryItem::load_from_bytes(&kind, b"from-bytes", "txt").unwrap(); + assert_eq!(item.kind(), kind); + assert_eq!(item.as_custom::().unwrap().label, "from-bytes"); + assert_eq!(item.export_to_bytes("txt").unwrap(), b"from-bytes"); + assert!(item.export_to_bytes("xes").is_err()); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("handle.txt"); + item.export_to_path(&path).unwrap(); + let reloaded = RegistryItem::load_from_path(&kind, path.to_str().unwrap()).unwrap(); + assert_eq!( + reloaded.as_custom::().unwrap(), + item.as_custom::().unwrap() + ); + } + + #[test] + fn custom_handle_crosses_the_binding_boundary() { + let state = AppState::default(); + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "one".to_string(), + hits: 0, + }), + ); + + { + let items = state.items.read().unwrap(); + let item = items.get("d1").unwrap(); + assert_eq!(item.kind(), RegistryItemKind::Custom("DummyHandle")); + assert_eq!(item.as_custom::().unwrap().label, "one"); + assert!(RegistryItem::EventLog(EventLog::default()) + .as_custom::() + .is_none()); + assert_eq!( + item.to_value().unwrap(), + serde_json::json!({ "label": "one", "hits": 0 }) + ); + } + + // A shared handle argument arrives as an id and is borrowed out of the registry. + let shared = binding_named("dummy_label"); + assert_eq!( + (shared.args)()[0].1["x-registry-ref"], + serde_json::json!("DummyHandle") + ); + let out = call(shared, &serde_json::json!({ "h": "d1" }), &state).unwrap(); + assert_eq!(out, b"\"one\""); + + // A `&mut` handle argument goes through the write lock and the change is visible after. + let bump = binding_named("dummy_bump"); + let out = call(bump, &serde_json::json!({ "h": "d1" }), &state).unwrap(); + assert_eq!(out, b"1"); + let out = call(bump, &serde_json::json!({ "h": "d1" }), &state).unwrap(); + assert_eq!(out, b"2"); + assert_eq!( + state + .items + .read() + .unwrap() + .get("d1") + .unwrap() + .as_custom::() + .unwrap() + .hits, + 2 + ); + + // A `returns_handle` binding stores its result and reports the new id. + let make = binding_named("dummy_new"); + assert_eq!( + (make.return_type)()["x-registry-ref"], + serde_json::json!("DummyHandle") + ); + let out = call(make, &serde_json::json!({ "label": "two" }), &state).unwrap(); + let new_id: String = serde_json::from_slice(&out).unwrap(); + let items = state.items.read().unwrap(); + let created = items.get(&new_id).unwrap(); + assert_eq!(created.kind(), RegistryItemKind::Custom("DummyHandle")); + assert_eq!( + created.as_custom::().unwrap(), + &DummyHandle { + label: "two".to_string(), + hits: 0 + } + ); + + // A wrong id is reported, not silently mistaken for a handle. + assert!(call(shared, &serde_json::json!({ "h": "nope" }), &state).is_err()); + } + + #[test] + fn state_mut_reaches_every_item_by_id_not_just_one_named_argument() { + let state = AppState::default(); + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "one".to_string(), + hits: 0, + }), + ); + state.add("d2", RegistryItem::EventLog(EventLog::default())); + + // `#[bind(state_mut)]` is not a JSON argument: no schema, not in the required list. + let clear_all = binding_named("dummy_clear_all"); + assert!((clear_all.args)().is_empty()); + assert!((clear_all.required_args)().is_empty()); + + let out = call(clear_all, &serde_json::json!({}), &state).unwrap(); + let n: usize = serde_json::from_slice(&out).unwrap(); + assert_eq!(n, 2, "both items were counted before being cleared"); + assert!( + state.items.read().unwrap().is_empty(), + "state_mut actually reached and cleared items no argument named" + ); + } + + fn arg_schema(binding: &str, arg: &str) -> Value { + (binding_named(binding).args)() + .into_iter() + .find(|(n, _)| n == arg) + .unwrap_or_else(|| panic!("{} has no argument {}", binding, arg)) + .1 + } + + fn base64_of(bytes: &[u8]) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(bytes) + } + + /// A stored handle behind the id the resolved value names. + fn stored_handle(state: &AppState, resolved: &Value) -> DummyHandle { + state + .items + .read() + .unwrap() + .get(resolved.as_str().unwrap()) + .unwrap() + .as_custom::() + .unwrap() + .clone() + } + + #[test] + fn handle_argument_accepts_every_documented_form() { + let schema = arg_schema("dummy_label", "h"); + let state = AppState::default(); + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "stored".to_string(), + hits: 0, + }), + ); + + // A bare id that is stored comes back untouched, exactly as before. + assert_eq!( + resolve_argument("h", serde_json::json!("d1"), &schema, &state).unwrap(), + serde_json::json!("d1") + ); + // The explicit id wrapper does the same, without ever looking at the filesystem. + assert_eq!( + resolve_argument("h", serde_json::json!({ "id": "d1" }), &schema, &state).unwrap(), + serde_json::json!("d1") + ); + + // A path wrapper. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("handle.txt"); + std::fs::write(&path, b"from-path").unwrap(); + let resolved = resolve_argument( + "h", + serde_json::json!({ "path": path.to_str().unwrap() }), + &schema, + &state, + ) + .unwrap(); + assert_eq!(stored_handle(&state, &resolved).label, "from-path"); + + // Bytes plus a format, the form that needs no filesystem. + let resolved = resolve_argument( + "h", + serde_json::json!({ "bytes": base64_of(b"from-bytes"), "format": "txt" }), + &schema, + &state, + ) + .unwrap(); + assert_eq!(stored_handle(&state, &resolved).label, "from-bytes"); + // Unpadded base64 is accepted too. + let resolved = resolve_argument( + "h", + serde_json::json!({ "bytes": base64_of(b"abcde").trim_end_matches('='), "format": "txt" }), + &schema, + &state, + ) + .unwrap(); + assert_eq!(stored_handle(&state, &resolved).label, "abcde"); + + // Wrapped inline JSON. + let resolved = resolve_argument( + "h", + serde_json::json!({ "inline": { "label": "wrapped", "hits": 3 } }), + &schema, + &state, + ) + .unwrap(); + assert_eq!( + stored_handle(&state, &resolved), + DummyHandle { + label: "wrapped".to_string(), + hits: 3 + } + ); + + // Bare inline JSON: two keys, so it is exactly the shape a wrapper could have been. + let resolved = resolve_argument( + "h", + serde_json::json!({ "label": "bare", "hits": 7 }), + &schema, + &state, + ) + .unwrap(); + assert_eq!( + stored_handle(&state, &resolved), + DummyHandle { + label: "bare".to_string(), + hits: 7 + } + ); + + // `null` still reaches the binding untouched, so `#[bind(default)]` keeps working. + assert_eq!( + resolve_argument("h", Value::Null, &schema, &state).unwrap(), + Value::Null + ); + + // Bad inputs name the argument. + let err = resolve_argument( + "h", + serde_json::json!({ "bytes": "not base64!!", "format": "txt" }), + &schema, + &state, + ) + .unwrap_err(); + assert!(err.starts_with("Invalid Argument: h\n"), "{}", err); + assert!(err.contains("base64"), "{}", err); + + let err = resolve_argument( + "h", + serde_json::json!({ "bytes": base64_of(b"x"), "format": "xes" }), + &schema, + &state, + ) + .unwrap_err(); + assert!(err.starts_with("Invalid Argument: h\n"), "{}", err); + + let err = resolve_argument("h", serde_json::json!({ "id": "nope" }), &schema, &state) + .unwrap_err(); + assert!(err.contains("nope"), "{}", err); + + let err = + resolve_argument("h", serde_json::json!({ "hits": 1 }), &schema, &state).unwrap_err(); + assert!(err.starts_with("Invalid Argument: h\n"), "{}", err); + + // A missing path is still an error, and the bare form is still a path fallback. + assert!( + resolve_argument("h", serde_json::json!("no/such/file.txt"), &schema, &state).is_err() + ); + } + + fn tiny_ocel_json() -> Value { + serde_json::json!({ + "eventTypes": [], + "objectTypes": [{ "name": "item", "attributes": [] }], + "events": [], + "objects": [ + { "id": "i1", "type": "item" }, + { "id": "i2", "type": "item" } + ] + }) + } + + #[test] + fn built_in_handle_argument_forms_convert() { + // `num_objects` takes a `SlimLinkedOCEL`, so an `OCEL` in any form needs converting. + let schema = arg_schema("num_objects", "ocel"); + assert_eq!( + schema["x-registry-ref"], + serde_json::json!("SlimLinkedOCEL") + ); + let state = AppState::default(); + let ocel: OCEL = serde_json::from_value(tiny_ocel_json()).unwrap(); + state.add("o1", ocel); + + // A stored id of the wrong kind is converted and stored under the derived name. + let resolved = resolve_argument("ocel", serde_json::json!("o1"), &schema, &state).unwrap(); + assert_eq!(resolved, serde_json::json!("o1_as_SlimLinkedOCEL")); + assert_eq!( + state.items.read().unwrap()["o1_as_SlimLinkedOCEL"].kind(), + RegistryItemKind::SlimLinkedOCEL + ); + // The id wrapper converts the same way. + assert_eq!( + resolve_argument("ocel", serde_json::json!({ "id": "o1" }), &schema, &state).unwrap(), + serde_json::json!("o1_as_SlimLinkedOCEL") + ); + + // Inline JSON, and the bytes of the same JSON, both land as a `SlimLinkedOCEL`. + for value in [ + serde_json::json!({ "inline": tiny_ocel_json() }), + tiny_ocel_json(), + serde_json::json!({ + "bytes": base64_of(serde_json::to_string(&tiny_ocel_json()).unwrap().as_bytes()), + "format": "json" + }), + ] { + let resolved = resolve_argument("ocel", value, &schema, &state).unwrap(); + let items = state.items.read().unwrap(); + let item = &items[resolved.as_str().unwrap()]; + assert_eq!(item.kind(), RegistryItemKind::SlimLinkedOCEL); + let RegistryItem::SlimLinkedOCEL(locel) = item else { + unreachable!() + }; + assert_eq!(locel.get_num_obs(), 2); + } + + let err = resolve_argument( + "ocel", + serde_json::json!({ "inline": { "not": "an ocel", "at": "all" } }), + &schema, + &state, + ) + .unwrap_err(); + assert!(err.starts_with("Invalid Argument: ocel\n"), "{}", err); + } + + #[test] + fn call_resolved_accepts_the_new_forms() { + let state = AppState::default(); + let out = call_resolved( + binding_named("dummy_label"), + &serde_json::json!({ "h": { "inline": { "label": "inline-label", "hits": 0 } } }), + &state, + ) + .unwrap(); + assert_eq!(out, b"\"inline-label\""); + + let out = call_resolved( + binding_named("num_objects"), + &serde_json::json!({ "ocel": tiny_ocel_json() }), + &state, + ) + .unwrap(); + assert_eq!(out, b"2"); + + // A plain id keeps working through the same entry point. + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "stored".to_string(), + hits: 0, + }), + ); + let out = call_resolved( + binding_named("dummy_label"), + &serde_json::json!({ "h": "d1" }), + &state, + ) + .unwrap(); + assert_eq!(out, b"\"stored\""); + } + + /// The id a call reports back. + fn called_id(name: &str, args: Value, state: &AppState) -> String { + let out = call(binding_named(name), &args, state).unwrap(); + serde_json::from_slice(&out).unwrap() + } + + fn stored_handle_at(state: &AppState, id: &str) -> DummyHandle { + state + .items + .read() + .unwrap() + .get(id) + .unwrap_or_else(|| panic!("nothing stored under {}", id)) + .as_custom::() + .unwrap() + .clone() + } + + #[test] + fn output_id_stores_the_result_under_exactly_that_id() { + let state = AppState::default(); + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "src".to_string(), + hits: 0, + }), + ); + state.add( + "o1", + serde_json::from_value::(tiny_ocel_json()).unwrap(), + ); + + // Custom handle, read-lock path. + let id = called_id( + "dummy_new", + serde_json::json!({ "label": "two", "output_id": "chosen" }), + &state, + ); + assert_eq!(id, "chosen"); + assert_eq!( + stored_handle_at(&state, "chosen"), + DummyHandle { + label: "two".to_string(), + hits: 0 + } + ); + + // Custom handle, write-guard path. + let id = called_id( + "dummy_fork", + serde_json::json!({ "h": "d1", "output_id": "forked" }), + &state, + ); + assert_eq!(id, "forked"); + assert_eq!(stored_handle_at(&state, "forked").hits, 1); + + // Big type, write-guard path. + let id = called_id( + "dummy_log_of", + serde_json::json!({ "h": "d1", "output_id": "a_log" }), + &state, + ); + assert_eq!(id, "a_log"); + assert_eq!( + state.items.read().unwrap()["a_log"].kind(), + RegistryItemKind::EventLog + ); + + // Big type, read-lock path. + let id = called_id( + "index_link_ocel", + serde_json::json!({ "ocel": "o1", "output_id": "linked" }), + &state, + ); + assert_eq!(id, "linked"); + assert_eq!( + state.items.read().unwrap()["linked"].kind(), + RegistryItemKind::IndexLinkedOCEL + ); + + // The same id twice replaces, rather than minting a second item. + let before = state.items.read().unwrap().len(); + let id = called_id( + "dummy_new", + serde_json::json!({ "label": "again", "output_id": "chosen" }), + &state, + ); + assert_eq!(id, "chosen"); + assert_eq!(stored_handle_at(&state, "chosen").label, "again"); + assert_eq!(state.items.read().unwrap().len(), before); + + // A non-string is rejected by name rather than silently ignored. + let err = call( + binding_named("dummy_new"), + &serde_json::json!({ "label": "x", "output_id": 7 }), + &state, + ) + .unwrap_err(); + assert!(err.contains("output_id"), "{}", err); + } + + #[test] + fn omitting_output_id_keeps_the_generated_id() { + let state = AppState::default(); + state.add( + "d1", + RegistryItem::custom(DummyHandle { + label: "src".to_string(), + hits: 0, + }), + ); + + for args in [ + serde_json::json!({ "label": "two" }), + serde_json::json!({ "label": "two", "output_id": null }), + ] { + let id = called_id("dummy_new", args, &state); + assert!(id.starts_with("res_"), "got {}", id); + assert_eq!(stored_handle_at(&state, &id).label, "two"); + } + + // Every insert site keeps the old behaviour, not just the read-lock one. + let id = called_id("dummy_fork", serde_json::json!({ "h": "d1" }), &state); + assert!(id.starts_with("res_"), "got {}", id); + let id = called_id("dummy_log_of", serde_json::json!({ "h": "d1" }), &state); + assert!(id.starts_with("res_"), "got {}", id); + } + + #[test] + fn output_id_is_declared_only_where_a_handle_is_returned() { + for name in [ + "dummy_new", + "dummy_fork", + "dummy_log_of", + "index_link_ocel", + "slim_link_ocel", + ] { + let binding = binding_named(name); + let schema = arg_schema(name, "output_id"); + assert_eq!(schema["type"], serde_json::json!(["string", "null"])); + assert_eq!(schema["title"], serde_json::json!("output_id")); + assert!(schema["description"].is_string(), "{}", name); + assert!( + !(binding.required_args)().iter().any(|a| a == "output_id"), + "{} requires output_id", + name + ); + // Appended after the function's own arguments, so their order is untouched. + let names: Vec = (binding.args)().into_iter().map(|(n, _)| n).collect(); + assert_eq!(names.last().unwrap(), "output_id"); + } + assert_eq!( + (binding_named("dummy_new").args)() + .into_iter() + .map(|(n, _)| n) + .collect::>(), + vec!["label", "output_id"] + ); + assert_eq!( + (binding_named("dummy_new").required_args)(), + vec!["label".to_string()] + ); + + // A value-returning binding is untouched, including the ones that take a handle. + for name in [ + "dummy_label", + "dummy_bump", + "num_objects", + "test_some_inputs", + ] { + let binding = binding_named(name); + assert!( + !(binding.args)().iter().any(|(n, _)| n == "output_id"), + "{} grew an output_id", + name + ); + } + assert_eq!((binding_named("num_objects").args)().len(), 1); + assert_eq!((binding_named("test_some_inputs").args)().len(), 5); + } + + #[test] + fn a_concurrent_insert_disturbs_neither_side() { + use std::sync::Arc; + let state = Arc::new(AppState::default()); + let other = Arc::clone(&state); + let writer = std::thread::spawn(move || { + for i in 0..64usize { + other.add( + format!("unrelated_{}", i), + RegistryItem::custom(DummyHandle { + label: format!("u{}", i), + hits: i, + }), + ); + } + }); + + let id = called_id( + "dummy_new", + serde_json::json!({ "label": "named", "output_id": "chosen" }), + &state, + ); + writer.join().unwrap(); + + assert_eq!(id, "chosen"); + assert_eq!(stored_handle_at(&state, "chosen").label, "named"); + let items = state.items.read().unwrap(); + for i in 0..64usize { + assert_eq!( + items[&format!("unrelated_{}", i)] + .as_custom::() + .unwrap() + .hits, + i + ); + } + assert_eq!(items.len(), 65); + } } diff --git a/process_mining/src/bindings/path_schema_bindings.rs b/process_mining/src/bindings/path_schema_bindings.rs index d06a2ea9..61020c6a 100644 --- a/process_mining/src/bindings/path_schema_bindings.rs +++ b/process_mining/src/bindings/path_schema_bindings.rs @@ -12,7 +12,7 @@ use crate::analysis::object_centric::path_schemas::{ schema_stats, Connection, PathConnectionParams, PathSchemaDiscovery, PathSchemaQuery, ResolvedPathSchema, SchemaStats, TypeEdge, TypeGraph, TypeRef, }; -use crate::core::event_data::object_centric::linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}; +use crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL; /// A node (event or object type) of the OCEL type graph, with its entity count. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] diff --git a/process_mining/src/bindings/slim_ocel_bindings.rs b/process_mining/src/bindings/slim_ocel_bindings.rs index a1fb03aa..9d928c0c 100644 --- a/process_mining/src/bindings/slim_ocel_bindings.rs +++ b/process_mining/src/bindings/slim_ocel_bindings.rs @@ -1,7 +1,10 @@ //! Binding wrappers for [`SlimLinkedOCEL`] functionality +use std::collections::HashMap; + use chrono::{DateTime, FixedOffset}; use macros_process_mining::register_binding; +use rayon::prelude::*; use crate::core::event_data::object_centric::{ linked_ocel::{ @@ -12,8 +15,6 @@ use crate::core::event_data::object_centric::{ }; use crate::core::OCEL; -// ── Creation ────────────────────────────────────────────────────────── - /// Create a new empty [`SlimLinkedOCEL`]. /// /// A [`SlimLinkedOCEL`] is an object-centric event log where events and objects are referenced @@ -25,8 +26,6 @@ fn locel_new() -> SlimLinkedOCEL { SlimLinkedOCEL::new() } -// ── Type Management ─────────────────────────────────────────────────── - /// Add an event type with the given ordered attribute declarations. /// /// No-op if the event type already exists. @@ -51,12 +50,10 @@ fn locel_add_object_type( ocel.add_object_type(&object_type, attributes); } -// ── Adding Events & Objects ─────────────────────────────────────────── - /// Add an event and return its [`EventIndex`]. /// -/// The event type must have been declared via [`locel_add_event_type`] first; -/// otherwise this returns `None`. +/// The event type must have been declared via [`locel_add_event_type`] first. +/// Otherwise this returns `None`. /// /// `id`: If `None`, a UUID is assigned. Returns `None` if the id is already taken. /// `attributes`: Positional values in the declared attribute order. Padded with `Null` or truncated on length mismatch (with a warning). @@ -75,8 +72,8 @@ fn locel_add_event( /// Add an object and return its [`ObjectIndex`]. /// -/// The object type must have been declared via [`locel_add_object_type`] first; -/// otherwise this returns `None`. +/// The object type must have been declared via [`locel_add_object_type`] first. +/// Otherwise this returns `None`. /// /// `id`: If `None`, a UUID is assigned. Returns `None` if the id is already taken. /// `attributes`: Positional list of time-indexed attribute histories (one `(timestamp, value)` list per declared attribute, in order). Use `1970-01-01T00:00:00Z` for constant/initial values. Padded with empty lists or truncated on length mismatch (with a warning). @@ -92,11 +89,9 @@ fn locel_add_object( ocel.add_object(&object_type, id, attributes, relationships) } -// ── Relationship Management ─────────────────────────────────────────── - /// Add an E2O (event-to-object) relationship with the given qualifier. /// -/// Multiple qualifiers between the same `(event, object)` pair are allowed; re-adding the exact +/// Multiple qualifiers between the same `(event, object)` pair are allowed. Re-adding the exact /// same `(event, object, qualifier)` triple is a no-op. Returns `true` on success, `false` if /// either index is out of bounds (with a stderr warning). #[register_binding] @@ -111,7 +106,7 @@ fn locel_add_e2o( /// Add a directed O2O (object-to-object) relationship from `from_obj` to `to_obj` with the given qualifier. /// -/// Multiple qualifiers between the same `(from_obj, to_obj)` pair are allowed; re-adding the exact +/// Multiple qualifiers between the same `(from_obj, to_obj)` pair are allowed. Re-adding the exact /// same `(from_obj, to_obj, qualifier)` triple is a no-op. Returns `true` on success, `false` if /// either index is out of bounds (with a stderr warning). #[register_binding] @@ -140,8 +135,6 @@ fn locel_delete_o2o(ocel: &mut SlimLinkedOCEL, from_obj: ObjectIndex, to_obj: Ob ocel.delete_o2o(&from_obj, &to_obj) } -// ── Read Access (LinkedOCELAccess) ──────────────────────────────────── - /// Get all declared event type names, in declaration order. #[register_binding] fn locel_get_ev_types(ocel: &SlimLinkedOCEL) -> Vec { @@ -244,6 +237,90 @@ fn get_obj_activity_trace(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec .collect() } +/// Merge `b` into `a` by summing values for matching keys. Used as the rayon reduce step. +fn merge_sum_maps(mut a: HashMap, b: HashMap) -> HashMap +where + K: std::hash::Hash + Eq, + V: Default + std::ops::AddAssign, +{ + for (k, v) in b { + *a.entry(k).or_default() += v; + } + a +} + +/// Get all activity-trace variants for objects of the given object type, with their occurrence counts +/// +/// Each entry is a tuple `(activity_trace, count)`, where `activity_trace` is the sequence of event types +/// connected to an object (ordered by event timestamp), and `count` is the number of objects of the +/// requested type that share that exact trace. +#[register_binding] +fn get_variants_of_object_type( + ocel: &SlimLinkedOCEL, + ob_type: String, +) -> Vec<(Vec, usize)> { + let obs: Vec = ocel.get_obs_of_type(&ob_type).copied().collect(); + let counts: HashMap, usize> = obs + .into_par_iter() + .fold(HashMap::new, |mut acc, ob| { + let trace: Vec = ob.get_obj_activity_trace_evtype_indices(ocel).collect(); + *acc.entry(trace).or_insert(0) += 1; + acc + }) + .reduce(HashMap::new, merge_sum_maps); + let ev_type_names: Vec<&str> = + ::get_ev_types(ocel).collect(); + counts + .into_iter() + .map(|(trace_idx, count)| { + let trace: Vec = trace_idx + .into_iter() + .map(|i| ev_type_names[i].to_string()) + .collect(); + (trace, count) + }) + .collect() +} + +/// Get the directly-follows graph (DFG) for objects of the given object type. +/// +/// Each entry is `((from_activity, to_activity), count)`, counting adjacent pairs in each +/// object's timestamp-ordered activity trace. Result order is unspecified. +#[register_binding] +fn get_dfg_of_object_type( + ocel: &SlimLinkedOCEL, + ob_type: String, +) -> Vec<((String, String), usize)> { + let obs: Vec = ocel.get_obs_of_type(&ob_type).copied().collect(); + let counts: HashMap<(usize, usize), usize> = obs + .into_par_iter() + .fold(HashMap::new, |mut acc, ob| { + let mut iter = ob.get_obj_activity_trace_evtype_indices(ocel); + if let Some(mut prev) = iter.next() { + for next in iter { + *acc.entry((prev, next)).or_insert(0) += 1; + prev = next; + } + } + acc + }) + .reduce(HashMap::new, merge_sum_maps); + let ev_type_names: Vec<&str> = + ::get_ev_types(ocel).collect(); + counts + .into_iter() + .map(|((from, to), count)| { + ( + ( + ev_type_names[from].to_string(), + ev_type_names[to].to_string(), + ), + count, + ) + }) + .collect() +} + /// Get the outgoing O2O relationships of an object as `(qualifier, object_index)` pairs. #[register_binding] fn locel_get_o2o(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec<(String, ObjectIndex)> { @@ -262,7 +339,7 @@ fn locel_get_o2o_rev(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec<(String, Obj /// Get the full [`OCELEvent`] (resolved type name, named attributes, string object IDs). /// -/// Allocates; prefer the specific `locel_get_ev_*` accessors for single fields. +/// Allocates. Prefer the specific `locel_get_ev_*` accessors for single fields. /// Panics if the index is out of bounds. #[register_binding] fn locel_get_full_ev(ocel: &SlimLinkedOCEL, ev: EventIndex) -> OCELEvent { @@ -271,7 +348,7 @@ fn locel_get_full_ev(ocel: &SlimLinkedOCEL, ev: EventIndex) -> OCELEvent { /// Get the full [`OCELObject`] (resolved type name, named time-indexed attributes, string object IDs). /// -/// Allocates; prefer the specific `locel_get_ob_*` accessors for single fields. +/// Allocates. Prefer the specific `locel_get_ob_*` accessors for single fields. /// Panics if the index is out of bounds. #[register_binding] fn locel_get_full_ob(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> OCELObject { @@ -368,3 +445,224 @@ fn get_event_timestamp_of_id(ocel: &SlimLinkedOCEL, ev_id: &String) -> Option Vec<(String, String, i64)> { + let num_events = ocel.get_num_evs() as u32; + let counts: HashMap<(usize, usize), i64> = (0..num_events) + .into_par_iter() + .fold(HashMap::new, |mut acc, i| { + let ev = EventIndex::from(i).get_ev(ocel); + for (_q, ob) in &ev.relationships { + let ot = ob.get_ob(ocel).object_type; + *acc.entry((ev.event_type, ot)).or_insert(0) += 1; + } + acc + }) + .reduce(HashMap::new, merge_sum_maps); + let ev_types: Vec<&str> = ::get_ev_types(ocel).collect(); + let ob_types: Vec<&str> = ::get_ob_types(ocel).collect(); + counts + .into_iter() + .map(|((e, o), c)| (ev_types[e].to_string(), ob_types[o].to_string(), c)) + .collect() +} + +/// Conversion rate from `source_type` to `target_type` via O2O, restricted to targets touched by `activity`. +/// +/// Returns the fraction of `source_type` objects that have at least one outgoing O2O edge to a +/// `target_type` object related (via E2O) to some event of the given event type. +/// +/// # Errors +/// Returns an error if `activity` names no declared event type, or if either object type is not +/// declared, so a rate of `0.0` cannot mean a misspelled name. +#[register_binding(stringify_error)] +fn locel_conversion_rate( + ocel: &SlimLinkedOCEL, + activity: String, + source_type: String, + target_type: String, +) -> Result { + if ocel.get_ev_type(&activity).is_none() { + return Err(format!("no event type '{activity}'")); + } + for ob_type in [&source_type, &target_type] { + if ocel.get_ob_type(ob_type).is_none() { + return Err(format!("no object type '{ob_type}'")); + } + } + let sources: Vec = ocel.get_obs_of_type(&source_type).copied().collect(); + let total = sources.len(); + if total == 0 { + return Ok(0.0); + } + let reached = sources + .par_iter() + .filter(|&&s| { + s.get_o2o(ocel).any(|&t| { + t.get_ob_type(ocel) == &target_type + && t.get_e2o_rev(ocel) + .any(|&e| e.get_ev_type(ocel) == &activity) + }) + }) + .count(); + Ok(reached as f64 / total as f64) +} + +/// Each object's reverse-E2O events in `(time, id)` order. +/// +/// Events are sorted per call, so this makes no assumption about global event ordering +fn sorted_events_per_object(ocel: &SlimLinkedOCEL) -> Vec> { + (0..ocel.get_num_obs() as u32) + .into_par_iter() + .map(|i| { + let mut evs: Vec = + ObjectIndex::from(i).get_e2o_rev(ocel).copied().collect(); + evs.sort_by(|a, b| { + a.get_time(ocel) + .cmp(b.get_time(ocel)) + .then_with(|| ocel.get_ev_id(a).cmp(ocel.get_ev_id(b))) + }); + evs + }) + .collect() +} + +/// The `(time, id)`-immediate predecessor of `e` on object `o`, using the +/// per-object sorted lists from [`sorted_events_per_object`]. +/// `None` if `e` is the first event on `o`. +/// +/// # Panics +/// If `e` is not in `o`'s own list, which means the forward and reverse E2O indices disagree. +#[inline] +fn df_predecessor( + sorted: &[Vec], + ocel: &SlimLinkedOCEL, + e: EventIndex, + o: ObjectIndex, +) -> Option { + let evs = &sorted[o.into_inner() as usize]; + let key = (e.get_time(ocel), ocel.get_ev_id(&e)); + let pos = evs + .binary_search_by(|x| (x.get_time(ocel), ocel.get_ev_id(x)).cmp(&key)) + .unwrap_or_else(|_| { + panic!( + "event '{}' is related to object '{}' but missing from its reverse-E2O list", + ocel.get_ev_id(&e), + ocel.get_ob_id(&o) + ) + }); + pos.checked_sub(1).map(|p| evs[p]) +} + +/// Keep only the `k` rows `cmp` ranks first, still ordered by `cmp`. +fn keep_top_k(rows: &mut Vec, k: usize, cmp: F) +where + F: Fn(&T, &T) -> std::cmp::Ordering, +{ + if k < rows.len() { + rows.select_nth_unstable_by(k, &cmp); + rows.truncate(k); + } + rows.sort_unstable_by(&cmp); +} + +/// Per-event synchronization time and the delaying object. +/// +/// For each event with at least one directly-follows predecessor, the synchronization time is +/// `max_predecessor_time - min_predecessor_time` in integer microseconds (the span between its +/// earliest and latest directly-preceding event). The delaying object is the object linking the +/// latest predecessor (ties broken by ascending object id). +/// Returns one row `(event_id, sync_us, delaying_object_id)` per qualifying event. +/// +/// `top_k`: if `Some(k)`, return only the `k` rows with the largest `sync_us`, ties broken by +/// ascending event id, sorted descending. `None` returns every qualifying event. +#[register_binding] +fn locel_oc_perf_sync_per_event( + ocel: &SlimLinkedOCEL, + #[bind(default)] top_k: Option, +) -> Vec<(String, i64, String)> { + let sorted = sorted_events_per_object(ocel); + let mut rows: Vec<(EventIndex, i64, ObjectIndex)> = (0..ocel.get_num_evs() as u32) + .into_par_iter() + .filter_map(|i| { + let e = EventIndex::from(i); + let mut min_us = i64::MAX; + // (latest predecessor time, its object) = the delaying edge. + let mut delaying: Option<(i64, ObjectIndex)> = None; + for &o in e.get_e2o(ocel) { + if let Some(p) = df_predecessor(&sorted, ocel, e, o) { + let t = p.get_time(ocel).timestamp_micros(); + min_us = min_us.min(t); + let keep = match delaying { + Some((bt, bo)) => { + bt > t || (bt == t && ocel.get_ob_id(&bo) <= ocel.get_ob_id(&o)) + } + None => false, + }; + if !keep { + delaying = Some((t, o)); + } + } + } + delaying.map(|(max_us, o)| (e, max_us - min_us, o)) + }) + .collect(); + if let Some(k) = top_k { + keep_top_k(&mut rows, k, |a, b| { + b.1.cmp(&a.1) + .then_with(|| ocel.get_ev_id(&a.0).cmp(ocel.get_ev_id(&b.0))) + }); + } + rows.into_iter() + .map(|(e, max_minus_min, o)| { + ( + ocel.get_ev_id(&e).to_string(), + max_minus_min, + ocel.get_ob_id(&o).to_string(), + ) + }) + .collect() +} + +/// Per-event sojourn time. +/// +/// For each event with at least one directly-follows predecessor, the sojourn time is +/// `event_time - latest_predecessor_time` in integer microseconds. Returns one row +/// `(event_id, sojourn_us)` per qualifying event. +/// +/// `top_k`: if `Some(k)`, return only the `k` rows with the largest `sojourn_us`, ties broken by +/// ascending event id, sorted descending. `None` returns every qualifying event. +#[register_binding] +fn locel_oc_perf_sojourn_per_event( + ocel: &SlimLinkedOCEL, + #[bind(default)] top_k: Option, +) -> Vec<(String, i64)> { + let sorted = sorted_events_per_object(ocel); + let mut rows: Vec<(EventIndex, i64)> = (0..ocel.get_num_evs() as u32) + .into_par_iter() + .filter_map(|i| { + let e = EventIndex::from(i); + let latest = e + .get_e2o(ocel) + .filter_map(|&o| df_predecessor(&sorted, ocel, e, o)) + .map(|p| p.get_time(ocel).timestamp_micros()) + .max()?; + Some((e, e.get_time(ocel).timestamp_micros() - latest)) + }) + .collect(); + if let Some(k) = top_k { + keep_top_k(&mut rows, k, |a, b| { + b.1.cmp(&a.1) + .then_with(|| ocel.get_ev_id(&a.0).cmp(ocel.get_ev_id(&b.0))) + }); + } + rows.into_iter() + .map(|(e, sojourn_us)| (ocel.get_ev_id(&e).to_string(), sojourn_us)) + .collect() +} diff --git a/process_mining/src/conformance/object_centric/oc_declare.rs b/process_mining/src/conformance/object_centric/oc_declare.rs index 740bcf3f..0165e567 100644 --- a/process_mining/src/conformance/object_centric/oc_declare.rs +++ b/process_mining/src/conformance/object_centric/oc_declare.rs @@ -50,7 +50,10 @@ pub(crate) fn target_events_for_binding<'a>( SetFilter::Any(items) => Box::new(items.iter().flat_map(move |o| for_ob(**o))), SetFilter::All(items) => { if items.is_empty() { - Box::new(Vec::new().into_iter()) + // `SetFilter::check` is vacuously true for an empty `All`, i.e. it imposes + // no constraint, so every event of the type is a candidate, exactly as when + // `objs` itself is empty. + Box::new(EventOrSynthetic::get_all_syn_evs(linked_ocel, etype).into_iter()) } else { Box::new(for_ob(*items[0]).filter(|e| { items @@ -81,11 +84,24 @@ fn directly_adjacent_event<'a>( // reference_event: &'a OCELEvent, following: bool, ) -> Option { + let in_direction = move |e: &EventOrSynthetic| { + let e_time = e.get_timestamp(linked_ocel); + if following { + e_time > *reference_time + } else { + e_time < *reference_time + } + }; let initial: Box> = if objs.is_empty() { // If no requirements are specified, consider all events // TODO: Maybe also consider synthetic events here? // But in general, this is not very relevant as there are usually some object requirements - Box::new(linked_ocel.get_all_evs().map(EventOrSynthetic::Event)) + Box::new( + linked_ocel + .get_all_evs() + .map(EventOrSynthetic::Event) + .filter(in_direction), + ) } else { match &objs[0] { SetFilter::Any(items) => Box::new(items.iter().flat_map(|o| { @@ -102,7 +118,14 @@ fn directly_adjacent_event<'a>( })), SetFilter::All(items) => { if items.is_empty() { - Box::new(Vec::new().into_iter()) + // `SetFilter::check` is vacuously true for an empty `All`: no constraint, + // so every event is a candidate, as when `objs` itself is empty. + Box::new( + linked_ocel + .get_all_evs() + .map(EventOrSynthetic::Event) + .filter(in_direction), + ) } else { Box::new( EventOrSynthetic::get_all_for_ob(linked_ocel, *items[0]) @@ -474,4 +497,21 @@ mod tests { "expected at least 500 comparison instead of just {compared}; something is wrong with the input SlimLinked OCEL." ); } + + // `SetFilter::check` is vacuously true for `All(&[])`, since an event trivially references + // "all of" an empty set, so the candidate iterator must not special-case it to empty. + #[test] + fn all_of_empty_items_matches_every_event_of_the_type_like_no_objs_at_all() { + let locel = sample_locel(); + let no_objs: Vec> = Vec::new(); + let empty_all: Vec> = vec![SetFilter::All(Vec::new())]; + + let via_no_objs: FxHashSet<_> = + target_events_for_binding(&no_objs, &locel, "place", None).collect(); + let via_empty_all: FxHashSet<_> = + target_events_for_binding(&empty_all, &locel, "place", None).collect(); + + assert_eq!(via_empty_all, via_no_objs); + assert_eq!(via_empty_all.len(), locel.get_evs_of_type("place").count()); + } } diff --git a/process_mining/src/core/event_data/case_centric/xes/stream_xes.rs b/process_mining/src/core/event_data/case_centric/xes/stream_xes.rs index f27b073c..363f0af2 100644 --- a/process_mining/src/core/event_data/case_centric/xes/stream_xes.rs +++ b/process_mining/src/core/event_data/case_centric/xes/stream_xes.rs @@ -368,7 +368,7 @@ impl StreamingXESParser<'_> { } }, quick_xml::events::Event::End(t) => { - match t.as_ref() { + match t.name().as_ref() { b"event" => self.current_mode = Mode::Trace, b"trace" => { self.current_mode = Mode::Log; diff --git a/process_mining/src/core/event_data/object_centric/appendable.rs b/process_mining/src/core/event_data/object_centric/appendable.rs index 8c54e11e..6f5ec182 100644 --- a/process_mining/src/core/event_data/object_centric/appendable.rs +++ b/process_mining/src/core/event_data/object_centric/appendable.rs @@ -1,12 +1,20 @@ //! Appendable OCEL trait use std::convert::Infallible; +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::Path; use chrono::{DateTime, FixedOffset}; +use crate::core::event_data::object_centric::io::OCELIOError; +use crate::core::event_data::object_centric::ocel_json::import_ocel_json_into; use crate::core::event_data::object_centric::ocel_struct::{ OCELEvent, OCELEventAttribute, OCELObject, OCELObjectAttribute, OCELRelationship, OCELType, OCEL, }; +use crate::core::event_data::object_centric::ocel_xml::xml_ocel_import::import_ocel_xml_into; +use crate::core::event_data::object_centric::ocel_xml::OCELImportOptions; +use crate::core::io::infer_format_from_path; /// Appendable trait for OCEL data. /// @@ -112,3 +120,98 @@ impl AppendableOCEL for OCEL { Ok(()) } } + +/// Streaming counterpart to [`Importable`](crate::Importable): +/// Import an OCEL from a reader or path straight into an [`AppendableOCEL`] sink. +pub trait StreamImportOCEL: AppendableOCEL + Sized { + /// Stream an OCEL from `reader` in the given `format` into this sink. + /// + /// [`AppendableOCEL::finalize`] runs once the stream ends. Calling it again is harmless. + fn stream_ocel_from_reader( + &mut self, + reader: R, + format: &str, + options: OCELImportOptions, + ) -> Result<(), OCELIOError> + where + Self::Error: Into, + { + if let Some(inner) = format.strip_suffix(".gz") { + // Erase the reader type for recursion + let gz: Box = Box::new(flate2::read::GzDecoder::new(BufReader::new(reader))); + return self.stream_ocel_from_reader(gz, inner, options); + } + if format.ends_with("json") || format.ends_with("jsonocel") { + import_ocel_json_into(BufReader::new(reader), self)?; + } else if format.ends_with("xml") || format.ends_with("xmlocel") { + let mut xml = quick_xml::Reader::from_reader(BufReader::new(reader)); + import_ocel_xml_into(&mut xml, self, options)?; + } else { + return Err(OCELIOError::UnsupportedFormat(format!( + "no streaming OCEL importer for format {format:?}" + ))); + } + self.finalize().map_err(Into::into) + } + + /// Infer the format from `path` and stream the file into this sink. + fn stream_ocel_from_path>( + &mut self, + path: P, + options: OCELImportOptions, + ) -> Result<(), OCELIOError> + where + Self::Error: Into, + { + let path = path.as_ref(); + let format = infer_format_from_path(path).ok_or_else(|| { + OCELIOError::UnsupportedFormat(format!("cannot infer OCEL format from {path:?}")) + })?; + self.stream_ocel_from_reader(File::open(path)?, &format, options) + } +} + +impl StreamImportOCEL for A {} + +/// Whether streaming a given format directly is supported. +pub fn is_streaming_format(format: &str) -> bool { + let base = format.strip_suffix(".gz").unwrap_or(format); + base.ends_with("json") + || base.ends_with("jsonocel") + || base.ends_with("xml") + || base.ends_with("xmlocel") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::io::Importable; + + #[test] + fn is_streaming_format_agrees_with_the_dispatch() { + for f in OCEL::known_import_formats() { + let mut ocel = OCEL { + event_types: Vec::new(), + object_types: Vec::new(), + events: Vec::new(), + objects: Vec::new(), + }; + let err = ocel + .stream_ocel_from_reader( + std::io::empty(), + &f.extension, + OCELImportOptions::default(), + ) + .err(); + // Empty input fails in whatever way the format's parser fails. Only the + // "no streaming importer" rejection means the dispatch picked no arm. + let dispatched = !matches!(err, Some(OCELIOError::UnsupportedFormat(_))); + assert_eq!( + dispatched, + is_streaming_format(&f.extension), + "format {:?}", + f.extension + ); + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/blueprint.rs b/process_mining/src/core/event_data/object_centric/extraction/blueprint.rs new file mode 100644 index 00000000..a931769f --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/blueprint.rs @@ -0,0 +1,424 @@ +//! The blueprint itself: a node graph producing rows, and mappings turning rows into entities. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::expr::{AttributeMapping, SplitSpec, TimestampSource, ValueExpression}; +use super::predicate::Predicate; +use super::MODEL_VERSION; + +/// How entity ids are rendered, for events and objects alike. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum IdRendering { + /// Use the id expression's value verbatim. + #[default] + Raw, + /// Prefix the id with its type name, so ids from different types cannot collide. + /// + /// Under this setting every relation endpoint must declare its type, since otherwise the + /// prefixed id cannot be rebuilt. Validation enforces that. + TypePrefixed, +} + +/// What to do when a relation names an entity that does not exist. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum MissingEndpointPolicy { + /// Skip the relation. + #[default] + Drop, + /// Create the object. Requires the endpoint to declare its type. + Create, + /// Record an error. + Error, +} + +/// What to do when an object id is produced more than once. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum DuplicateObjectPolicy { + /// Keep the first, and count the rest as deduplicated rather than lost. + #[default] + FirstWins, + /// Record an error. + Error, +} + +/// One operation in the row graph. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum NodeOp { + /// Read a table from a source. + Source { + /// Source id, resolved to a connection at execution time. + source_id: String, + /// Table name. + table: String, + }, + /// Keep the rows of `input` satisfying `condition`. + Filter { + /// Input node id. + input: String, + /// The condition. + condition: Predicate, + }, + /// Inner-join two nodes on the given column pairs. + Join { + /// Left input node id. + left: String, + /// Right input node id. + right: String, + /// Column pairs, as `(left column, right column)`. + on: Vec<(String, String)>, + }, + /// Concatenate the rows of several nodes, aligning columns by name. + /// + /// `UNION ALL`, not `UNION`: dropping duplicates would drop the entities they produce. Output + /// columns are the union of the inputs' column names, and an input lacking one contributes + /// `Null`. Both are model semantics a compiler must reproduce. + Union { + /// Input node ids. + inputs: Vec, + }, +} + +/// A node in the row graph. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Node { + /// Unique id, referenced by other nodes and by mappings. + pub id: String, + /// Display label. No semantic role. + pub label: Option, + /// The operation. + pub op: NodeOp, +} + +/// A reference to an object, used at every position where one is named. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObjectEndpoint { + /// The object's id. + pub id: ValueExpression, + /// The object's type. Required under [`IdRendering::TypePrefixed`] and under + /// [`MissingEndpointPolicy::Create`]. + pub object_type: Option, + /// Split the id cell into several ids, producing one relation per part. + pub split: Option, +} + +/// A reference to an event. Mirrors [`ObjectEndpoint`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct EventEndpoint { + /// The event's id. + pub id: ValueExpression, + /// The event's type. Required under [`IdRendering::TypePrefixed`]. + pub event_type: Option, +} + +/// An object related to an event declared by the same mapping. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct InlineObjectRef { + /// The object. + pub object: ObjectEndpoint, + /// Relation qualifier. + pub qualifier: Option, +} + +/// What a mapping produces from a row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum Target { + /// An event. + Event { + /// Event type. + event_type: ValueExpression, + /// Event id. `None` assigns a UUID, which is not reproducible across runs and cannot + /// be compiled to a view. + /// + /// It is also what coalesces a fan-out: over a join of orders and their items, a `None` + /// id makes one event per item, an id naming the order one event per order, still related + /// to every item. The repeated rows count as + /// [`MappingStats::deduplicated`](super::report::MappingStats::deduplicated) while + /// `objects` below is emitted for each. + id: Option, + /// When it happened. + timestamp: TimestampSource, + /// Event attributes. + #[serde(default)] + attributes: Vec, + /// Objects related to this event. + #[serde(default)] + objects: Vec, + }, + /// An object. + Object { + /// Object type. + object_type: ValueExpression, + /// Object id. + id: ValueExpression, + /// When the attribute values below were observed. `None` records them as static + /// values stamped at the Unix epoch. + #[serde(default)] + timestamp: Option, + /// Object attributes. + #[serde(default)] + attributes: Vec, + }, + /// An event-to-object relation. + #[serde(rename = "e2o")] + E2O { + /// The event. + event: EventEndpoint, + /// The object. + object: ObjectEndpoint, + /// Relation qualifier. + qualifier: Option, + }, + /// An object-to-object relation. + #[serde(rename = "o2o")] + O2O { + /// The source object. + source: ObjectEndpoint, + /// The target object. + target: ObjectEndpoint, + /// Relation qualifier. + qualifier: Option, + }, +} + +/// One mapping from a node's rows to entities. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Mapping { + /// The node whose rows this reads. + pub node: String, + /// Display label, also used to name this mapping in diagnostics. + pub label: Option, + /// Only rows satisfying this produce anything. `None` accepts every row. + pub when: Option, + /// What to produce. + pub target: Target, +} + +/// A mapping, or an ordered group of them. +/// +/// `Single` is not boxed: it is the overwhelmingly common case, and callers construct and match +/// it directly. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +#[allow(clippy::large_enum_variant)] +pub enum MappingEntry { + /// One independent mapping. + Single(Mapping), + /// Mappings tried in order, where the first match wins. + /// + /// Surface sugar: desugaring rewrites each guard to exclude the earlier ones, so nothing + /// downstream of validation sees this variant. + Ordered { + /// Mappings, in priority order. + mappings: Vec, + }, +} + +/// A declarative mapping from relational rows to an OCEL. +/// +/// Carries no connection details and no schema snapshot: both are supplied by the caller, which +/// keeps a blueprint portable, shareable and free of secrets. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Blueprint { + /// Schema version. Checked against [`super::MODEL_VERSION`] during validation. + pub version: u32, + /// How entity ids are rendered. + #[serde(default)] + pub id_rendering: IdRendering, + /// The row graph. + pub nodes: Vec, + /// The mappings. + pub mappings: Vec, + /// What to do about relations naming a missing entity. + #[serde(default)] + pub on_missing_endpoint: MissingEndpointPolicy, + /// What to do about a repeated object id. + #[serde(default)] + pub on_duplicate_object: DuplicateObjectPolicy, +} + +impl Blueprint { + /// The node with this id, if any. + #[must_use] + pub fn node(&self, id: &str) -> Option<&Node> { + self.nodes.iter().find(|n| n.id == id) + } + + /// Parse a blueprint from JSON, checking its declared `version` before attempting to parse + /// the body. + /// + /// Checking `version` first is what turns a document from a newer build into an "unsupported + /// version" message rather than serde's "unknown variant" naming a construct the caller can do + /// nothing about. An unknown variant within a supported version still fails. + /// + /// # Errors + /// Returns [`BlueprintParseError::UnsupportedVersion`] when `version` exceeds + /// [`MODEL_VERSION`], or [`BlueprintParseError::Malformed`] when `input` is not valid JSON, + /// has no `version` field, or otherwise does not parse as a `Blueprint`. + pub fn from_json(input: &str) -> Result { + // Probed off the parsed tree rather than by a second `from_str`, so the common case does + // not pay a full extra parse. + let document: serde_json::Value = + serde_json::from_str(input).map_err(BlueprintParseError::Malformed)?; + if let Some(found) = document.get("version").and_then(serde_json::Value::as_u64) { + if found > u64::from(MODEL_VERSION) { + return Err(BlueprintParseError::UnsupportedVersion { + found: u32::try_from(found).unwrap_or(u32::MAX), + supported: MODEL_VERSION, + }); + } + } + serde_json::from_value(document).map_err(BlueprintParseError::Malformed) + } +} + +/// Why [`Blueprint::from_json`] failed. +#[derive(Debug)] +pub enum BlueprintParseError { + /// The document's `version` exceeds what this build supports. Returned before the body is + /// parsed, so an unknown construct in the body never masks this as a generic serde error. + UnsupportedVersion { + /// The document's `version`. + found: u32, + /// The newest version this build reads, [`MODEL_VERSION`]. + supported: u32, + }, + /// `input` was not valid JSON, had no `version` field, or otherwise did not parse as a + /// `Blueprint`. An unknown variant within a supported version lands here too: that is a parse + /// failure, not something the version check is meant to catch. + Malformed(serde_json::Error), +} + +impl std::fmt::Display for BlueprintParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BlueprintParseError::UnsupportedVersion { found, supported } => write!( + f, + "blueprint version {found} is newer than the supported version {supported}" + ), + BlueprintParseError::Malformed(e) => write!(f, "malformed blueprint: {e}"), + } + } +} + +impl std::error::Error for BlueprintParseError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + BlueprintParseError::UnsupportedVersion { .. } => None, + BlueprintParseError::Malformed(e) => Some(e), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ACCOUNT_MOVE: &str = r#"{ + "version": 1, + "id_rendering": "type-prefixed", + "on_missing_endpoint": "create", + "on_duplicate_object": "first-wins", + "nodes": [ + { "id": "account_move", + "op": { "type": "source", "source_id": "odoo", "table": "account_move" } } + ], + "mappings": [ + { "type": "single", "node": "account_move", + "when": { "type": "compare", + "left": { "type": "column", "column": "move_type" }, + "op": "eq", + "right": { "type": "literal", "value": "out_invoice" } }, + "target": { "type": "object", + "object_type": { "type": "constant", "value": "customer_invoice" }, + "id": { "type": "column", "column": "id" }, + "attributes": [] } } + ] + }"#; + + #[test] + fn parses_the_discriminated_table_example_from_the_spec() { + let bp: Blueprint = serde_json::from_str(ACCOUNT_MOVE).expect("parse"); + assert_eq!(bp.version, 1); + assert_eq!(bp.id_rendering, IdRendering::TypePrefixed); + assert_eq!(bp.nodes.len(), 1); + assert!(bp.node("account_move").is_some()); + assert!(bp.node("nope").is_none()); + } + + #[test] + fn round_trips_through_json_unchanged() { + let bp: Blueprint = serde_json::from_str(ACCOUNT_MOVE).expect("parse"); + let again: Blueprint = + serde_json::from_str(&serde_json::to_string(&bp).expect("serialize")).expect("reparse"); + assert_eq!(bp, again); + } + + #[test] + fn an_unknown_field_is_ignored_rather_than_rejected() { + // Forward compatibility: an older build must degrade, not refuse. + let json = ACCOUNT_MOVE.replace(r#""version": 1,"#, r#""version": 1, "future_field": 42,"#); + assert!(serde_json::from_str::(&json).is_ok()); + } + + #[test] + fn from_json_parses_a_valid_v1_document() { + let bp = Blueprint::from_json(ACCOUNT_MOVE).expect("parse"); + assert_eq!(bp.version, 1); + assert!(bp.node("account_move").is_some()); + } + + #[test] + fn from_json_reports_an_unsupported_version_before_touching_the_body() { + // The body uses a node op this build does not implement at all: plain + // serde_json::from_str:: would fail with "unknown variant", masking the + // actionable "unsupported version" message from_json exists to give instead. + let json = r#"{ + "version": 999, + "nodes": [ + { "id": "a", "op": { "type": "a-future-node-op-this-build-does-not-know" } } + ], + "mappings": [] + }"#; + let err = Blueprint::from_json(json).expect_err("must reject a future version"); + assert!( + matches!( + err, + BlueprintParseError::UnsupportedVersion { + found: 999, + supported: 1 + } + ), + "got {err:?}" + ); + } + + #[test] + fn from_json_still_reports_an_unknown_variant_within_a_supported_version() { + // A supported-version document using a construct this build does not implement must + // still fail loudly, naming the offending variant. This build cannot execute what it + // does not implement, and serde's message is the right diagnostic. + let json = r#"{ + "version": 1, + "nodes": [ + { "id": "a", "op": { "type": "a-future-node-op-this-build-does-not-know" } } + ], + "mappings": [] + }"#; + let err = Blueprint::from_json(json).expect_err("must reject an unknown variant"); + let BlueprintParseError::Malformed(inner) = &err else { + panic!("expected Malformed, got {err:?}"); + }; + assert!( + inner + .to_string() + .contains("a-future-node-op-this-build-does-not-know"), + "error should name the unknown variant: {inner}" + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/case_centric.rs b/process_mining/src/core/event_data/object_centric/extraction/case_centric.rs new file mode 100644 index 00000000..bdd8be78 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/case_centric.rs @@ -0,0 +1,696 @@ +//! Getting case-centric data into the object-centric world: a blueprint for the traditional +//! case/activity/timestamp table, and a writer for an already-parsed [`EventLog`]. + +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, FixedOffset}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::blueprint::{ + Blueprint, DuplicateObjectPolicy, IdRendering, InlineObjectRef, Mapping, MappingEntry, + MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Target, +}; +use super::expr::{AttributeMapping, TimestampSource, ValueExpression}; +use super::sink::{ExtractionSink, SinkError}; +use super::slim_sink::SlimOcelSink; +use crate::core::event_data::case_centric::constants::{ACTIVITY_NAME, TRACE_ID_NAME}; +use crate::core::event_data::case_centric::{ + Attribute, AttributeValue, Event, EventLog, Trace, XESEditableAttribute, +}; +use crate::core::event_data::object_centric::linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}; +use crate::core::event_data::object_centric::{ + OCELAttributeType, OCELAttributeValue, OCELTypeAttribute, OCEL, +}; + +/// A flat table with one row per event, the usual shape of a case-centric log. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct FlatEventTable { + /// Source id, resolved to a connection at execution time. + pub source_id: String, + /// Table name. + pub table: String, + /// Column identifying the case. + pub case_id: String, + /// Column naming the activity, which becomes the event type. + pub activity: String, + /// Column holding the event's timestamp. + pub timestamp: String, + /// Object type to give the case, conventionally `Case`. + pub case_object_type: String, + /// Case-level columns. Recorded as static attributes, never change-tracked. + pub case_attributes: Vec, + /// Event-level columns. + pub event_attributes: Vec, +} + +impl Blueprint { + /// Build a blueprint for a flat event table. + /// + /// One node and one event mapping, whose inline object reference creates the case objects -- + /// hence `on_missing_endpoint: Create`. A second mapping appears only when there are case + /// attributes, and is static (`timestamp: None`), so a case-level column repeated across the + /// case's events does not store one identical timed value per event. + #[must_use] + pub fn from_flat_event_table(spec: FlatEventTable) -> Blueprint { + let node_id = "events".to_string(); + let case_endpoint = || ObjectEndpoint { + id: ValueExpression::Column { + column: spec.case_id.clone(), + }, + object_type: Some(ValueExpression::Constant { + value: spec.case_object_type.clone(), + }), + split: None, + }; + + let mut mappings = vec![MappingEntry::Single(Mapping { + node: node_id.clone(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: ValueExpression::Column { + column: spec.activity.clone(), + }, + id: None, + timestamp: TimestampSource::column(spec.timestamp.clone()), + attributes: spec.event_attributes.clone(), + objects: vec![InlineObjectRef { + object: case_endpoint(), + qualifier: Some(ValueExpression::Constant { + value: "case".into(), + }), + }], + }, + })]; + + if !spec.case_attributes.is_empty() { + mappings.push(MappingEntry::Single(Mapping { + node: node_id.clone(), + label: Some("cases".into()), + when: None, + target: Target::Object { + object_type: ValueExpression::Constant { + value: spec.case_object_type.clone(), + }, + id: ValueExpression::Column { + column: spec.case_id.clone(), + }, + timestamp: None, + attributes: spec.case_attributes.clone(), + }, + })); + } + + Blueprint { + version: super::MODEL_VERSION, + id_rendering: IdRendering::Raw, + nodes: vec![Node { + id: node_id, + label: Some(spec.table.clone()), + op: NodeOp::Source { + source_id: spec.source_id, + table: spec.table, + }, + }], + mappings, + on_missing_endpoint: MissingEndpointPolicy::Create, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } + } +} + +/// The object type given to the one object each trace becomes. +pub const CASE_OBJECT_TYPE: &str = "Case"; + +/// The qualifier on every event-to-case relation [`write_event_log_to_sink`] writes. +pub const CASE_QUALIFIER: &str = "case"; + +/// The event type given to an event whose [`ACTIVITY_NAME`] attribute is missing or not a string. +const UNKNOWN_ACTIVITY: &str = "UNKNOWN"; + +/// The XES key an event's timestamp is read from. +const TIMESTAMP_NAME: &str = "time:timestamp"; + +/// The one OCEL value a XES value maps to. +/// +/// OCEL attribute values do not nest, so a list or container keeps its debug rendering rather +/// than being dropped. Lossy, but not silently empty. +fn xes_attribute_to_ocel(value: &AttributeValue) -> OCELAttributeValue { + match value { + AttributeValue::String(s) => OCELAttributeValue::String(s.clone()), + AttributeValue::Date(t) => OCELAttributeValue::Time(*t), + AttributeValue::Int(i) => OCELAttributeValue::Integer(*i), + AttributeValue::Float(f) => OCELAttributeValue::Float(*f), + AttributeValue::Boolean(b) => OCELAttributeValue::Boolean(*b), + AttributeValue::ID(uuid) => OCELAttributeValue::String(uuid.to_string()), + AttributeValue::List(attrs) => OCELAttributeValue::String(format!("{attrs:?}")), + AttributeValue::Container(attrs) => OCELAttributeValue::String(format!("{attrs:?}")), + AttributeValue::None() => OCELAttributeValue::Null, + } +} + +fn activity_of(event: &Event) -> &str { + event + .attributes + .get_by_key(ACTIVITY_NAME) + .and_then(|a| a.value.try_as_string()) + .map_or(UNKNOWN_ACTIVITY, String::as_str) +} + +/// An event's instant, or `None` when [`TIMESTAMP_NAME`] is absent or holds something other +/// than a date. +fn timestamp_of(event: &Event) -> Option> { + event + .attributes + .get_by_key(TIMESTAMP_NAME) + .and_then(|a| a.value.try_as_date()) + .copied() +} + +/// The attribute names one type carries, in first-seen order, each typed to cover every value +/// seen under that name. This is the schema a sink needs before any entity of the type. +/// +/// Widened rather than pinned to the first value seen, which would type an attribute from a row +/// that happens to hold nothing, or an integer, where later rows hold text. +#[derive(Debug, Default)] +struct TypeAttributes<'a> { + order: Vec<&'a str>, + types: HashMap<&'a str, OCELAttributeType>, +} + +impl<'a> TypeAttributes<'a> { + fn observe(&mut self, attribute: &'a Attribute) { + let observed = xes_attribute_to_ocel(&attribute.value).get_type(); + match self.types.entry(attribute.key.as_str()) { + Entry::Occupied(mut e) => { + let widened = e.get().coalesce(observed); + e.insert(widened); + } + Entry::Vacant(e) => { + self.order.push(attribute.key.as_str()); + e.insert(observed); + } + } + } + + /// The declaration, valid only once every value has been observed. + fn declared(&self) -> Vec { + self.order + .iter() + .map(|name| OCELTypeAttribute::new(name, &self.types[name])) + .collect() + } +} + +/// A case id no earlier trace has taken. +/// +/// Two traces sharing a `concept:name` are still two cases, so the repeat is disambiguated +/// rather than merged: an OCEL holding two objects under one id is not a well-formed OCEL, and +/// a sink rejects the second outright. +fn unique_case_id(trace: &Trace, trace_index: usize, used: &mut HashSet) -> String { + let base = trace + .attributes + .get_by_key(TRACE_ID_NAME) + .and_then(|a| a.value.try_as_string()) + .cloned() + .unwrap_or_else(|| format!("ob_{trace_index}")); + if used.insert(base.clone()) { + return base; + } + let mut n = 2; + loop { + let candidate = format!("{base}~{n}"); + if used.insert(candidate.clone()) { + return candidate; + } + n += 1; + } +} + +/// What [`write_event_log_to_sink`] wrote, and what it could not represent faithfully. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EventLogWriteReport { + /// Events handed to the sink. + pub events_written: u64, + /// How many of those had no usable `time:timestamp` (absent, or not a date) and were written + /// at the Unix epoch. + /// + /// Counted here because an epoch timestamp is indistinguishable from a real one in the log + /// itself, unlike the `UNKNOWN` type a nameless event takes. + pub events_without_timestamp: u64, +} + +/// Write a case-centric log into an object-centric sink: one object per case. +/// +/// Each trace becomes one object of type [`CASE_OBJECT_TYPE`] carrying the trace's attributes, +/// timed at the epoch because a case-level attribute has no instant of its own. Each of its +/// events becomes an event of the type its [`ACTIVITY_NAME`] names, carrying the event's own +/// attributes, related back to its case under [`CASE_QUALIFIER`]. Events are numbered `ev_` +/// across the whole log. A case takes its [`TRACE_ID_NAME`], or `ob_` when it has +/// none. +/// +/// The log is read twice: once for each type's attribute schema, which a sink must be told before +/// it is given anything of that type, and once to write. Only the schema is held between. +/// +/// Does not call [`ExtractionSink::finalize`], so several logs can go into one sink. A +/// file-backed sink is not readable until finalized. [`event_log_to_slim_ocel`] and +/// [`event_log_to_ocel`] do both. +/// +/// # Errors +/// +/// Returns whatever the sink returns. +pub fn write_event_log_to_sink( + log: &EventLog, + sink: &mut S, +) -> Result { + let mut case_type = TypeAttributes::default(); + let mut event_type_names: Vec<&str> = Vec::new(); + let mut event_types: Vec> = Vec::new(); + let mut event_type_index: HashMap<&str, usize> = HashMap::new(); + + for trace in &log.traces { + for attribute in &trace.attributes { + case_type.observe(attribute); + } + for event in &trace.events { + let activity = activity_of(event); + let index = *event_type_index.entry(activity).or_insert_with(|| { + event_type_names.push(activity); + event_types.push(TypeAttributes::default()); + event_types.len() - 1 + }); + for attribute in &event.attributes { + event_types[index].observe(attribute); + } + } + } + + sink.declare_object_type(CASE_OBJECT_TYPE, &case_type.declared())?; + for (name, attributes) in event_type_names.iter().zip(&event_types) { + sink.declare_event_type(name, &attributes.declared())?; + } + + let epoch: DateTime = DateTime::UNIX_EPOCH.into(); + let mut used_case_ids: HashSet = HashSet::with_capacity(log.traces.len()); + let mut report = EventLogWriteReport::default(); + + for (trace_index, trace) in log.traces.iter().enumerate() { + let case_id = unique_case_id(trace, trace_index, &mut used_case_ids); + let attributes: Vec<_> = trace + .attributes + .iter() + .map(|a| (a.key.clone(), epoch, xes_attribute_to_ocel(&a.value))) + .collect(); + let case = sink.add_object(CASE_OBJECT_TYPE, &case_id, &attributes)?; + + for event in &trace.events { + let attributes: Vec<_> = event + .attributes + .iter() + .map(|a| (a.key.clone(), xes_attribute_to_ocel(&a.value))) + .collect(); + let id = format!("ev_{}", report.events_written); + report.events_written += 1; + let time = match timestamp_of(event) { + Some(t) => t, + None => { + report.events_without_timestamp += 1; + epoch + } + }; + let written = sink.add_event(activity_of(event), time, &id, &attributes)?; + // `add_e2o`'s contract: exactly one `resolve_object` for this row's own object, + // immediately before it. A deferring sink has no id index and needs the adjacency to + // link the ask to the relation that made it. + let related = sink + .resolve_object(&case_id, Some(CASE_OBJECT_TYPE)) + .into_ref() + .unwrap_or_else(|| case.clone()); + sink.add_e2o(&written, &related, CASE_QUALIFIER)?; + } + } + + Ok(report) +} + +/// [`write_event_log_to_sink`] into an in-memory [`SlimOcelSink`], finalized. +/// +/// Discards the [`EventLogWriteReport`]. A caller that needs it drives the sink itself. +/// +/// # Errors +/// +/// Returns whatever the sink returns. +pub fn event_log_to_slim_ocel(log: &EventLog) -> Result { + let mut sink = SlimOcelSink::new(); + write_event_log_to_sink(log, &mut sink)?; + sink.finalize()?; + Ok(sink.into_ocel()) +} + +/// [`event_log_to_slim_ocel`], materialised as a plain [`OCEL`]. +/// +/// # Errors +/// +/// Returns whatever the sink returns. +pub fn event_log_to_ocel(log: &EventLog) -> Result { + Ok(event_log_to_slim_ocel(log)?.construct_ocel()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::blueprint::*; + use crate::core::event_data::object_centric::extraction::catalog::{ + ExtractionCatalog, TableSchema, + }; + use crate::core::event_data::object_centric::extraction::validate::validate; + + #[test] + fn flat_event_table_crosses_a_bindings_boundary() { + // FlatEventTable is a public input to Blueprint::from_flat_event_table, so a caller on + // the other side of a bindings boundary needs to be able to build and send one. + let spec = FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![], + event_attributes: vec![], + }; + let json = serde_json::to_string(&spec).expect("serialize"); + let back: FlatEventTable = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, spec); + let schema = schemars::schema_for!(FlatEventTable); + assert!(serde_json::to_value(&schema).is_ok()); + } + + #[test] + fn builds_a_valid_single_mapping_blueprint() { + let bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![], + event_attributes: vec![], + }); + + assert_eq!(bp.nodes.len(), 1); + assert_eq!(bp.mappings.len(), 1); + assert_eq!(bp.on_missing_endpoint, MissingEndpointPolicy::Create); + + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + } + + #[test] + fn case_attributes_land_on_a_static_object_mapping() { + // Static, not change-tracked: a case-level column repeated across every one of the + // case's events would otherwise store one identical timed value per event. + let bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![AttributeMapping { + source_column: "region".into(), + name: "region".into(), + value_type: None, + }], + event_attributes: vec![], + }); + + assert_eq!(bp.mappings.len(), 2); + let object_mapping = bp.mappings.iter().find_map(|m| match m { + MappingEntry::Single(m) => match &m.target { + Target::Object { + timestamp, + attributes, + .. + } => Some((timestamp, attributes)), + _ => None, + }, + MappingEntry::Ordered { .. } => None, + }); + let (timestamp, attributes) = object_mapping.expect("an object mapping for the case"); + assert!(timestamp.is_none(), "case attributes must be static"); + assert_eq!(attributes.len(), 1); + } + + fn at(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339).expect("a timestamp") + } + + fn event(activity: &str, time: &str, extra: Vec<(&str, AttributeValue)>) -> Event { + let mut event = Event::new(activity.to_string()); + event + .attributes + .add_to_attributes(TIMESTAMP_NAME.to_string(), AttributeValue::Date(at(time))); + for (key, value) in extra { + event.attributes.add_to_attributes(key.to_string(), value); + } + event + } + + fn trace(case_id: &str, extra: Vec<(&str, AttributeValue)>, events: Vec) -> Trace { + let mut trace = Trace::new(); + trace.attributes.add_to_attributes( + TRACE_ID_NAME.to_string(), + AttributeValue::String(case_id.to_string()), + ); + for (key, value) in extra { + trace.attributes.add_to_attributes(key.to_string(), value); + } + trace.events = events; + trace + } + + /// One case with two events, a case-level attribute and an event-level one. + fn two_event_log() -> EventLog { + EventLog { + traces: vec![trace( + "case-1", + vec![("region", AttributeValue::String("EU".into()))], + vec![ + event( + "create", + "2020-01-01T00:00:00Z", + vec![("amount", AttributeValue::Int(7))], + ), + event("close", "2020-01-02T00:00:00Z", vec![]), + ], + )], + ..EventLog::default() + } + } + + #[test] + fn every_trace_becomes_one_case_object_its_events_hang_off() { + let locel = event_log_to_slim_ocel(&two_event_log()).expect("convert"); + + assert_eq!(locel.get_ob_types().collect::>(), [CASE_OBJECT_TYPE]); + assert_eq!(locel.get_num_obs(), 1); + assert_eq!(locel.get_num_evs(), 2); + + let case = locel.get_ob_by_id("case-1").expect("the case object"); + let mut attached: Vec<_> = locel + .get_e2o_rev(case) + .map(|(qualifier, ev)| (qualifier, locel.get_ev_type_of(ev))) + .collect(); + attached.sort_unstable(); + assert_eq!(attached, [("case", "close"), ("case", "create")]); + } + + #[test] + fn case_attributes_land_on_the_object_and_event_attributes_on_the_event() { + let locel = event_log_to_slim_ocel(&two_event_log()).expect("convert"); + + let case = locel.get_ob_by_id("case-1").expect("the case object"); + assert_eq!( + locel.get_ob_attr_vals(case, "region").collect::>(), + [( + &DateTime::UNIX_EPOCH.into(), + &OCELAttributeValue::String("EU".into()) + )] + ); + // The event-level attribute is not on the case, and the case-level one is not on the + // event: neither type was ever declared as carrying the other's. + assert!(locel.get_ob_attr_vals(case, "amount").next().is_none()); + + let created = locel.get_ev_by_id("ev_0").expect("the first event"); + assert_eq!(locel.get_ev_type_of(created), "create"); + assert_eq!( + locel.get_ev_time(created), + &at("2020-01-01T00:00:00Z"), + "the event keeps its own timestamp" + ); + assert_eq!( + locel.get_ev_attr_val(created, "amount"), + Some(&OCELAttributeValue::Integer(7)) + ); + assert_eq!(locel.get_ev_attr_val(created, "region"), None); + + // An event of a type that never carried the attribute still declares it, since the type + // is declared from every event of that type at once. An attribute no event carried is + // not declared. + let closed = locel.get_ev_by_id("ev_1").expect("the second event"); + assert_eq!( + locel.get_ev_attr_val(closed, "amount"), + None, + "'amount' belongs to 'create', which 'close' is not" + ); + } + + #[test] + fn traces_sharing_a_name_stay_two_cases() { + let log = EventLog { + traces: vec![ + trace( + "dup", + vec![], + vec![event("a", "2020-01-01T00:00:00Z", vec![])], + ), + trace( + "dup", + vec![], + vec![event("b", "2020-01-02T00:00:00Z", vec![])], + ), + ], + ..EventLog::default() + }; + + let locel = event_log_to_slim_ocel(&log).expect("convert"); + assert_eq!(locel.get_num_obs(), 2); + let ids: HashSet<&str> = locel.get_all_obs().map(|o| locel.get_ob_id(o)).collect(); + assert_eq!(ids.len(), 2, "two objects must not share an id"); + assert!(ids.contains("dup")); + } + + #[test] + fn a_xes_file_becomes_a_slim_linked_ocel() { + use crate::Importable; + + let path = crate::test_utils::get_test_data_path() + .join("xes") + .join("small-example.xes"); + let log = EventLog::import_from_path(&path).expect("import the fixture"); + let locel = event_log_to_slim_ocel(&log).expect("convert"); + + assert_eq!(locel.get_num_obs(), log.traces.len()); + assert_eq!( + locel.get_num_evs(), + log.traces.iter().map(|t| t.events.len()).sum::() + ); + assert!(locel.get_ob_by_id("Trace number one").is_some()); + for ev in locel.get_all_evs() { + assert_eq!( + locel.get_e2o(ev).count(), + 1, + "every event belongs to exactly one case" + ); + } + // The second trace's events carry no `concept:name`, so they fall back rather than being + // dropped. + assert!(locel.get_ev_types().any(|t| t == UNKNOWN_ACTIVITY)); + } + + /// The point of writing this against [`ExtractionSink`]: the deferring, file-backed sink + /// takes the same log with no OCEL in between, and settles the case relations at finalize. + #[cfg(feature = "ocel-duckdb")] + #[test] + fn the_same_log_streams_into_a_file_backed_sink() { + use super::super::duckdb_sink::DuckDbSink; + use crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb; + + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("sink.duckdb"); + let mut sink = DuckDbSink::new(&path).expect("open sink"); + write_event_log_to_sink(&two_event_log(), &mut sink).expect("write"); + ExtractionSink::finalize(&mut sink).expect("finalize"); + + let con = duckdb::Connection::open(&path).expect("reopen"); + let ocel = read_ocel_from_duckdb(&con).expect("read back"); + assert_eq!(ocel.objects.len(), 1); + assert_eq!(ocel.objects[0].id, "case-1"); + assert_eq!(ocel.objects[0].object_type, CASE_OBJECT_TYPE); + assert_eq!(ocel.events.len(), 2); + for event in &ocel.events { + assert_eq!( + event + .relationships + .iter() + .map(|r| (r.object_id.as_str(), r.qualifier.as_str())) + .collect::>(), + [("case-1", CASE_QUALIFIER)] + ); + } + } + + #[cfg(feature = "bindings")] + #[test] + fn the_registry_reads_xes_bytes_as_object_centric_data() { + use crate::bindings::{RegistryItem, RegistryItemKind}; + + let path = crate::test_utils::get_test_data_path() + .join("xes") + .join("small-example.xes"); + let bytes = std::fs::read(&path).expect("read the fixture"); + + assert!( + RegistryItemKind::SlimLinkedOCEL + .known_import_formats() + .iter() + .any(|f| f.extension == "xes"), + "a host asking what the kind accepts must be told about xes" + ); + + let from_path = RegistryItem::load_from_path( + &RegistryItemKind::SlimLinkedOCEL, + &path.to_string_lossy(), + ) + .expect("load a xes path as a slim linked OCEL"); + assert!(matches!(from_path, RegistryItem::SlimLinkedOCEL(_))); + + let slim = RegistryItem::load_from_bytes(&RegistryItemKind::SlimLinkedOCEL, &bytes, "xes") + .expect("load a xes as a slim linked OCEL"); + match slim { + RegistryItem::SlimLinkedOCEL(locel) => { + assert!(locel.get_ob_by_id("Trace number one").is_some()); + } + other => panic!("expected a SlimLinkedOCEL, got {other:?}"), + } + + let ocel = RegistryItem::load_from_bytes(&RegistryItemKind::OCEL, &bytes, "xes") + .expect("load a xes as an OCEL"); + match ocel { + RegistryItem::OCEL(ocel) => { + assert_eq!( + ocel.object_types + .iter() + .map(|t| &t.name) + .collect::>(), + [CASE_OBJECT_TYPE] + ); + } + other => panic!("expected an OCEL, got {other:?}"), + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/catalog.rs b/process_mining/src/core/event_data/object_centric/extraction/catalog.rs new file mode 100644 index 00000000..ff023c4d --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/catalog.rs @@ -0,0 +1,439 @@ +//! Schema and column-domain facts about the data sources a blueprint reads. +//! +//! Deliberately not part of a blueprint: embedding a schema snapshot in the artifact makes it +//! bloated and lets it go stale silently. The caller supplies these, either discovered live or +//! loaded from a pinned snapshot. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Debug; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::value::ValueKind; + +/// One column's declared shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ColumnSchema { + /// Column name. + pub name: String, + /// The source's own type name, verbatim, for example `INTEGER` or `timestamp`. + pub col_type: String, + /// Whether the source permits `NULL` here. + pub nullable: bool, +} + +/// The `col_type` to record for a column whose kind nothing was able to establish, chosen so +/// [`ColumnSchema::declared_kind`] reads it back as `None`. +pub(crate) const UNTYPED_COL_TYPE: &str = "UNKNOWN"; + +impl ColumnSchema { + /// Map [`ColumnSchema::col_type`] to the [`ValueKind`] it denotes, for literal coercion in + /// `Predicate::prepare`. + /// + /// `col_type` comes verbatim from a real database and its spelling, case and decoration vary + /// by engine and by driver, so the match is case-insensitive and per word, splitting on + /// every non-alphanumeric character and ignoring a trailing width. The first word that names + /// a kind decides: + /// + /// - `bool`, `boolean`: [`ValueKind::Boolean`] + /// - `date`, or any word holding `timestamp` or `datetime`: [`ValueKind::Timestamp`] + /// (`timestamp`, `TIMESTAMPTZ`, `DATE`, `SMALLDATETIME`) + /// - `int`, `integer`, `bigint`, `smallint`, `tinyint`, `mediumint`, `serial`, `bigserial`, + /// `smallserial`: [`ValueKind::Integer`] (`INTEGER`, `int4`, `INT UNSIGNED`) + /// - `float`, `double`, `real`, `numeric`, `decimal`: [`ValueKind::Float`] + /// (`DOUBLE PRECISION`, `NUMERIC(10,2)`) + /// - `char`, `character`, `varchar`, `nchar`, `nvarchar`, `bpchar`, `text`, `citext`, `clob`, + /// `string`: [`ValueKind::Text`] (`VARCHAR(45)`, `character varying`) + /// - anything else: `None`, which disables coercion for that column rather than guessing. + /// + /// Matched per word rather than by substring, since `interval` and `point` contain `int` and + /// `daterange` contains `date`. A wrong kind here is silent: it coerces literals and drives + /// the compiler's `CAST`. + #[must_use] + pub fn declared_kind(&self) -> Option { + let lowered = self.col_type.to_ascii_lowercase(); + lowered + .split(|c: char| !c.is_ascii_alphanumeric()) + .find_map(word_kind) + } +} + +/// The [`ValueKind`] one already-lowercased word of a `col_type` names, if any. See +/// [`ColumnSchema::declared_kind`]. +fn word_kind(word: &str) -> Option { + const INTEGER: &[&str] = &[ + "int", + "integer", + "bigint", + "smallint", + "tinyint", + "mediumint", + "serial", + "bigserial", + "smallserial", + ]; + const FLOAT: &[&str] = &["float", "double", "real", "numeric", "decimal"]; + const TEXT: &[&str] = &[ + "char", + "character", + "varchar", + "nchar", + "nvarchar", + "bpchar", + "text", + "citext", + "clob", + "string", + ]; + + // A trailing width is part of the spelling, not of the type: `int4`, `float8`, `varchar2`. + let base = word.trim_end_matches(|c: char| c.is_ascii_digit()); + if base == "bool" || base == "boolean" { + Some(ValueKind::Boolean) + } else if word == "date" || word.contains("timestamp") || word.contains("datetime") { + Some(ValueKind::Timestamp) + } else if INTEGER.contains(&base) { + Some(ValueKind::Integer) + } else if FLOAT.contains(&base) { + Some(ValueKind::Float) + } else if TEXT.contains(&base) { + Some(ValueKind::Text) + } else { + None + } +} + +/// One table's declared shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TableSchema { + /// Table name. + pub name: String, + /// Columns, keyed by name. + pub columns: BTreeMap, +} + +impl TableSchema { + /// Build a schema from `(name, type, nullable)` triples. + pub fn new(name: &str, columns: I) -> Self + where + I: IntoIterator, + S: Into, + { + let columns = columns + .into_iter() + .map(|(n, t, nullable)| { + let n = n.into(); + ( + n.clone(), + ColumnSchema { + name: n, + col_type: t.into(), + nullable, + }, + ) + }) + .collect(); + Self { + name: name.to_string(), + columns, + } + } +} + +/// What the compiler and extractor know about the sources a blueprint names. +pub trait Catalog: Debug { + /// Whether `source_id` has any entry in the catalog, known or not. + /// + /// Lets a caller tell a source-id typo (no entry at all) apart from a known source with an + /// unknown table, which [`Catalog::table`] alone cannot distinguish. + fn has_source(&self, source_id: &str) -> bool; + + /// The schema of `table` in `source_id`, if known. + fn table(&self, source_id: &str, table: &str) -> Option<&TableSchema>; + + /// The distinct values of a column, when the caller has determined them. + /// + /// Used to name per-type views when a type is read from a column. `None` means "not + /// determined", which is different from a known-empty domain. + fn column_domain( + &self, + source_id: &str, + table: &str, + column: &str, + ) -> Option<&BTreeSet>; +} + +/// The concrete, serializable [`Catalog`]. +/// +/// This is the form that crosses a bindings boundary, that an editor holds and sends back, and +/// that gets pinned to disk so a compile can be reproduced against a schema that has since +/// changed. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ExtractionCatalog { + /// Table schemas, keyed by source id then table name. + pub tables: BTreeMap>, + /// Column domains, keyed by source id, then table name, then column name. + pub domains: BTreeMap>>>, + /// A handful of real rows per table, keyed by source id then table name, to show a person + /// what the data looks like. + /// + /// Deliberately unreachable through the [`Catalog`] trait: unlike + /// [`domains`](ExtractionCatalog::domains), a preview is incomplete, so compiling from one + /// would emit views only for the types that happened to appear first. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub previews: BTreeMap>, +} + +/// A few real rows of one table, for display only. +/// +/// Rows are aligned to [`TablePreview::columns`] so a wide table can be read across. A cell is +/// `None` for SQL `NULL`, distinct from `Some(String::new())`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TablePreview { + /// Column names, in the order the rows are aligned to. + pub columns: Vec, + /// Rows, each the same length as `columns`. + pub rows: Vec>>, +} + +impl TablePreview { + /// Distinct non-null values seen for `column`, in first-seen order, capped at `limit`. + #[must_use] + pub fn column_values(&self, column: &str, limit: usize) -> Vec<&str> { + if limit == 0 { + return Vec::new(); + } + let Some(idx) = self.columns.iter().position(|c| c == column) else { + return Vec::new(); + }; + let mut seen = Vec::new(); + for row in &self.rows { + let Some(Some(v)) = row.get(idx) else { + continue; + }; + if !seen.contains(&v.as_str()) { + seen.push(v.as_str()); + if seen.len() == limit { + break; + } + } + } + seen + } +} + +impl ExtractionCatalog { + /// An empty catalog. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Add a table schema, replacing any schema already recorded for that name. + #[must_use] + pub fn with_table(mut self, source_id: &str, schema: TableSchema) -> Self { + self.tables + .entry(source_id.to_string()) + .or_default() + .insert(schema.name.clone(), schema); + self + } + + /// Record the distinct values of a column. + #[must_use] + pub fn with_domain>( + mut self, + source_id: &str, + table: &str, + column: &str, + values: I, + ) -> Self { + self.domains + .entry(source_id.to_string()) + .or_default() + .entry(table.to_string()) + .or_default() + .insert(column.to_string(), values.into_iter().collect()); + self + } + + /// Record preview rows for a table, replacing any already held. + #[must_use] + pub fn with_preview(mut self, source_id: &str, table: &str, preview: TablePreview) -> Self { + self.previews + .entry(source_id.to_string()) + .or_default() + .insert(table.to_string(), preview); + self + } + + /// The preview rows for a table, if any were fetched. + #[must_use] + pub fn preview(&self, source_id: &str, table: &str) -> Option<&TablePreview> { + self.previews.get(source_id)?.get(table) + } +} + +impl Catalog for ExtractionCatalog { + fn has_source(&self, source_id: &str) -> bool { + self.tables.contains_key(source_id) + } + + fn table(&self, source_id: &str, table: &str) -> Option<&TableSchema> { + self.tables.get(source_id)?.get(table) + } + + fn column_domain( + &self, + source_id: &str, + table: &str, + column: &str, + ) -> Option<&BTreeSet> { + self.domains.get(source_id)?.get(table)?.get(column) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn catalog() -> ExtractionCatalog { + ExtractionCatalog::new() + .with_table( + "erp", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("state", "TEXT", true)], + ), + ) + .with_domain( + "erp", + "orders", + "state", + ["draft".to_string(), "sale".to_string()], + ) + } + + #[test] + fn looks_up_tables_and_columns() { + let c = catalog(); + let t = c.table("erp", "orders").expect("table present"); + assert_eq!( + t.columns.get("id").map(|c| c.col_type.as_str()), + Some("INTEGER") + ); + assert!(c.table("erp", "missing").is_none()); + assert!(c.table("other", "orders").is_none()); + } + + #[test] + fn has_source_distinguishes_an_unknown_source_from_an_unknown_table() { + let c = catalog(); + assert!(c.has_source("erp")); + assert!(!c.has_source("nope")); + } + + #[test] + fn distinguishes_an_unknown_domain_from_an_empty_one() { + let c = catalog().with_domain("erp", "orders", "empty_col", Vec::::new()); + assert_eq!( + c.column_domain("erp", "orders", "state").map(BTreeSet::len), + Some(2) + ); + assert!(c.column_domain("erp", "orders", "id").is_none()); + assert_eq!( + c.column_domain("erp", "orders", "empty_col") + .map(BTreeSet::len), + Some(0), + "a recorded-but-empty domain must not read back as unrecorded" + ); + } + + fn kind_of(col_type: &str) -> Option { + ColumnSchema { + name: "c".into(), + col_type: col_type.into(), + nullable: false, + } + .declared_kind() + } + + #[test] + fn declared_kind_maps_common_col_types_case_insensitively_per_word() { + assert_eq!(kind_of("INTEGER"), Some(ValueKind::Integer)); + assert_eq!(kind_of("int4"), Some(ValueKind::Integer)); + assert_eq!(kind_of("BIGINT"), Some(ValueKind::Integer)); + assert_eq!(kind_of("INT UNSIGNED"), Some(ValueKind::Integer)); + assert_eq!(kind_of("TEXT"), Some(ValueKind::Text)); + assert_eq!(kind_of("VARCHAR"), Some(ValueKind::Text)); + assert_eq!(kind_of("VARCHAR(45)"), Some(ValueKind::Text)); + assert_eq!(kind_of("character varying"), Some(ValueKind::Text)); + assert_eq!(kind_of("timestamp"), Some(ValueKind::Timestamp)); + assert_eq!(kind_of("TIMESTAMPTZ"), Some(ValueKind::Timestamp)); + assert_eq!( + kind_of("TIMESTAMP(3) WITHOUT TIME ZONE"), + Some(ValueKind::Timestamp) + ); + assert_eq!(kind_of("SMALLDATETIME"), Some(ValueKind::Timestamp)); + assert_eq!(kind_of("DOUBLE"), Some(ValueKind::Float)); + assert_eq!(kind_of("DOUBLE PRECISION"), Some(ValueKind::Float)); + assert_eq!(kind_of("REAL"), Some(ValueKind::Float)); + assert_eq!(kind_of("NUMERIC(10,2)"), Some(ValueKind::Float)); + assert_eq!(kind_of("BOOLEAN"), Some(ValueKind::Boolean)); + assert_eq!(kind_of("bool"), Some(ValueKind::Boolean)); + } + + #[test] + fn a_type_that_merely_contains_a_kind_word_declares_nothing() { + for col_type in [ + "interval", + "point", + "int4range", + "daterange", + "numrange", + "GEOMETRY", + "tsvector", + UNTYPED_COL_TYPE, + ] { + assert_eq!(kind_of(col_type), None, "{col_type} must declare no kind"); + } + } + + #[test] + fn column_values_are_distinct_in_first_seen_order_and_respect_the_limit() { + let preview = TablePreview { + columns: vec!["state".to_string()], + rows: vec![ + vec![Some("draft".to_string())], + vec![None], + vec![Some("sale".to_string())], + vec![Some("draft".to_string())], + vec![Some("done".to_string())], + ], + }; + assert_eq!( + preview.column_values("state", 10), + ["draft", "sale", "done"] + ); + assert_eq!(preview.column_values("state", 2), ["draft", "sale"]); + assert!(preview.column_values("state", 0).is_empty()); + assert!(preview.column_values("missing", 10).is_empty()); + } + + #[test] + fn round_trips_through_json() { + let c = catalog(); + let json = serde_json::to_string(&c).expect("serialize"); + let back: ExtractionCatalog = serde_json::from_str(&json).expect("deserialize"); + assert_eq!( + back.table("erp", "orders").map(|t| t.columns.len()), + Some(2) + ); + assert_eq!( + back.column_domain("erp", "orders", "state") + .map(BTreeSet::len), + Some(2) + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/compile.rs b/process_mining/src/core/event_data/object_centric/extraction/compile.rs new file mode 100644 index 00000000..302b3efe --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/compile.rs @@ -0,0 +1,2491 @@ +//! Compile a [`Blueprint`] into SQL presenting the OCEL 2.0 surface over the untouched source +//! tables, so a log can be queried without ever being materialised. +//! +//! [`extract`](super::extract::extract) is the reference semantics. A mapping the emitter cannot +//! reproduce becomes a [`CompileError`] naming the mapping and the reason, and the rest of the +//! blueprint still compiles. +//! +//! Each relation is stored as a bare `SELECT` body ([`ViewDef`]), which [`CompiledOcel`] emits +//! three ways: [`CompiledOcel::ddl`] wraps them in `CREATE VIEW`, [`CompiledOcel::with_prelude`] +//! inlines them as a `WITH` prelude in front of an analysis query (needs no DDL right, so it runs +//! against a read-only database), and [`CompiledOcel::materialize_ddl`] emits `CREATE TABLE ... AS` +//! for callers that have write rights. +//! +//! # Preconditions +//! +//! * The catalog describes the kinds the source's values actually have. The extractor decides +//! literal coercion, identity rendering and join-key matching from the runtime +//! [`Value`](super::value::Value) in each cell, while the compiler only has +//! [`ColumnSchema::declared_kind`](super::catalog::ColumnSchema::declared_kind). Those agree +//! under a statically-typed engine but not under `SQLite`, which stores a type per cell. +//! * Every source table is reachable under its bare name. The emitted SQL names only the table, so +//! a multi-source blueprint requires the caller to attach or alias the sources into one +//! namespace. +//! * Text semantics are the engine's, not Rust's, and the two differ at the edges: regular +//! expressions are handed over verbatim, where Rust's `regex`, RE2 (`DuckDB`) and POSIX AREs +//! (`PostgreSQL`) disagree on some constructs; and `trim` strips ASCII space where `str::trim` +//! strips all Unicode whitespace. Either can keep different rows than an extraction. +//! * The compiler is a pure function of blueprint plus catalog: it opens no connection and reads no +//! row. Domains that have to be measured arrive through [`Catalog::column_domain`]. + +pub(crate) mod dialect; +pub(crate) mod emit; + +#[cfg(test)] +mod tests; + +#[cfg(all(test, feature = "ocel-duckdb"))] +mod differential; + +use std::collections::{BTreeMap, BTreeSet}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub use dialect::SqlDialect; + +use emit::{ + attribute_sql, identity_sql, predicate_sql, split_sql, timestamp_sql, Emitter, ROW_ALIAS, +}; + +use super::blueprint::{ + Blueprint, EventEndpoint, IdRendering, InlineObjectRef, Mapping, MissingEndpointPolicy, NodeOp, + ObjectEndpoint, Target, +}; +use super::catalog::{Catalog, ColumnSchema, TableSchema}; +use super::desugar::desugar_with_paths; +use super::expr::{AttributeMapping, ValueExpression}; +use super::report::MappingRef; +use super::schema::full_node_schemas; +use crate::core::event_data::object_centric::OCELAttributeType; + +/// How many distinct values a column domain may have before a per-type emission refuses it. +/// +/// A per-type shape emits one view per type name, so an open-ended column would emit thousands. +/// Exceeding the cap is an error naming the column, never a silent truncation. The consolidated +/// shape carries the type as a column value and has no such limit. +pub const MAX_TYPE_DOMAIN: usize = 512; + +/// Relation names the emitter reserves, so a type named after one of them is reported rather +/// than silently overwriting the relation it collides with. +const RESERVED_RELATIONS: &[&str] = &[ + "event", + "object", + "event_object", + "object_object", + "event_map_type", + "object_map_type", +]; + +/// Which OCEL surface the compiler emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +pub enum EmissionShape { + /// One `event_` / `object_` view per declared type, plus `event`, `object`, + /// `event_object`, `object_object` and the two type maps: the OCEL 2.0 layout external tooling + /// reads. + #[default] + PerType, + /// The `events`/`objects`/`e2o`/`o2o`/`object_attribute_changes`/`event_attr_meta` layout + /// `rust4pm`'s own reader (`DuckDbLinkedOCEL`) consumes. A type is stored as a column value + /// rather than encoded into a view name, so no + /// [`Catalog::column_domain`] lookup is needed and + /// [`RejectReason::DynamicTypeName`] never fires. + Consolidated, +} + +/// One compiled relation: a name and the bare `SELECT` that defines it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ViewDef { + /// The relation's name, unquoted. + pub name: String, + /// A bare `SELECT` body with no `CREATE` wrapper, so the same text serves a view, a CTE and + /// a `CREATE TABLE AS`. + pub body: String, +} + +/// Why a mapping could not be compiled to a view. Every variant names something the emitter +/// refused to guess at. +/// +/// Serializable but not deserializable: several variants carry `&'static str` fields, so this only +/// ever crosses a bindings boundary outbound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[non_exhaustive] +pub enum RejectReason { + /// A `Target::Event` with no `id` expression: the extractor mints a fresh `UUID` per row, a + /// nondeterministic side effect with no relational denotation. + SynthesizedId { + /// The absent field. + field: &'static str, + }, + /// A type name is read from the data and the catalog supplies no domain for the column it + /// comes from, so there is no name to put in `CREATE VIEW event_`. + DynamicTypeName { + /// The position whose type is dynamic. + field: &'static str, + /// Why no domain was available. + detail: String, + }, + /// A column domain has more distinct values than [`MAX_TYPE_DOMAIN`], so a per-type shape + /// would emit one view per value. + TypeDomainTooLarge { + /// The column the domain came from. + column: String, + /// How many values it has. + size: usize, + /// The cap. + cap: usize, + }, + /// A type name collides with one of the relations the emitter itself defines. + ReservedTypeName { + /// The offending type name. + name: String, + }, + /// A mapping reads a node that is not declared. + UnknownNode { + /// The node id. + node: String, + }, + /// A node's column shape could not be resolved, typically because of an unknown source table. + UnresolvedNodeSchema { + /// The node id. + node: String, + }, + /// The node graph contains a cycle, so there is no order in which to emit it. + NodeCycle { + /// A node id taking part in the cycle. + node: String, + }, + /// A node has no columns, and SQL has no zero-column `SELECT`. + EmptyProjection { + /// The node id. + node: String, + }, + /// A `Union` node with no inputs. + EmptyUnion { + /// The node id. + node: String, + }, + /// An expression reads a column the node does not have. + UnknownColumn { + /// The column name. + column: String, + /// Which position referenced it. + field: &'static str, + }, + /// The catalog's `col_type` for a column maps to no + /// [`ValueKind`](super::value::ValueKind), so the rule the extractor applies to that + /// column's values cannot be decided at compile time. + UndeclaredColumnKind { + /// The column name. + column: String, + /// The catalog's own type string. + col_type: String, + /// Which position referenced it. + field: &'static str, + }, + /// A `Float` or `Timestamp` column, which has no + /// [`Value::canonical_string`](super::value::Value::canonical_string), is used at an identity + /// position, where it makes the expression `None` for every row. + UnstableIdentityRendering { + /// The column name. + column: String, + /// The catalog's own type string. + col_type: String, + /// Which position referenced it. + field: &'static str, + }, + /// A column whose [`Value::display_string`](super::value::Value::display_string) a SQL cast + /// does not reproduce feeds a position that reads it as text. Rust writes `1` where `DuckDB` + /// writes `1.0`, and keeps a timestamp's original offset. + UnstableDisplayRendering { + /// The column name. + column: String, + /// The catalog's own type string. + col_type: String, + /// Which position referenced it. + field: &'static str, + }, + /// A timestamp is parsed by a `chrono` cascade that is not translated to SQL. + ResidualTimestamp { + /// What about it is residual. + detail: String, + }, + /// A join key column's declared kind is unknown, so whether the extractor's kind-tagged keys + /// can match is not decidable at compile time. + UndecidableJoinKey { + /// The join node's id. + node: String, + /// Which side the column is on. + side: &'static str, + /// The column name. + column: String, + /// The catalog's own type string. + col_type: String, + }, + /// A regular expression does not compile at all. + InvalidRegex { + /// The pattern. + pattern: String, + /// The compiler's message. + message: String, + }, + /// A `Template` has an unterminated or empty placeholder, which makes it `None` for every + /// row. + InvalidTemplate { + /// The template text. + template: String, + /// What is wrong with it. + reason: String, + }, + /// An attribute's declared type is not the source column's kind, and the coercion + /// `attribute_value` applies is not one a typed SQL column can reproduce: its fallback on + /// failure is the cell's own natural value, which has a different type. + AttributeCoercion { + /// The attribute name. + attribute: String, + /// The source column. + column: String, + /// The catalog's own type string. + col_type: String, + /// The declared OCEL attribute type. + declared: &'static str, + }, + /// A mapping whose type name comes from the data declares an attribute that another mapping + /// declares under a different type. A statically-named type has its declarations reconciled + /// before the first row. A data-named one declares lazily, one row at a time, so which type + /// wins depends on row order. + DynamicTypeAttributeConflict { + /// The attribute name. + attribute: String, + }, + /// A relation view's dependencies on other relation views form a cycle, so no order exists + /// in which its `CREATE VIEW` (or CTE, or `CREATE TABLE ... AS`) could run. Only reachable + /// from a blueprint that skipped [`validate`](super::validate()). + ViewCycle { + /// The relation's name. + view: String, + }, + /// The blueprint does not [`validate`](super::validate::validate) against the catalog, so + /// nothing was compiled. One per + /// [`ValidationError`](super::validate::ValidationError), rendered through its `Display`. + Invalid { + /// The rendered validation error. + detail: String, + }, +} + +impl std::fmt::Display for RejectReason { + #[allow(clippy::too_many_lines)] + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RejectReason::SynthesizedId { field } => write!( + f, + "'{field}' is absent, so ids would be random UUIDs that differ every run. Point \ + '{field}' at a column that identifies the row." + ), + RejectReason::DynamicTypeName { field, detail } => write!( + f, + "'{field}' is read from the data and no column domain is available ({detail}), \ + so the per-type view name is unknown at compile time" + ), + RejectReason::TypeDomainTooLarge { column, size, cap } => write!( + f, + "the domain of '{column}' has {size} values, above the per-type cap of {cap}" + ), + RejectReason::ReservedTypeName { name } => write!( + f, + "type name '{name}' collides with a relation the compiler defines" + ), + RejectReason::UnknownNode { node } => write!(f, "no node '{node}' is declared"), + RejectReason::UnresolvedNodeSchema { node } => { + write!(f, "the column shape of node '{node}' could not be resolved") + } + RejectReason::NodeCycle { node } => { + write!(f, "node '{node}' takes part in a cycle") + } + RejectReason::EmptyProjection { node } => write!( + f, + "node '{node}' has no columns, and SQL has no zero-column SELECT" + ), + RejectReason::EmptyUnion { node } => { + write!(f, "union node '{node}' has no inputs") + } + RejectReason::UnknownColumn { column, field } => write!( + f, + "column '{column}' used by '{field}' is not declared for this node" + ), + RejectReason::UndeclaredColumnKind { + column, + col_type, + field, + } => write!( + f, + "column '{column}' is declared '{col_type}', which maps to no value kind, so \ + '{field}' cannot be decided without reading the data" + ), + RejectReason::UnstableIdentityRendering { + column, + col_type, + field, + } => write!( + f, + "column '{column}' ({col_type}) has no canonical identity rendering, so \ + '{field}' is None for every row" + ), + RejectReason::UnstableDisplayRendering { + column, + col_type, + field, + } => write!( + f, + "column '{column}' ({col_type}) feeds '{field}' through a text rendering a SQL \ + cast does not reproduce" + ), + RejectReason::ResidualTimestamp { detail } => { + write!(f, "timestamp is residual: {detail}") + } + RejectReason::UndecidableJoinKey { + node, + side, + column, + col_type, + } => write!( + f, + "join '{node}': the {side} key '{column}' is declared '{col_type}', which maps \ + to no value kind, so whether the extractor's kind-tagged keys can match is not \ + decidable at compile time" + ), + RejectReason::InvalidRegex { pattern, message } => { + write!(f, "invalid regular expression '{pattern}': {message}") + } + RejectReason::InvalidTemplate { template, reason } => { + write!(f, "invalid template '{template}': {reason}") + } + RejectReason::AttributeCoercion { + attribute, + column, + col_type, + declared, + } => write!( + f, + "attribute '{attribute}' reads column '{column}' ({col_type}) as '{declared}', a \ + coercion whose fallback value has a different type than the column it would be \ + stored in" + ), + RejectReason::DynamicTypeAttributeConflict { attribute } => write!( + f, + "attribute '{attribute}' is declared under conflicting types, and this mapping's \ + type name comes from the data, so which declaration wins depends on row order" + ), + RejectReason::ViewCycle { view } => write!( + f, + "relation '{view}' could not be ordered: its dependencies on other relations \ + form a cycle" + ), + RejectReason::Invalid { detail } => { + write!(f, "the blueprint does not validate: {detail}") + } + } + } +} + +impl std::error::Error for RejectReason {} + +/// A mapping that produced no view, and why. +/// +/// Compilation never fails wholesale: an uncompilable mapping is skipped and recorded here, and +/// everything else still compiles. +/// +/// Serializable but not deserializable: [`RejectReason`] is not, so neither is this. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct CompileError { + /// The mapping this is about, or `None` for a blueprint-level problem. + pub mapping: Option, + /// Why it could not be compiled. + pub reason: RejectReason, +} + +impl std::fmt::Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.mapping { + Some(m) => write!(f, "{}: {}", m.path, self.reason), + None => write!(f, "{}", self.reason), + } + } +} + +impl std::error::Error for CompileError {} + +/// What a [`Probe`] guards. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[non_exhaustive] +pub enum ProbeKind { + /// Two objects claim one id under different types. The extractor keeps the first and reports + /// the collision, and the views keep both. + AmbiguousObjectIdentity, + /// Two events claim one id. See [`Self::AmbiguousObjectIdentity`]. + AmbiguousEventIdentity, + /// Two rows of one mapping give one object id different static attribute values. The + /// extractor writes the mapping's first row for an id and ignores later repeats. SQL rows + /// are unordered, so the views agree only when the repeats carry the same values. + AmbiguousStaticObjectAttributes, + /// A type name read from a column has a value outside the domain the catalog supplied, so + /// the compiled view set is missing the entities carrying it. + StaleTypeDomain { + /// The column the domain came from. + column: String, + }, +} + +impl std::fmt::Display for ProbeKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeKind::AmbiguousObjectIdentity => { + write!(f, "one object id carries more than one type") + } + ProbeKind::AmbiguousEventIdentity => { + write!(f, "one event id carries more than one event") + } + ProbeKind::AmbiguousStaticObjectAttributes => write!( + f, + "one object id is given different static attribute values by one mapping" + ), + ProbeKind::StaleTypeDomain { column } => write!( + f, + "column '{column}' holds a value outside the domain this compile pinned" + ), + } + } +} + +/// A data-dependent assumption the compiled relations make, as SQL that returns zero rows when +/// the assumption holds. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Probe { + /// The mapping this is about, or `None` for a whole-log check. + pub mapping: Option, + /// What it guards. + pub kind: ProbeKind, + /// The check itself, as a `SELECT` returning zero rows when the guard holds. + pub sql: String, +} + +/// A blueprint compiled to SQL. +/// +/// Serializable but not deserializable: [`Self::errors`] holds [`CompileError`], which is not, so +/// neither is this. Crosses a bindings boundary outbound only, as a compile binding's return +/// value. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct CompiledOcel { + dialect: SqlDialect, + shape: EmissionShape, + views: Vec, + probes: Vec, + errors: Vec, +} + +impl CompiledOcel { + /// Which shape this was compiled for. + #[must_use] + pub fn shape(&self) -> EmissionShape { + self.shape + } + + /// The relation bodies, in dependency order: a relation another one's body `EXISTS`-checks + /// against comes first, so every emission path can write this list top to bottom. A relation + /// whose dependencies could not be ordered is dropped and reported in [`Self::errors`] as + /// [`RejectReason::ViewCycle`], which only a blueprint that skipped + /// [`validate`](super::validate()) can reach. + #[must_use] + pub fn relations(&self) -> &[ViewDef] { + &self.views + } + + /// The probes. Each must return zero rows for the relations to agree with an extraction. + #[must_use] + pub fn probes(&self) -> &[Probe] { + &self.probes + } + + /// The mappings that produced no view, and why. Never a reason to discard the rest. + #[must_use] + pub fn errors(&self) -> &[CompileError] { + &self.errors + } + + /// Every relation as a `CREATE VIEW`, in dependency order, each terminated by the dialect's + /// statement separator. + #[must_use] + pub fn ddl(&self) -> String { + self.statements(|d, v| d.create_view(&v.name, &v.body)) + } + + /// Every relation as a `CREATE TABLE ... AS`, so a relation referenced many times is + /// computed once instead of re-inlined. Needs write rights, unlike [`Self::with_prelude`]. + #[must_use] + pub fn materialize_ddl(&self) -> String { + self.statements(|d, v| d.create_table_as(&v.name, &v.body)) + } + + fn statements(&self, render: impl Fn(SqlDialect, &ViewDef) -> String) -> String { + self.views + .iter() + .map(|v| { + format!( + "{}{}", + render(self.dialect, v), + self.dialect.statement_separator() + ) + }) + .collect::>() + .join("\n") + } + + /// `analysis_sql` with every relation bound as a `WITH` CTE in dependency order, so an + /// analysis query can name `event`, `object`, `event_object` and friends without any view + /// existing. Needs no DDL right, so it runs against a read-only database. + /// + /// A relation referenced many times is re-inlined by the engine each time. Prefer + /// [`Self::ddl`] or [`Self::materialize_ddl`] where that matters. + #[must_use] + pub fn with_prelude(&self, analysis_sql: &str) -> String { + if self.views.is_empty() { + return analysis_sql.to_string(); + } + let ctes: Vec = self + .views + .iter() + .map(|v| format!("{} AS (\n{}\n)", self.dialect.quote_ident(&v.name), v.body)) + .collect(); + format!("WITH {}\n{analysis_sql}", ctes.join(",\n")) + } + + /// Each probe rewritten to run with no views present, by prepending the relation CTEs. + #[must_use] + pub fn probe_statements(&self) -> Vec { + self.probes + .iter() + .map(|p| self.with_prelude(&p.sql)) + .collect() + } +} + +/// Compile `blueprint` against `catalog` into SQL. +/// +/// Pure: no connection is opened and no row is read. A mapping the emitter cannot reproduce +/// exactly is skipped and recorded in [`CompiledOcel::errors`], and the rest still compiles. +/// +/// `catalog` supplies both the column types every rule here is decided from and, for a type name +/// read from a column, the [`column_domain`](super::catalog::Catalog::column_domain) the +/// per-type view names come from. +/// +/// [`validate`](super::validate::validate) runs first, as it does in +/// [`extract`](super::extract::extract): a blueprint that does not pass it compiles to no +/// relations and one [`RejectReason::Invalid`] per validation error. Three of `validate`'s rules +/// matter here in particular: the version check (which lives only there), duplicate node ids +/// (which would make the emitter read a node's op and its columns from two different +/// declarations), and node cycles or unknown tables (which would otherwise degrade to a silently +/// empty log). +#[must_use] +pub fn compile( + blueprint: &Blueprint, + catalog: &dyn Catalog, + dialect: SqlDialect, + shape: EmissionShape, +) -> CompiledOcel { + let mut out = CompiledOcel { + dialect, + shape, + views: Vec::new(), + probes: Vec::new(), + errors: Vec::new(), + }; + + let validation_errors = super::validate::validate(blueprint, catalog); + if !validation_errors.is_empty() { + out.errors = validation_errors + .iter() + .map(|e| CompileError { + mapping: None, + reason: RejectReason::Invalid { + detail: e.to_string(), + }, + }) + .collect(); + return out; + } + + let full = full_node_schemas(blueprint, catalog); + let emitter = Emitter::from_schemas(blueprint, &full, dialect); + let desugared = desugar_with_paths(blueprint); + let mappings: Vec<(MappingRef, Mapping)> = desugared + .into_iter() + .enumerate() + .map(|(index, (path, m))| (MappingRef::new(index, path, &m), m)) + .collect(); + + let attrs = AttributePlan::build(blueprint, catalog, &emitter, &mappings, shape); + let mut acc = Accumulator::default(); + + for (mapping_ref, mapping) in &mappings { + let mut ctx = MappingCtx { + dialect, + shape, + blueprint, + catalog, + emitter: &emitter, + attrs: &attrs, + mapping_ref, + mapping, + probes: Vec::new(), + }; + match ctx.compile(&mut acc) { + Ok(()) => out.probes.append(&mut ctx.probes), + Err(reason) => out.errors.push(CompileError { + mapping: Some(mapping_ref.clone()), + reason, + }), + } + } + + match shape { + EmissionShape::PerType => assemble(dialect, &attrs, &acc, &mut out), + EmissionShape::Consolidated => assemble_consolidated(dialect, &attrs, &acc, &mut out), + } + out +} + +/// The declared attribute type of every `(kind, type name, attribute)` the blueprint produces, +/// reconciled across mappings as [`mapping_exec`](super::mapping_exec) does at extraction time: +/// conflicting declarations widen via [`OCELAttributeType::coalesce`], and every mapping's rows +/// convert to the widened type. +#[derive(Debug, Default)] +struct AttributePlan { + declared: BTreeMap<(&'static str, String, String), OCELAttributeType>, + conflicted: BTreeSet<(&'static str, String, String)>, + /// Event attributes of a mapping whose type name is read from the data under + /// [`EmissionShape::Consolidated`], where there is no name to key them under. Without them + /// [`Self::event_columns_wide`] would drop every dynamically-typed event's attributes. + dynamic_event_attrs: BTreeMap, +} + +impl AttributePlan { + /// Reconcile every mapping's declarations. + /// + /// A mapping whose node schema does not resolve, or whose type-name expression [`type_names`] + /// refuses, is skipped without a [`RejectReason`]: [`MappingCtx::compile`] re-derives the + /// identical reason and is the only place one is recorded. + fn build( + blueprint: &Blueprint, + catalog: &dyn Catalog, + emitter: &Emitter<'_, '_>, + mappings: &[(MappingRef, Mapping)], + shape: EmissionShape, + ) -> Self { + let mut plan = Self::default(); + for (_, m) in mappings { + let Some(schema) = emitter.schema_of(&m.node) else { + continue; + }; + let (kind, type_expr, attributes) = match &m.target { + Target::Event { + event_type, + attributes, + .. + } => ("event", event_type, attributes), + Target::Object { + object_type, + attributes, + .. + } => ("object", object_type, attributes), + Target::E2O { .. } | Target::O2O { .. } => continue, + }; + // `PerType` needs every name a domain enumerates, since each becomes a view. Under + // `Consolidated` the type is a column value, so only a statically-known (`Constant`) + // name is ever reconciled here. + let names = match shape { + EmissionShape::PerType => { + type_names(blueprint, catalog, emitter, &m.node, type_expr) + .map(|t| t.names) + .unwrap_or_default() + } + EmissionShape::Consolidated => static_type_names(type_expr), + }; + if names.is_empty() && shape == EmissionShape::Consolidated && kind == "event" { + for a in attributes { + let declared = resolve_attribute_type(a, schema); + plan.dynamic_event_attrs + .entry(a.name.clone()) + .and_modify(|prev| *prev = prev.coalesce(declared)) + .or_insert(declared); + } + } + for name in names { + for a in attributes { + let declared = resolve_attribute_type(a, schema); + let key = (kind, name.clone(), a.name.clone()); + match plan.declared.get(&key).copied() { + Some(prev) if prev != declared => { + plan.declared.insert(key.clone(), prev.coalesce(declared)); + plan.conflicted.insert(key); + } + Some(_) => {} + None => { + plan.declared.insert(key, declared); + } + } + } + } + } + plan + } + + /// The reconciled type for one attribute, falling back to this mapping's own resolution when + /// nothing was recorded (which only happens for a type name no plan pass saw). + fn type_of( + &self, + kind: &'static str, + type_name: &str, + a: &AttributeMapping, + schema: &TableSchema, + ) -> OCELAttributeType { + self.declared + .get(&(kind, type_name.to_string(), a.name.clone())) + .copied() + .unwrap_or_else(|| resolve_attribute_type(a, schema)) + } + + fn is_conflicted(&self, kind: &'static str, type_name: &str, attribute: &str) -> bool { + self.conflicted + .contains(&(kind, type_name.to_string(), attribute.to_string())) + } + + /// Every attribute name declared for `type_name`, sorted: the column list of its per-type + /// view. + fn columns_of(&self, kind: &'static str, type_name: &str) -> Vec<(String, OCELAttributeType)> { + self.declared + .iter() + .filter(|((k, t, _), _)| *k == kind && t == type_name) + .map(|((_, _, a), ty)| (a.clone(), *ty)) + .collect() + } + + /// The wide `events` table's attribute columns under [`EmissionShape::Consolidated`]: every + /// declared event attribute name, widened across every event type that declares it (not + /// merely within one, as [`Self::columns_of`] does), since that table is one row per event. + /// Mirrors the streaming `DuckDB` sink's `ev_attr_types`. + fn event_columns_wide(&self) -> Vec<(String, OCELAttributeType)> { + let mut widened: BTreeMap = self.dynamic_event_attrs.clone(); + for ((kind, _type_name, attribute), ty) in &self.declared { + if *kind != "event" { + continue; + } + widened + .entry(attribute.clone()) + .and_modify(|existing| *existing = existing.coalesce(*ty)) + .or_insert(*ty); + } + widened.into_iter().collect() + } +} + +/// The statically-known names a type expression can take without consulting a column domain: one +/// for a `Constant`, none for anything else. +fn static_type_names(expr: &ValueExpression) -> Vec { + match expr { + ValueExpression::Constant { value } => vec![value.clone()], + _ => Vec::new(), + } +} + +/// An attribute's declared type: `value_type` if given, else the source column's declared kind, +/// else `String`. Mirrors `resolve_attribute_type` in `mapping_exec`. +fn resolve_attribute_type(a: &AttributeMapping, schema: &TableSchema) -> OCELAttributeType { + if let Some(t) = a.value_type { + return t; + } + schema + .columns + .get(&a.source_column) + .and_then(ColumnSchema::declared_kind) + .map(|k| match k { + super::value::ValueKind::Text => OCELAttributeType::String, + super::value::ValueKind::Integer => OCELAttributeType::Integer, + super::value::ValueKind::Float => OCELAttributeType::Float, + super::value::ValueKind::Boolean => OCELAttributeType::Boolean, + super::value::ValueKind::Timestamp => OCELAttributeType::Time, + }) + .unwrap_or(OCELAttributeType::String) +} + +/// A type-name expression resolved to the set of names it can take, plus the SQL producing it. +#[derive(Debug)] +struct TypeNames { + sql: String, + names: Vec, + /// `Some(column)` when the names came from a catalog domain rather than a constant, so the + /// caller can attach the staleness probe. + domain_column: Option, +} + +/// Resolve a type-name expression. +/// +/// A `Constant` names one type. Anything else is read from the data, and per-type emission needs +/// the set of names up front, which +/// [`Catalog::column_domain`](super::catalog::Catalog::column_domain) supplies for a `Column` +/// expression tracing back to a single `Source`. The pinned set can go stale, which a +/// [`ProbeKind::StaleTypeDomain`] probe detects. +fn type_names( + blueprint: &Blueprint, + catalog: &dyn Catalog, + emitter: &Emitter<'_, '_>, + node_id: &str, + expr: &ValueExpression, +) -> Result { + if let ValueExpression::Constant { value } = expr { + return Ok(TypeNames { + sql: emitter.dialect.string_literal(value), + names: vec![value.clone()], + domain_column: None, + }); + } + let ValueExpression::Column { column } = expr else { + return Err(RejectReason::DynamicTypeName { + field: "type", + detail: "only a plain column expression can be given a domain".to_string(), + }); + }; + let Some((source_id, table)) = source_of(blueprint, node_id) else { + return Err(RejectReason::DynamicTypeName { + field: "type", + detail: format!("node '{node_id}' is not a source or a chain of filters over one"), + }); + }; + let Some(domain) = catalog.column_domain(&source_id, &table, column) else { + return Err(RejectReason::DynamicTypeName { + field: "type", + detail: format!("no domain for '{table}'.'{column}' in source '{source_id}'"), + }); + }; + if domain.is_empty() { + // An empty domain names no view, so every entity of this mapping would vanish while the + // compile reported nothing. + return Err(RejectReason::DynamicTypeName { + field: "type", + detail: format!( + "the domain recorded for '{table}'.'{column}' in source '{source_id}' is empty" + ), + }); + } + if domain.len() > MAX_TYPE_DOMAIN { + return Err(RejectReason::TypeDomainTooLarge { + column: column.clone(), + size: domain.len(), + cap: MAX_TYPE_DOMAIN, + }); + } + let schema = emitter + .schema_of(node_id) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: node_id.to_string(), + })?; + Ok(TypeNames { + sql: identity_sql(emitter.dialect, expr, schema, ROW_ALIAS, "type")?, + names: domain.iter().cloned().collect(), + domain_column: Some(column.clone()), + }) +} + +/// The `(source_id, table)` a node's rows ultimately come from, following `Filter` chains. A +/// `Join` or `Union` has no single one. +fn source_of(blueprint: &Blueprint, node_id: &str) -> Option<(String, String)> { + let mut current = node_id.to_string(); + for _ in 0..blueprint.nodes.len().max(1) { + match &blueprint.node(¤t)?.op { + NodeOp::Source { source_id, table } => return Some((source_id.clone(), table.clone())), + NodeOp::Filter { input, .. } => current = input.clone(), + NodeOp::Join { .. } | NodeOp::Union { .. } => return None, + } + } + None +} + +/// One event's projection into `event_`. +#[derive(Debug)] +struct EventBranch { + type_sql: String, + id_sql: String, + time_sql: String, + from: String, + filters: Vec, + /// Attribute name to the SQL producing its value on this branch. + attributes: BTreeMap, + /// Attribute name to the declared type [`Self::attributes`]'s SQL was rendered at, so + /// [`EmissionShape::Consolidated`]'s wide `events` table can coerce it to a further-widened + /// column type. + attribute_types: BTreeMap, + types: Vec, +} + +/// One object's projection into `object`. +#[derive(Debug)] +struct ObjectBranch { + type_sql: String, + id_sql: String, + from: String, + filters: Vec, + types: Vec, + /// Relation names this branch's `filters` semi-join against via `EXISTS`, so the `object` + /// view built from it is emitted after those relations rather than before. + depends_on: BTreeSet, +} + +/// One `(object, attribute)` observation's projection into `object_`. +#[derive(Debug)] +struct ObjectAttrBranch { + type_sql: String, + id_sql: String, + time_sql: String, + attribute: String, + value_sql: String, + /// The declared type [`Self::value_sql`] was rendered at, so + /// [`EmissionShape::Consolidated`]'s EAV `object_attribute_changes` can render `value` as text + /// and record `value_type` alongside it. + value_type: OCELAttributeType, + from: String, + filters: Vec, + types: Vec, +} + +/// One relation's projection into `event_object` or `object_object`. +#[derive(Debug)] +struct RelBranch { + left_sql: String, + right_sql: String, + qualifier_sql: String, + from: String, + filters: Vec, + /// Relation names this branch's `filters` semi-join against via `EXISTS`, so the relation + /// view built from it is emitted after those relations rather than before. + depends_on: BTreeSet, +} + +/// What an identity relation (`PerType`'s `object` and `event`, `Consolidated`'s `objects`) reads +/// off one branch, so one body builder serves both branch types. +trait EntityBranch { + fn id_sql(&self) -> &str; + fn type_sql(&self) -> &str; + fn from(&self) -> &str; + fn filters(&self) -> &[String]; +} + +impl EntityBranch for ObjectBranch { + fn id_sql(&self) -> &str { + &self.id_sql + } + fn type_sql(&self) -> &str { + &self.type_sql + } + fn from(&self) -> &str { + &self.from + } + fn filters(&self) -> &[String] { + &self.filters + } +} + +impl EntityBranch for EventBranch { + fn id_sql(&self) -> &str { + &self.id_sql + } + fn type_sql(&self) -> &str { + &self.type_sql + } + fn from(&self) -> &str { + &self.from + } + fn filters(&self) -> &[String] { + &self.filters + } +} + +/// A branch that can semi-join against another relation, so whatever is built from it has to be +/// emitted after that relation. +trait BranchDependencies { + fn depends_on(&self) -> &BTreeSet; +} + +impl BranchDependencies for ObjectBranch { + fn depends_on(&self) -> &BTreeSet { + &self.depends_on + } +} + +impl BranchDependencies for RelBranch { + fn depends_on(&self) -> &BTreeSet { + &self.depends_on + } +} + +#[derive(Debug, Default)] +struct Accumulator { + events: Vec, + objects: Vec, + object_attrs: Vec, + e2o: Vec, + o2o: Vec, + /// Set when at least one object mapping writes static attributes, so the assembler knows the + /// ambiguity probe is worth emitting. + static_attr_probe_needed: bool, +} + +struct MappingCtx<'a> { + dialect: SqlDialect, + shape: EmissionShape, + blueprint: &'a Blueprint, + catalog: &'a dyn Catalog, + emitter: &'a Emitter<'a, 'a>, + attrs: &'a AttributePlan, + mapping_ref: &'a MappingRef, + mapping: &'a Mapping, + probes: Vec, +} + +impl MappingCtx<'_> { + fn compile(&mut self, acc: &mut Accumulator) -> Result<(), RejectReason> { + let node = &self.mapping.node; + let schema = self + .emitter + .schema_of(node) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { node: node.clone() })? + .clone(); + let from = self + .dialect + .derived_table(&self.emitter.node_sql(node)?, ROW_ALIAS); + + let mut base_filters = Vec::new(); + if let Some(when) = &self.mapping.when { + base_filters.push(predicate_sql(self.dialect, when, &schema, ROW_ALIAS)?); + } + + match &self.mapping.target { + Target::Event { + event_type, + id, + timestamp, + attributes, + objects, + } => { + let id = id + .as_ref() + .ok_or(RejectReason::SynthesizedId { field: "id" })?; + self.compile_event( + acc, + &schema, + &from, + base_filters, + event_type, + id, + timestamp, + attributes, + objects, + ) + } + Target::Object { + object_type, + id, + timestamp, + attributes, + } => self.compile_object( + acc, + &schema, + &from, + base_filters, + object_type, + id, + timestamp.as_ref(), + attributes, + ), + Target::E2O { + event, + object, + qualifier, + } => self.compile_e2o(acc, &schema, &from, base_filters, event, object, qualifier), + Target::O2O { + source, + target, + qualifier, + } => self.compile_o2o(acc, &schema, &from, base_filters, source, target, qualifier), + } + } + + /// The relation an object-existence check (an `E2O`/`O2O`/inline-object endpoint guard) runs + /// `EXISTS` against, and the column its id lives under: `object`/`ocel_id` for + /// [`EmissionShape::PerType`], `objects`/`id` for [`EmissionShape::Consolidated`]. + fn object_relation(&self) -> (&'static str, &'static str) { + match self.shape { + EmissionShape::PerType => ("object", "ocel_id"), + EmissionShape::Consolidated => ("objects", "id"), + } + } + + /// [`Self::object_relation`] for an `E2O`'s event endpoint. + fn event_relation(&self) -> (&'static str, &'static str) { + match self.shape { + EmissionShape::PerType => ("event", "ocel_id"), + EmissionShape::Consolidated => ("events", "id"), + } + } + + /// [`type_names`] for [`EmissionShape::PerType`]. Under [`EmissionShape::Consolidated`] the + /// type is a column value with no per-type view to name, so this never calls + /// [`Catalog::column_domain`] and anything but a `Constant` resolves to SQL with no names. + fn resolve_type_names(&self, expr: &ValueExpression) -> Result { + if self.shape != EmissionShape::Consolidated { + return type_names( + self.blueprint, + self.catalog, + self.emitter, + &self.mapping.node, + expr, + ); + } + if let ValueExpression::Constant { value } = expr { + return Ok(TypeNames { + sql: self.dialect.string_literal(value), + names: vec![value.clone()], + domain_column: None, + }); + } + let schema = self.emitter.schema_of(&self.mapping.node).ok_or_else(|| { + RejectReason::UnresolvedNodeSchema { + node: self.mapping.node.clone(), + } + })?; + Ok(TypeNames { + sql: identity_sql(self.dialect, expr, schema, ROW_ALIAS, "type")?, + names: Vec::new(), + domain_column: None, + }) + } + + fn types_of( + &self, + kind: &'static str, + expr: &ValueExpression, + ) -> Result { + let resolved = self.resolve_type_names(expr)?; + if self.shape == EmissionShape::Consolidated { + // The type is a column value, not a view name: no collision is possible, and with + // no domain (see `resolve_type_names`) there is nothing that can go stale. + return Ok(resolved); + } + check_reserved(kind, &resolved.names)?; + Ok(resolved) + } + + /// The staleness probe for a type set that came from a catalog domain. + /// + /// `filters` are the mapping's final row-level guards, not merely its `when`: a row dropped + /// for a null id or an unparseable timestamp is absent from the views either way, so + /// reporting it as stale would fail over rows neither side emits. + fn push_stale_type_probe(&mut self, types: &TypeNames, from: &str, filters: &[String]) { + let Some(column) = &types.domain_column else { + return; + }; + let listed: Vec = types + .names + .iter() + .map(|n| self.dialect.string_literal(n)) + .collect(); + let mut probe_filters = filters.to_vec(); + probe_filters.push(format!("{} IS NOT NULL", types.sql)); + probe_filters.push(format!("{} NOT IN ({})", types.sql, listed.join(", "))); + self.probes.push(Probe { + mapping: Some(self.mapping_ref.clone()), + kind: ProbeKind::StaleTypeDomain { + column: column.clone(), + }, + sql: format!( + "SELECT DISTINCT {} AS ocel_type FROM {from} WHERE {}", + types.sql, + probe_filters.join(" AND ") + ), + }); + } + + /// Each attribute's reconciled declared type and the SQL producing its value, in the order + /// `attributes` gives them. `kind` is `"event"` or `"object"`. + fn attribute_values( + &self, + kind: &'static str, + types: &TypeNames, + attributes: &[AttributeMapping], + schema: &TableSchema, + ) -> Result, RejectReason> { + attributes + .iter() + .map(|a| { + for type_name in &types.names { + if types.domain_column.is_some() + && self.attrs.is_conflicted(kind, type_name, &a.name) + { + return Err(RejectReason::DynamicTypeAttributeConflict { + attribute: a.name.clone(), + }); + } + } + // Every name this branch can take shares one reconciled declaration. Picking the + // first is only ambiguous when they conflict, which the check above rules out. + let declared = self.attrs.type_of( + kind, + types.names.first().map_or("", String::as_str), + a, + schema, + ); + let sql = attribute_sql( + self.dialect, + &a.source_column, + &a.name, + declared, + schema, + ROW_ALIAS, + )?; + Ok((declared, sql)) + }) + .collect() + } + + /// `render_id`: the raw identity verbatim, or `-` under + /// [`IdRendering::TypePrefixed`]. + fn render_id(&self, type_sql: &str, raw_sql: &str) -> String { + match self.blueprint.id_rendering { + IdRendering::Raw => raw_sql.to_string(), + IdRendering::TypePrefixed => self.dialect.concat(&[ + type_sql.to_string(), + self.dialect.string_literal("-"), + raw_sql.to_string(), + ]), + } + } + + #[allow(clippy::too_many_arguments)] + fn compile_event( + &mut self, + acc: &mut Accumulator, + schema: &TableSchema, + from: &str, + mut filters: Vec, + event_type: &ValueExpression, + id: &ValueExpression, + timestamp: &super::expr::TimestampSource, + attributes: &[AttributeMapping], + objects: &[InlineObjectRef], + ) -> Result<(), RejectReason> { + let types = self.types_of("event", event_type)?; + let raw_id = identity_sql(self.dialect, id, schema, ROW_ALIAS, "id")?; + let time = timestamp_sql(self.dialect, timestamp, schema, ROW_ALIAS)?; + + // `run_event` drops a row whose type does not render, whose id is None or empty, or whose + // timestamp does not parse. + filters.push(format!("{} IS NOT NULL", types.sql)); + filters.push(format!("{raw_id} IS NOT NULL")); + filters.push(format!("{raw_id} <> ''")); + filters.extend(time.filter(self.dialect)); + self.push_stale_type_probe(&types, from, &filters); + + let id_sql = self.render_id(&types.sql, &raw_id); + + let mut attribute_sqls = BTreeMap::new(); + let mut attribute_types = BTreeMap::new(); + for (a, (declared, sql)) in attributes + .iter() + .zip(self.attribute_values("event", &types, attributes, schema)?) + { + attribute_sqls.insert(a.name.clone(), sql); + attribute_types.insert(a.name.clone(), declared); + } + + for inline in objects { + self.compile_inline_object(acc, schema, from, &filters, &id_sql, inline)?; + } + + acc.events.push(EventBranch { + type_sql: types.sql, + id_sql, + time_sql: time.sql(self.dialect), + from: from.to_string(), + filters, + attributes: attribute_sqls, + attribute_types, + types: types.names, + }); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn compile_object( + &mut self, + acc: &mut Accumulator, + schema: &TableSchema, + from: &str, + mut filters: Vec, + object_type: &ValueExpression, + id: &ValueExpression, + timestamp: Option<&super::expr::TimestampSource>, + attributes: &[AttributeMapping], + ) -> Result<(), RejectReason> { + let types = self.types_of("object", object_type)?; + let raw_id = identity_sql(self.dialect, id, schema, ROW_ALIAS, "id")?; + filters.push(format!("{} IS NOT NULL", types.sql)); + filters.push(format!("{raw_id} IS NOT NULL")); + filters.push(format!("{raw_id} <> ''")); + + // A change-tracked object stamps each observation with the row's own timestamp. A static + // one stamps a single observation at the Unix epoch. + let time_sql = match timestamp { + Some(ts) => { + let time = timestamp_sql(self.dialect, ts, schema, ROW_ALIAS)?; + filters.extend(time.filter(self.dialect)); + time.sql(self.dialect) + } + None => self.dialect.epoch_timestamp(), + }; + self.push_stale_type_probe(&types, from, &filters); + let id_sql = self.render_id(&types.sql, &raw_id); + + let values = self.attribute_values("object", &types, attributes, schema)?; + for (a, (declared, value_sql)) in attributes.iter().zip(&values) { + acc.object_attrs.push(ObjectAttrBranch { + type_sql: types.sql.clone(), + id_sql: id_sql.clone(), + time_sql: time_sql.clone(), + attribute: a.name.clone(), + value_sql: value_sql.clone(), + value_type: *declared, + from: from.to_string(), + filters: filters.clone(), + types: types.names.clone(), + }); + } + + if timestamp.is_none() && !attributes.is_empty() { + // Static attributes are written on this mapping's first row for an id and never + // again. SQL rows are unordered, so the views agree only when the repeats carry the + // same values. + acc.static_attr_probe_needed = true; + let projections: Vec = attributes + .iter() + .zip(&values) + .map(|(a, (_, value_sql))| { + format!("{value_sql} AS {}", self.dialect.quote_ident(&a.name)) + }) + .collect(); + let inner = format!( + "SELECT DISTINCT {id_sql} AS ocel_id, {} FROM {from} WHERE {}", + projections.join(", "), + filters.join(" AND ") + ); + self.probes.push(Probe { + mapping: Some(self.mapping_ref.clone()), + kind: ProbeKind::AmbiguousStaticObjectAttributes, + sql: format!( + "SELECT ocel_id FROM {} GROUP BY ocel_id HAVING COUNT(*) > 1", + self.dialect.derived_table(&inner, "static_attrs") + ), + }); + } + + acc.objects.push(ObjectBranch { + type_sql: types.sql, + id_sql, + from: from.to_string(), + filters, + types: types.names, + // This mapping's own filters are type/id/timestamp checks over its own row, never + // an `EXISTS` against another relation. + depends_on: BTreeSet::new(), + }); + Ok(()) + } + + /// An object endpoint: the rendered id, the type SQL, the `FROM` the split (if any) needs, + /// and the filters that keep exactly the parts the extractor keeps. + fn endpoint( + &mut self, + schema: &TableSchema, + from: &str, + endpoint: &ObjectEndpoint, + part_column: &str, + field: &'static str, + ) -> Result { + let raw = identity_sql(self.dialect, &endpoint.id, schema, ROW_ALIAS, field)?; + let type_sql = match &endpoint.object_type { + Some(e) => Some(identity_sql(self.dialect, e, schema, ROW_ALIAS, field)?), + None => None, + }; + if self.shape != EmissionShape::Consolidated { + if let Some(t) = &type_sql { + // Under `Create` and `PerType` this endpoint's type becomes a per-type view of + // its own. `Consolidated` has no such view to collide with. + let names = constant_names(t); + if names.is_empty() + && self.blueprint.on_missing_endpoint == MissingEndpointPolicy::Create + { + // `validate` asks only that the type is declared, so it may be read from + // the data. A created object would then reach `object` carrying a type no + // `object_` view and no `object_map_type` row names. + return Err(RejectReason::DynamicTypeName { + field, + detail: "the missing-endpoint policy creates this object, and a per-type \ + shape has no name for the view it would need" + .to_string(), + }); + } + check_reserved("object", &names)?; + } + } + let split = split_sql( + self.dialect, + from, + &raw, + endpoint.split.as_ref(), + part_column, + )?; + let mut id_filters = vec![format!("{raw} IS NOT NULL"), format!("{raw} <> ''")]; + id_filters.extend(split.filters); + let mut type_filters = Vec::new(); + let id_sql = self.render_endpoint_id(type_sql.as_deref(), &split.part, &mut type_filters); + Ok(Endpoint { + from: split.from, + id_sql, + type_sql, + id_filters, + type_filters, + }) + } + + /// An endpoint's id: the raw identity verbatim, or `-` under + /// [`IdRendering::TypePrefixed`], which needs the type to render and pushes the guard saying + /// so onto `type_filters`. + fn render_endpoint_id( + &self, + type_sql: Option<&str>, + raw_sql: &str, + type_filters: &mut Vec, + ) -> String { + match self.blueprint.id_rendering { + IdRendering::Raw => raw_sql.to_string(), + IdRendering::TypePrefixed => { + // `resolve_object_endpoint` drops the row outright when the type is None here. + let t = type_sql.map_or_else(|| self.dialect.null_text(), str::to_string); + type_filters.push(format!("{t} IS NOT NULL")); + self.dialect + .concat(&[t, self.dialect.string_literal("-"), raw_sql.to_string()]) + } + } + } + + /// The semi-join that drops a relation whose endpoint the extractor's own lookup would have + /// rejected, plus, under [`MissingEndpointPolicy::Create`], the object branch that creates it + /// instead. An endpoint that declares a type semi-joins on `(id, type)`, one that does not on + /// the id alone, matching `resolve_object`. + /// + /// `extra_depends` names the relations `extra_filters` already semi-joins against, so a + /// created object branch inherits them too. `create_from` is the `FROM` such a branch reads, + /// which is the endpoint's own unless `extra_filters` names a column only a later endpoint's + /// `FROM` carries. + fn endpoint_guard( + &self, + acc: &mut Accumulator, + endpoint: &Endpoint, + create_from: &str, + extra_filters: &[String], + extra_depends: &BTreeSet, + ) -> Option { + if self.blueprint.on_missing_endpoint == MissingEndpointPolicy::Create { + // `validate` guarantees a declared type under this policy, so the object always + // exists once it is created here. + if let Some(type_sql) = &endpoint.type_sql { + let mut filters = extra_filters.to_vec(); + filters.extend(endpoint.filters().cloned()); + filters.push(format!("{type_sql} IS NOT NULL")); + acc.objects.push(ObjectBranch { + type_sql: type_sql.clone(), + id_sql: endpoint.id_sql.clone(), + from: create_from.to_string(), + filters, + // Recorded so a constant type's view exists even with no entity mapping. + // `Self::endpoint` has already refused a dynamic one under a per-type shape. + types: constant_names(type_sql), + // An endpoint's own filters never carry an `EXISTS`, so the created branch + // depends on exactly what `extra_filters` already did. + depends_on: extra_depends.clone(), + }); + return None; + } + } + let (relation, id_column) = self.object_relation(); + Some(self.exists_guard( + relation, + id_column, + &endpoint.id_sql, + endpoint.type_sql.as_deref(), + )) + } + + /// The semi-join that keeps a row only when the relation carrying entity identity already + /// has this id, and, when the endpoint declares a type, only under that type. + fn exists_guard( + &self, + relation: &str, + id_column: &str, + id_sql: &str, + type_sql: Option<&str>, + ) -> String { + let type_column = self.dialect.quote_ident("ocel_type"); + let type_test = match type_sql { + Some(t) => format!(" AND ({t} IS NULL OR e.{type_column} = {t})"), + None => String::new(), + }; + format!( + "EXISTS (SELECT 1 FROM {} AS e WHERE e.{} = {id_sql}{type_test})", + self.dialect.quote_ident(relation), + self.dialect.quote_ident(id_column) + ) + } + + fn event_endpoint( + &mut self, + schema: &TableSchema, + endpoint: &EventEndpoint, + field: &'static str, + ) -> Result<(String, Vec), RejectReason> { + let raw = identity_sql(self.dialect, &endpoint.id, schema, ROW_ALIAS, field)?; + let type_sql = match &endpoint.event_type { + Some(e) => Some(identity_sql(self.dialect, e, schema, ROW_ALIAS, field)?), + None => None, + }; + let mut filters = vec![format!("{raw} IS NOT NULL"), format!("{raw} <> ''")]; + let id_sql = self.render_endpoint_id(type_sql.as_deref(), &raw, &mut filters); + let (relation, id_column) = self.event_relation(); + filters.push(self.exists_guard(relation, id_column, &id_sql, type_sql.as_deref())); + Ok((id_sql, filters)) + } + + fn compile_inline_object( + &mut self, + acc: &mut Accumulator, + schema: &TableSchema, + from: &str, + event_filters: &[String], + event_id_sql: &str, + inline: &InlineObjectRef, + ) -> Result<(), RejectReason> { + let endpoint = self.endpoint(schema, from, &inline.object, "__part0", "inline object")?; + let qualifier = self.qualifier_sql(inline.qualifier.as_ref(), schema)?; + let mut filters = event_filters.to_vec(); + filters.extend(endpoint.filters().cloned()); + // The event's own filters are type/id/timestamp checks over its own row, never an + // `EXISTS`, so this branch starts with no dependency. + let mut depends_on = BTreeSet::new(); + if let Some(guard) = + self.endpoint_guard(acc, &endpoint, &endpoint.from, event_filters, &depends_on) + { + filters.push(guard); + depends_on.insert(self.object_relation().0.to_string()); + } + acc.e2o.push(RelBranch { + left_sql: event_id_sql.to_string(), + right_sql: endpoint.id_sql, + qualifier_sql: qualifier, + from: endpoint.from, + filters, + depends_on, + }); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn compile_e2o( + &mut self, + acc: &mut Accumulator, + schema: &TableSchema, + from: &str, + mut filters: Vec, + event: &EventEndpoint, + object: &ObjectEndpoint, + qualifier: &Option, + ) -> Result<(), RejectReason> { + let (event_id, event_filters) = self.event_endpoint(schema, event, "event")?; + filters.extend(event_filters); + // `event_endpoint` always emits an `EXISTS` against the event relation, unconditionally + // of policy. + let mut depends_on: BTreeSet = + BTreeSet::from([self.event_relation().0.to_string()]); + let endpoint = self.endpoint(schema, from, object, "__part0", "object")?; + let qualifier_sql = self.qualifier_sql(qualifier.as_ref(), schema)?; + // The event is resolved first: an unresolved one drops the row before the object is + // even looked at, so a created object must inherit the event's filters too. + let created_filters = filters.clone(); + let created_depends = depends_on.clone(); + filters.extend(endpoint.filters().cloned()); + if let Some(guard) = self.endpoint_guard( + acc, + &endpoint, + &endpoint.from, + &created_filters, + &created_depends, + ) { + filters.push(guard); + depends_on.insert(self.object_relation().0.to_string()); + } + acc.e2o.push(RelBranch { + left_sql: event_id, + right_sql: endpoint.id_sql, + qualifier_sql, + from: endpoint.from, + filters, + depends_on, + }); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn compile_o2o( + &mut self, + acc: &mut Accumulator, + schema: &TableSchema, + from: &str, + mut filters: Vec, + source: &ObjectEndpoint, + target: &ObjectEndpoint, + qualifier: &Option, + ) -> Result<(), RejectReason> { + let src = self.endpoint(schema, from, source, "__part0", "source")?; + // Nesting the target's split over the source's reproduces the extractor's nested loops + // as a cross product. + let tgt = self.endpoint(schema, &src.from, target, "__part1", "target")?; + let qualifier_sql = self.qualifier_sql(qualifier.as_ref(), schema)?; + // Neither the mapping's own filters nor an endpoint's split filters ever carry an + // `EXISTS` of their own; only a guard below can add one. + let mut depends_on: BTreeSet = BTreeSet::new(); + // `run_o2o` returns before it resolves either endpoint when the target's id is absent + // or splits to nothing, so an object created for the source inherits the target's id + // guards, but not its type guard: the source is created before the target's type is ever + // rendered. The created branch reads the target's `FROM`, the only one those guards can be + // evaluated against. + let mut source_stage = filters.clone(); + source_stage.extend(tgt.id_filters.iter().cloned()); + filters.extend(src.filters().cloned()); + filters.extend(tgt.filters().cloned()); + if let Some(guard) = self.endpoint_guard(acc, &src, &tgt.from, &source_stage, &depends_on) { + filters.push(guard); + depends_on.insert(self.object_relation().0.to_string()); + } + if let Some(guard) = self.endpoint_guard(acc, &tgt, &tgt.from, &filters, &depends_on) { + filters.push(guard); + depends_on.insert(self.object_relation().0.to_string()); + } + acc.o2o.push(RelBranch { + left_sql: src.id_sql, + right_sql: tgt.id_sql, + qualifier_sql, + from: tgt.from, + filters, + depends_on, + }); + Ok(()) + } + + /// A missing qualifier is `unwrap_or_default()`, so `''`. A present one that evaluates to + /// `None` is the same. + fn qualifier_sql( + &self, + qualifier: Option<&ValueExpression>, + schema: &TableSchema, + ) -> Result { + match qualifier { + None => Ok(self.dialect.string_literal("")), + Some(e) => { + let sql = identity_sql(self.dialect, e, schema, ROW_ALIAS, "qualifier")?; + Ok(self + .dialect + .coalesce(&[sql, self.dialect.string_literal("")])) + } + } + } +} + +/// A resolved object endpoint. +#[derive(Debug)] +struct Endpoint { + from: String, + id_sql: String, + type_sql: Option, + /// The guards on the raw id and on the split parts. + id_filters: Vec, + /// The guard that the type renders, under [`IdRendering::TypePrefixed`] only. Kept apart from + /// [`Self::id_filters`] because `resolve_object_endpoint` applies it only after the id is in + /// hand, which a caller reproducing an earlier endpoint's state has to reproduce too. + type_filters: Vec, +} + +impl Endpoint { + /// Every guard, id before type, in the order `resolve_object_endpoint` applies them. + fn filters(&self) -> impl Iterator { + self.id_filters.iter().chain(&self.type_filters) + } +} + +/// Reject a type name whose per-type view name would collide with a relation the emitter defines, +/// e.g. `object_map_type` is `object_` plus a type literally named `map_type`. +fn check_reserved(kind: &'static str, names: &[String]) -> Result<(), RejectReason> { + for name in names { + let view = format!("{kind}_{name}"); + if RESERVED_RELATIONS.contains(&view.as_str()) { + return Err(RejectReason::ReservedTypeName { name: name.clone() }); + } + } + Ok(()) +} + +/// The single type name a type expression that compiled to a plain string literal denotes. +/// Anything else contributes no statically known name. +fn constant_names(type_sql: &str) -> Vec { + let trimmed = type_sql + .strip_prefix('\'') + .and_then(|s| s.strip_suffix('\'')); + match trimmed { + // A literal's embedded quotes arrive doubled (`string_literal`); a lone quote means + // this is not a plain string literal after all. + Some(inner) if !inner.replace("''", "").contains('\'') => { + vec![inner.replace("''", "'")] + } + _ => Vec::new(), + } +} + +fn select(projections: &[String], from: &str, filters: &[String]) -> String { + let mut sql = format!("SELECT {} FROM {from}", projections.join(", ")); + if !filters.is_empty() { + sql.push_str(&format!(" WHERE {}", filters.join(" AND "))); + } + sql +} + +/// A relation's body: `branches` deduplicated under `header`, or a typed empty relation when +/// there is no branch at all, so a downstream reference still type-checks. +fn union_body( + dialect: SqlDialect, + header: &[String], + empty: &[(&str, String)], + alias: &str, + branches: &[String], +) -> String { + if branches.is_empty() { + return empty_relation(dialect, empty); + } + dialect.distinct_select( + header, + &dialect.derived_table(&dialect.union_all(branches), alias), + ) +} + +/// The `(, ocel_type)` projection of every entity branch. +fn identity_projections(branches: &[B], id_column: &str) -> Vec { + branches + .iter() + .map(|b| { + select( + &[ + format!("{} AS {id_column}", b.id_sql()), + format!("{} AS ocel_type", b.type_sql()), + ], + b.from(), + b.filters(), + ) + }) + .collect() +} + +/// The two columns every identity relation starts with, as its header and as the typed `NULL`s +/// an empty one stands in with. +fn identity_columns(dialect: SqlDialect, id_column: &str) -> (Vec, Vec<(&str, String)>) { + ( + vec![id_column.to_string(), "ocel_type".to_string()], + vec![ + (id_column, dialect.null_text()), + ("ocel_type", dialect.null_text()), + ], + ) +} + +/// Every relation the branches semi-join against. +fn depends_of(branches: &[B]) -> BTreeSet { + branches + .iter() + .flat_map(|b| b.depends_on().iter().cloned()) + .collect() +} + +/// The probe saying two entities claim one id, which the extractor answers by dropping one and +/// the views by keeping both. +fn push_ambiguity_probe( + out: &mut CompiledOcel, + dialect: SqlDialect, + relation: &str, + id_column: &str, + kind: ProbeKind, +) { + out.probes.push(Probe { + mapping: None, + kind, + sql: format!( + "SELECT {id_column} FROM {} GROUP BY {id_column} HAVING COUNT(*) > 1", + dialect.quote_ident(relation) + ), + }); +} + +fn assemble(dialect: SqlDialect, attrs: &AttributePlan, acc: &Accumulator, out: &mut CompiledOcel) { + let mut event_types: BTreeSet = BTreeSet::new(); + for b in &acc.events { + event_types.extend(b.types.iter().cloned()); + } + let mut object_types: BTreeSet = BTreeSet::new(); + for b in &acc.objects { + object_types.extend(b.types.iter().cloned()); + } + + // Every view goes here with the relation names its own body semi-joins against, and is + // reordered into `out.views` by dependency once the whole set is known. + let mut pending: Vec<(ViewDef, BTreeSet)> = Vec::new(); + + let (header, empty) = identity_columns(dialect, "ocel_id"); + pending.push(( + ViewDef { + name: "object".to_string(), + body: union_body( + dialect, + &header, + &empty, + "object_union", + &identity_projections(&acc.objects, "ocel_id"), + ), + }, + depends_of(&acc.objects), + )); + if !acc.objects.is_empty() { + push_ambiguity_probe( + out, + dialect, + "object", + "ocel_id", + ProbeKind::AmbiguousObjectIdentity, + ); + } + + for t in &object_types { + // `object_type_view` always reads straight off `object`, so this dependency is fixed + // rather than gathered from a branch. + pending.push(( + ViewDef { + name: format!("object_{t}"), + body: object_type_view(dialect, attrs, acc, t), + }, + BTreeSet::from(["object".to_string()]), + )); + } + + // `Target::Event` never guards against another relation, so `event` never depends on + // anything. Only `object` can, through the `Create`-policy branches `compile_e2o` leaves + // depending on `event`. + pending.push(( + ViewDef { + name: "event".to_string(), + body: union_body( + dialect, + &header, + &empty, + "event_union", + &identity_projections(&acc.events, "ocel_id"), + ), + }, + BTreeSet::new(), + )); + if !acc.events.is_empty() { + push_ambiguity_probe( + out, + dialect, + "event", + "ocel_id", + ProbeKind::AmbiguousEventIdentity, + ); + } + + for t in &event_types { + let (body, has_rows) = event_type_view(dialect, attrs, acc, t); + pending.push(( + ViewDef { + name: format!("event_{t}"), + body, + }, + BTreeSet::new(), + )); + if has_rows { + push_ambiguity_probe( + out, + dialect, + &format!("event_{t}"), + "ocel_id", + ProbeKind::AmbiguousEventIdentity, + ); + } + } + + pending.push(( + ViewDef { + name: "event_map_type".to_string(), + body: type_map(dialect, &event_types), + }, + BTreeSet::new(), + )); + pending.push(( + ViewDef { + name: "object_map_type".to_string(), + body: type_map(dialect, &object_types), + }, + BTreeSet::new(), + )); + + pending.push(( + ViewDef { + name: "event_object".to_string(), + body: relation_view( + dialect, + &acc.e2o, + ("ocel_event_id", "ocel_object_id"), + "ocel_qualifier", + ), + }, + depends_of(&acc.e2o), + )); + pending.push(( + ViewDef { + name: "object_object".to_string(), + body: relation_view( + dialect, + &acc.o2o, + ("ocel_source_id", "ocel_target_id"), + "ocel_qualifier", + ), + }, + depends_of(&acc.o2o), + )); + + out.views = order_views(pending, &mut out.errors); +} + +/// Assemble [`EmissionShape::Consolidated`]'s six relations (`events`, `objects`, +/// `object_attribute_changes`, `e2o`, `o2o` and `event_attr_meta`) from the same `Accumulator` +/// [`assemble`] reads for [`EmissionShape::PerType`]. Every branch was already compiled +/// shape-aware, so this only reshapes already-correct SQL. +#[allow(clippy::too_many_lines)] +fn assemble_consolidated( + dialect: SqlDialect, + attrs: &AttributePlan, + acc: &Accumulator, + out: &mut CompiledOcel, +) { + let mut event_types: BTreeSet = BTreeSet::new(); + for b in &acc.events { + event_types.extend(b.types.iter().cloned()); + } + + let mut pending: Vec<(ViewDef, BTreeSet)> = Vec::new(); + + // `objects(id, ocel_type)`. + let (header, empty) = identity_columns(dialect, "id"); + pending.push(( + ViewDef { + name: "objects".to_string(), + body: union_body( + dialect, + &header, + &empty, + "object_union", + &identity_projections(&acc.objects, "id"), + ), + }, + depends_of(&acc.objects), + )); + if !acc.objects.is_empty() { + push_ambiguity_probe( + out, + dialect, + "objects", + "id", + ProbeKind::AmbiguousObjectIdentity, + ); + } + + // `events(id, ocel_type, "time", )`, wide across + // every event type at once, unlike `PerType`'s bare `event`, which carries no attributes. + let wide_cols = attrs.event_columns_wide(); + let time_col = dialect.quote_ident("time"); + let mut events_header = vec!["id".to_string(), "ocel_type".to_string(), time_col.clone()]; + events_header.extend(wide_cols.iter().map(|(n, _)| dialect.quote_ident(n))); + + let mut events_empty: Vec<(&str, String)> = vec![ + ("id", dialect.null_text()), + ("ocel_type", dialect.null_text()), + ("time", dialect.null_timestamp()), + ]; + let owned: Vec<(String, String)> = wide_cols + .iter() + .map(|(n, t)| (n.clone(), dialect.null_attribute(*t))) + .collect(); + events_empty.extend(owned.iter().map(|(n, v)| (n.as_str(), v.clone()))); + + let event_branches: Vec = acc + .events + .iter() + .map(|b| { + let mut projections = vec![ + format!("{} AS id", b.id_sql), + format!("{} AS ocel_type", b.type_sql), + format!("{} AS {time_col}", b.time_sql), + ]; + for (name, target_ty) in &wide_cols { + let quoted = dialect.quote_ident(name); + // A branch that never declared `name` at all contributes a typed `NULL`, + // matching a `UNION ALL` branch missing a column elsewhere in this module. + let value = match (b.attributes.get(name), b.attribute_types.get(name)) { + (Some(sql), Some(&from_ty)) => { + coerce_attr_sql(dialect, sql, from_ty, *target_ty) + } + _ => dialect.null_attribute(*target_ty), + }; + projections.push(format!("{value} AS {quoted}")); + } + select(&projections, &b.from, &b.filters) + }) + .collect(); + pending.push(( + ViewDef { + name: "events".to_string(), + body: union_body( + dialect, + &events_header, + &events_empty, + "event_union", + &event_branches, + ), + }, + BTreeSet::new(), + )); + if !acc.events.is_empty() { + push_ambiguity_probe( + out, + dialect, + "events", + "id", + ProbeKind::AmbiguousEventIdentity, + ); + } + + // `event_attr_meta(event_type, attr_name, attr_type)`, best-effort: only a statically known + // event type contributes a row. A dynamically-typed mapping's attribute values still land in + // `events`, only this metadata is incomplete for it. + let mut meta_rows: Vec = Vec::new(); + for t in &event_types { + for (name, ty) in attrs.columns_of("event", t) { + meta_rows.push(format!( + "SELECT {} AS event_type, {} AS attr_name, {} AS attr_type", + dialect.string_literal(t), + dialect.string_literal(&name), + dialect.string_literal(ty.as_type_str()) + )); + } + } + let event_attr_meta_body = if meta_rows.is_empty() { + empty_relation( + dialect, + &[ + ("event_type", dialect.null_text()), + ("attr_name", dialect.null_text()), + ("attr_type", dialect.null_text()), + ], + ) + } else { + dialect.union_all(&meta_rows) + }; + pending.push(( + ViewDef { + name: "event_attr_meta".to_string(), + body: event_attr_meta_body, + }, + BTreeSet::new(), + )); + + // `object_attribute_changes(id, name, "time", value, value_type)`, one row per attribute + // observation, exactly `acc.object_attrs`. Unlike `PerType`'s `object_` there is no + // existence row: `objects` alone already carries identity. + let value_col = dialect.quote_ident("value"); + let value_type_col = dialect.quote_ident("value_type"); + let name_col = dialect.quote_ident("name"); + let object_attr_branches: Vec = acc + .object_attrs + .iter() + .map(|b| { + let text = attr_value_as_text(dialect, &b.value_sql, b.value_type); + let empty = dialect.string_literal(""); + // A `NULL` cell is a recorded observation of `Null`, not an absent attribute. + // `'null'` is outside `OCELAttributeType::as_type_str`'s outputs, so `from_sql_value` + // reconstructs `OCELAttributeValue::Null` regardless of `value`, never conflating a + // `Null` observation with an empty-string one. + let value_type_sql = format!( + "CASE WHEN {} IS NULL THEN {} ELSE {} END", + b.value_sql, + dialect.string_literal("null"), + dialect.string_literal(b.value_type.as_type_str()) + ); + let projections = vec![ + format!("{} AS id", b.id_sql), + format!("{} AS {name_col}", dialect.string_literal(&b.attribute)), + format!("{} AS {time_col}", b.time_sql), + format!("COALESCE({text}, {empty}) AS {value_col}"), + format!("{value_type_sql} AS {value_type_col}"), + ]; + select(&projections, &b.from, &b.filters) + }) + .collect(); + pending.push(( + ViewDef { + name: "object_attribute_changes".to_string(), + body: union_body( + dialect, + &[ + "id".to_string(), + name_col, + time_col, + value_col, + value_type_col, + ], + &[ + ("id", dialect.null_text()), + ("name", dialect.null_text()), + ("time", dialect.null_timestamp()), + ("value", dialect.null_text()), + ("value_type", dialect.null_text()), + ], + "object_attr_union", + &object_attr_branches, + ), + }, + BTreeSet::new(), + )); + + // `e2o(event_id, object_id, qualifier)` / `o2o(source_id, target_id, qualifier)`. + pending.push(( + ViewDef { + name: "e2o".to_string(), + body: relation_view(dialect, &acc.e2o, ("event_id", "object_id"), "qualifier"), + }, + depends_of(&acc.e2o), + )); + pending.push(( + ViewDef { + name: "o2o".to_string(), + body: relation_view(dialect, &acc.o2o, ("source_id", "target_id"), "qualifier"), + }, + depends_of(&acc.o2o), + )); + + out.views = order_views(pending, &mut out.errors); +} + +/// Reorder `pending` so every view comes after each view named in its own dependency set, using +/// Kahn's algorithm in the same style as the cycle sweep `validate.rs` runs over the node graph, +/// but building an order instead of only detecting a cycle. +/// +/// A cycle cannot arise from the emission rules alone. Only a blueprint that skipped +/// [`validate`](super::validate) can reach one, by omitting the endpoint type declaration +/// [`MissingEndpointPolicy::Create`] requires and so making `object` depend on itself. +/// Unorderable views are reported as [`RejectReason::ViewCycle`] rather than emitted out of order. +fn order_views( + pending: Vec<(ViewDef, BTreeSet)>, + errors: &mut Vec, +) -> Vec { + let mut remaining = pending; + let mut ordered: Vec = Vec::with_capacity(remaining.len()); + let mut ordered_names: BTreeSet = BTreeSet::new(); + loop { + let (ready, blocked): (Vec<_>, Vec<_>) = remaining + .into_iter() + .partition(|(_, deps)| deps.iter().all(|d| ordered_names.contains(d))); + if ready.is_empty() { + remaining = blocked; + break; + } + for (view, _) in ready { + ordered_names.insert(view.name.clone()); + ordered.push(view); + } + remaining = blocked; + } + for (view, _) in remaining { + errors.push(CompileError { + mapping: None, + reason: RejectReason::ViewCycle { view: view.name }, + }); + } + ordered +} + +/// `event_(ocel_id, ocel_time, )`, returning whether any +/// branch actually contributes rows. +fn event_type_view( + dialect: SqlDialect, + attrs: &AttributePlan, + acc: &Accumulator, + type_name: &str, +) -> (String, bool) { + let columns = attrs.columns_of("event", type_name); + let mut header: Vec = vec!["ocel_id".to_string(), "ocel_time".to_string()]; + header.extend(columns.iter().map(|(n, _)| dialect.quote_ident(n))); + + let branches: Vec = acc + .events + .iter() + .filter(|b| b.types.iter().any(|t| t == type_name)) + .map(|b| { + let mut projections = vec![ + format!("{} AS ocel_id", b.id_sql), + format!("{} AS ocel_time", b.time_sql), + ]; + for (name, ty) in &columns { + let quoted = dialect.quote_ident(name); + let value = b + .attributes + .get(name) + .cloned() + .unwrap_or_else(|| dialect.null_attribute(*ty)); + projections.push(format!("{value} AS {quoted}")); + } + let mut filters = b.filters.clone(); + // A branch whose type is read from a column contributes to several views. + filters.push(format!( + "{} = {}", + b.type_sql, + dialect.string_literal(type_name) + )); + select(&projections, &b.from, &filters) + }) + .collect(); + + let mut empty: Vec<(&str, String)> = vec![ + ("ocel_id", dialect.null_text()), + ("ocel_time", dialect.null_timestamp()), + ]; + let owned: Vec<(String, String)> = columns + .iter() + .map(|(n, t)| (n.clone(), dialect.null_attribute(*t))) + .collect(); + empty.extend(owned.iter().map(|(n, v)| (n.as_str(), v.clone()))); + + let has_rows = !branches.is_empty(); + ( + union_body(dialect, &header, &empty, "event_type_union", &branches), + has_rows, + ) +} + +/// `object_(ocel_id, ocel_time, ocel_changed_field, )`. +/// +/// Every attribute observation is a row naming its attribute in `ocel_changed_field`, static and +/// change-tracked alike, plus one `ocel_changed_field IS NULL` row per object carrying its +/// existence. An OCEL 2.0 exporter would instead put a static object's values on the `NULL` row, +/// conflating "observed as `Null`" with "never declared", a distinction the extractor draws. +fn object_type_view( + dialect: SqlDialect, + attrs: &AttributePlan, + acc: &Accumulator, + type_name: &str, +) -> String { + let columns = attrs.columns_of("object", type_name); + let mut header: Vec = vec![ + "ocel_id".to_string(), + "ocel_time".to_string(), + "ocel_changed_field".to_string(), + ]; + header.extend(columns.iter().map(|(n, _)| dialect.quote_ident(n))); + + let mut projections = vec![ + "ocel_id".to_string(), + format!("{} AS ocel_time", dialect.epoch_timestamp()), + format!("{} AS ocel_changed_field", dialect.null_text()), + ]; + for (name, ty) in &columns { + projections.push(format!( + "{} AS {}", + dialect.null_attribute(*ty), + dialect.quote_ident(name) + )); + } + let mut branches = vec![select( + &projections, + &dialect.quote_ident("object"), + &[format!("ocel_type = {}", dialect.string_literal(type_name))], + )]; + + for b in acc + .object_attrs + .iter() + .filter(|b| b.types.iter().any(|t| t == type_name)) + { + let mut projections = vec![ + format!("{} AS ocel_id", b.id_sql), + format!("{} AS ocel_time", b.time_sql), + format!( + "{} AS ocel_changed_field", + dialect.string_literal(&b.attribute) + ), + ]; + for (name, ty) in &columns { + let value = if *name == b.attribute { + b.value_sql.clone() + } else { + dialect.null_attribute(*ty) + }; + projections.push(format!("{value} AS {}", dialect.quote_ident(name))); + } + let mut filters = b.filters.clone(); + filters.push(format!( + "{} = {}", + b.type_sql, + dialect.string_literal(type_name) + )); + branches.push(select(&projections, &b.from, &filters)); + } + + dialect.distinct_select( + &header, + &dialect.derived_table(&dialect.union_all(&branches), "object_type_union"), + ) +} + +fn relation_view( + dialect: SqlDialect, + rows: &[RelBranch], + cols: (&str, &str), + qualifier_col: &str, +) -> String { + let (left, right) = cols; + let (left_col, right_col) = (dialect.quote_ident(left), dialect.quote_ident(right)); + let qualifier = dialect.quote_ident(qualifier_col); + let branches: Vec = rows + .iter() + .map(|r| { + select( + &[ + format!("{} AS {left_col}", r.left_sql), + format!("{} AS {right_col}", r.right_sql), + format!("{} AS {qualifier}", r.qualifier_sql), + ], + &r.from, + &r.filters, + ) + }) + .collect(); + union_body( + dialect, + &[left_col, right_col, qualifier], + &[ + (left, dialect.null_text()), + (right, dialect.null_text()), + (qualifier_col, dialect.null_text()), + ], + "rel_union", + &branches, + ) +} + +fn type_map(dialect: SqlDialect, types: &BTreeSet) -> String { + if types.is_empty() { + return empty_relation( + dialect, + &[ + ("ocel_type", dialect.null_text()), + ("ocel_type_map", dialect.null_text()), + ], + ); + } + let rows: Vec = types + .iter() + .map(|t| { + format!( + "SELECT {0} AS ocel_type, {0} AS ocel_type_map", + dialect.string_literal(t) + ) + }) + .collect(); + dialect.union_all(&rows) +} + +/// An always-empty relation with typed placeholder columns, so a downstream reference still +/// type-checks. +fn empty_relation(dialect: SqlDialect, cols: &[(&str, String)]) -> String { + let projections: Vec = cols + .iter() + .map(|(name, null_literal)| format!("{null_literal} AS {}", dialect.quote_ident(name))) + .collect(); + format!( + "SELECT {} WHERE {}", + projections.join(", "), + dialect.false_predicate() + ) +} + +/// Render a `declared`-typed SQL expression as the text `to_sql_value` would produce, so that +/// `from_sql_value`, which +/// `DuckDbLinkedOCEL`'s +/// reader uses, parses it back to the identical value. +/// +/// `Integer`/`Float` use the engine's round-trip-safe `CAST(.. AS VARCHAR)`, which need not match +/// Rust's `Display` digit-for-digit, only parse back to the identical value. +fn attr_value_as_text(dialect: SqlDialect, expr: &str, declared: OCELAttributeType) -> String { + use OCELAttributeType as A; + match declared { + A::String | A::Null => expr.to_string(), + A::Integer | A::Float => dialect.cast_to_text(expr), + A::Boolean => dialect.bool_to_text(expr), + A::Time => dialect.timestamptz_to_iso_text(expr), + } +} + +/// Coerce a `from`-typed SQL expression to `to`, mirroring [`OCELAttributeType::coalesce`]: +/// identical types are a no-op, `Integer` widens into `Float`, anything else widens into `String` +/// via [`attr_value_as_text`]. Used for [`EmissionShape::Consolidated`]'s wide `events` columns, +/// which may hold values several event types declared under different-but-reconciled types. +fn coerce_attr_sql( + dialect: SqlDialect, + expr: &str, + from: OCELAttributeType, + to: OCELAttributeType, +) -> String { + use OCELAttributeType as A; + if from == to { + return expr.to_string(); + } + if from == A::Integer && to == A::Float { + return format!("CAST({expr} AS DOUBLE)"); + } + debug_assert_eq!( + to, + A::String, + "OCELAttributeType::coalesce widens any mismatch other than Integer/Float to String" + ); + attr_value_as_text(dialect, expr, from) +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/compile/dialect.rs b/process_mining/src/core/event_data/object_centric/extraction/compile/dialect.rs new file mode 100644 index 00000000..2409787c --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/compile/dialect.rs @@ -0,0 +1,350 @@ +//! The one place a SQL string is shaped. +//! +//! Every fragment the emitter produces goes through [`SqlDialect`], so adding a second engine is +//! a matter of adding match arms here rather than hunting `format!` calls through the compiler. +//! Nothing outside this module writes a quote, a cast or a function name by hand. + +use chrono::{DateTime, FixedOffset}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::core::event_data::object_centric::extraction::value::{Value, ValueKind}; +use crate::core::event_data::object_centric::OCELAttributeType; + +/// Which SQL engine the emitted statements target. +/// +/// The two are not equally evidenced: `DuckDb` is checked row-for-row against the extractor by +/// the differential suite, while `Postgres` is only covered by unit tests over the emitted text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[non_exhaustive] +pub enum SqlDialect { + /// `DuckDB`, the engine the differential tests run against. + #[default] + DuckDb, + /// `PostgreSQL` 12+. Emitted from the same fragments as `DuckDb`, see the type-level note. + Postgres, +} + +impl SqlDialect { + /// Quote an identifier, doubling any embedded quote character. + #[must_use] + pub fn quote_ident(self, name: &str) -> String { + match self { + // SQL-standard double quoting in both. + SqlDialect::DuckDb | SqlDialect::Postgres => { + format!("\"{}\"", name.replace('"', "\"\"")) + } + } + } + + /// Quote a text value as a SQL string literal, doubling any embedded apostrophe. + #[must_use] + pub fn string_literal(self, value: &str) -> String { + match self { + SqlDialect::DuckDb | SqlDialect::Postgres => format!("'{}'", value.replace('\'', "''")), + } + } + + /// `CREATE VIEW AS\n`. `name` is always the bare view name, never pre-quoted. + #[must_use] + pub fn create_view(self, name: &str, body: &str) -> String { + format!("CREATE VIEW {} AS\n{body}", self.quote_ident(name)) + } + + /// `CREATE TABLE AS\n`. See [`Self::create_view`]. + #[must_use] + pub fn create_table_as(self, name: &str, body: &str) -> String { + format!("CREATE TABLE {} AS\n{body}", self.quote_ident(name)) + } + + /// What terminates one statement in a multi-statement script. + #[must_use] + pub fn statement_separator(self) -> &'static str { + ";" + } + + /// A parenthesized subquery in `FROM` position, always aliased: `PostgreSQL` and `MySQL` both + /// reject an unaliased derived table, and `DuckDB` merely tolerates it. + pub(crate) fn derived_table(self, sql: &str, alias: &str) -> String { + format!("(\n{sql}\n) AS {alias}") + } + + /// An always-true predicate. + pub(crate) fn true_predicate(self) -> &'static str { + "TRUE" + } + + /// An always-false predicate. + pub(crate) fn false_predicate(self) -> &'static str { + "FALSE" + } + + /// Concatenate already-rendered `SELECT` branches into one `UNION ALL` chain. + /// + /// Never plain `UNION`: deduplicating here would drop rows the extractor keeps, and each + /// relation applies its own `DISTINCT` where the extractor's own identity rules call for one. + pub(crate) fn union_all(self, selects: &[String]) -> String { + selects.join("\n UNION ALL\n") + } + + /// `SELECT DISTINCT FROM `, where `from` is an already-rendered `FROM` target. + pub(crate) fn distinct_select(self, cols: &[String], from: &str) -> String { + format!("SELECT DISTINCT {} FROM {from}", cols.join(", ")) + } + + /// Concatenate already-rendered expression fragments with the dialect's text concatenation + /// operator, parenthesized as one expression. + /// + /// `NULL`-propagating on purpose: a `Template` whose placeholder is `NULL` evaluates to + /// `None` in the extractor, and `||` reproduces that without a per-part fallback. + pub(crate) fn concat(self, parts: &[String]) -> String { + match self { + // `||` is standard and means text concatenation in both. + SqlDialect::DuckDb | SqlDialect::Postgres => format!("({})", parts.join(" || ")), + } + } + + /// `COALESCE()`. + pub(crate) fn coalesce(self, parts: &[String]) -> String { + format!("COALESCE({})", parts.join(", ")) + } + + /// Force a possibly-`NULL` boolean expression to `FALSE`. + /// + /// The extractor evaluates predicates in two-valued logic, so `NOT (col = 'x')` is true there + /// when `col` is `NULL` while SQL makes it `NULL`. Every compiled predicate goes through this + /// so `NOT` composes over a total boolean. + pub(crate) fn total_bool(self, expr: &str) -> String { + format!("COALESCE({expr}, {})", self.false_predicate()) + } + + /// `CAST( AS )`. + pub(crate) fn cast_to_text(self, expr: &str) -> String { + match self { + SqlDialect::DuckDb => format!("CAST({expr} AS VARCHAR)"), + SqlDialect::Postgres => format!("CAST({expr} AS TEXT)"), + } + } + + /// `CAST(NULL AS )`, for a placeholder column in an empty relation. + pub(crate) fn null_text(self) -> String { + self.cast_to_text("NULL") + } + + /// `CAST(NULL AS )`. See [`Self::null_text`]. + pub(crate) fn null_timestamp(self) -> String { + match self { + SqlDialect::DuckDb | SqlDialect::Postgres => "CAST(NULL AS TIMESTAMPTZ)".to_string(), + } + } + + /// A boolean column at an identity position, rendered exactly as + /// [`Value::canonical_string`] renders it, rather than trusting the engine's own + /// boolean-to-text cast. + /// + /// `NULL`-propagating: a `CASE WHEN {expr} THEN .. ELSE .. END` would take the `ELSE` branch + /// for `NULL` and mint the literal id `false`, past the caller's `IS NOT NULL` and `<> ''` + /// guards. The `CASE WHEN ..` form also names `expr` only once. + pub(crate) fn bool_to_text(self, expr: &str) -> String { + format!("CASE {expr} WHEN TRUE THEN 'true' WHEN FALSE THEN 'false' END") + } + + /// The column type an event/object attribute of `t` is stored as. + pub(crate) fn attribute_sql_type(self, t: OCELAttributeType) -> &'static str { + match self { + SqlDialect::DuckDb => match t { + OCELAttributeType::Integer => "BIGINT", + OCELAttributeType::Float => "DOUBLE", + OCELAttributeType::Boolean => "BOOLEAN", + OCELAttributeType::Time => "TIMESTAMPTZ", + OCELAttributeType::String | OCELAttributeType::Null => "VARCHAR", + }, + SqlDialect::Postgres => match t { + OCELAttributeType::Integer => "BIGINT", + OCELAttributeType::Float => "DOUBLE PRECISION", + OCELAttributeType::Boolean => "BOOLEAN", + OCELAttributeType::Time => "TIMESTAMPTZ", + OCELAttributeType::String | OCELAttributeType::Null => "TEXT", + }, + } + } + + /// `CAST(NULL AS )`, so a `UNION ALL` branch that does not carry an + /// attribute still contributes a column of the right type. + pub(crate) fn null_attribute(self, t: OCELAttributeType) -> String { + format!("CAST(NULL AS {})", self.attribute_sql_type(t)) + } + + /// An instant literal, always with an explicit UTC offset so no session time zone can + /// reinterpret it. + pub(crate) fn timestamp_literal(self, ts: &DateTime) -> String { + match self { + SqlDialect::DuckDb | SqlDialect::Postgres => { + format!("CAST('{}' AS TIMESTAMPTZ)", ts.to_utc().to_rfc3339()) + } + } + } + + /// The Unix epoch, the instant a static object attribute is stamped with. + pub(crate) fn epoch_timestamp(self) -> String { + match self { + SqlDialect::DuckDb | SqlDialect::Postgres => { + "CAST('1970-01-01T00:00:00+00:00' AS TIMESTAMPTZ)".to_string() + } + } + } + + /// Read a timestamp column as an absolute instant. + /// + /// `naive` says the column has no offset of its own (`TIMESTAMP`, `DATE`, `DATETIME`), in which + /// case it is anchored to UTC explicitly: the extractor's providers read such a column at UTC, + /// and a session-dependent reading would silently shift it. + pub(crate) fn timestamp_column(self, expr: &str, naive: bool) -> String { + match self { + // `timezone(zone, timestamp) -> timestamptz` is spelled and typed the same in both. + SqlDialect::DuckDb | SqlDialect::Postgres => { + if naive { + format!("timezone('UTC', CAST({expr} AS TIMESTAMP))") + } else { + format!("CAST({expr} AS TIMESTAMPTZ)") + } + } + } + } + + /// A `TIMESTAMPTZ` expression rendered as RFC 3339 text (UTC, `Z` suffix), independent of the + /// session time zone. + /// + /// `strftime` alone formats in the session's local time zone, so `timezone('UTC', ..)` converts + /// to a naive `TIMESTAMP` holding the UTC reading first. The microsecond field is always + /// emitted so the text round-trips exactly through + /// [`parse_timestamp`](crate::core::event_data::timestamp_utils::parse_timestamp). + pub(crate) fn timestamptz_to_iso_text(self, expr: &str) -> String { + match self { + SqlDialect::DuckDb => { + format!("strftime(timezone('UTC', {expr}), '%Y-%m-%dT%H:%M:%S.%fZ')") + } + // `timezone('UTC', timestamptz)` yields the naive UTC reading, which `to_char` renders + // verbatim; `US` is microseconds, six digits zero-padded, matching `%f`. + SqlDialect::Postgres => { + format!("to_char(timezone('UTC', {expr}), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')") + } + } + } + + /// Strip surrounding whitespace. See the dialect note in the [module docs](super). + pub(crate) fn trim(self, expr: &str) -> String { + format!("trim({expr})") + } + + /// One row per delimiter-separated part of `expr`. + pub(crate) fn split_to_rows(self, expr: &str, delimiter: &str) -> String { + match self { + SqlDialect::DuckDb => format!( + "unnest(string_split({expr}, {}))", + self.string_literal(delimiter) + ), + SqlDialect::Postgres => format!( + "unnest(string_to_array({expr}, {}))", + self.string_literal(delimiter) + ), + } + } + + /// One row per value a regular expression extracts from `expr`: every capture group of every + /// match when `groups` is non-zero, each whole match otherwise. + /// + /// Mirrors [`PreparedSplit::split`](crate::core::event_data::object_centric::extraction::expr::SplitSpec)'s + /// regex branch. The emission order differs from Rust's, which does not matter: every + /// relation this feeds is `DISTINCT` and compared as a set. + pub(crate) fn regex_split_to_rows(self, expr: &str, pattern: &str, groups: usize) -> String { + match self { + SqlDialect::DuckDb => { + let pat = self.string_literal(pattern); + if groups == 0 { + return format!("unnest(regexp_extract_all({expr}, {pat}))"); + } + // `list_concat` takes exactly two lists, so several groups fold into nested calls. + let group_list = |i: usize| format!("regexp_extract_all({expr}, {pat}, {i})"); + let mut list = group_list(groups); + for i in (1..groups).rev() { + list = format!("list_concat({}, {list})", group_list(i)); + } + format!("unnest({list})") + } + // `regexp_matches(.., 'g')` yields one `text[]` per match: the whole match without + // groups, the capture groups otherwise, matching what DuckDB's branch produces. + // The `ARRAY(..)` keeps the expression set-returning: a bare `(SELECT .. FROM ..)` in + // a `SELECT` list is a scalar subquery and errors on more than one match. + SqlDialect::Postgres => { + let pat = self.string_literal(pattern); + let m = format!("regexp_matches({expr}, {pat}, 'g')"); + if groups == 0 { + format!("unnest(ARRAY(SELECT g[1] FROM {m} AS g))") + } else { + format!("unnest(ARRAY(SELECT unnest(g) FROM {m} AS g))") + } + } + } + } + + /// Whether a text expression matches a regular expression, unanchored. + pub(crate) fn regex_match(self, expr: &str, pattern: &str) -> String { + match self { + SqlDialect::DuckDb => { + format!("regexp_matches({expr}, {})", self.string_literal(pattern)) + } + // `~`, not `regexp_matches`: in PostgreSQL that name is the set-returning function + // above, not a predicate. + SqlDialect::Postgres => format!("({expr} ~ {})", self.string_literal(pattern)), + } + } + + /// A [`Value`] as a typed SQL literal. `None` for [`Value::Null`], which has no literal + /// form the compiler ever needs. + pub(crate) fn value_literal(self, v: &Value) -> Option { + match v { + Value::Null => None, + Value::Text(s) => Some(self.string_literal(s)), + Value::Integer(i) => Some(format!("CAST({i} AS BIGINT)")), + Value::Float(f) => Some({ + let ty = match self { + SqlDialect::DuckDb => "DOUBLE", + SqlDialect::Postgres => "DOUBLE PRECISION", + }; + if f.is_nan() { + format!("CAST('NaN' AS {ty})") + } else if f.is_infinite() { + let sign = if *f < 0.0 { "-" } else { "" }; + format!("CAST('{sign}Infinity' AS {ty})") + } else { + // `{f:?}` is the shortest representation that round-trips. + format!("CAST('{f:?}' AS {ty})") + } + }), + Value::Boolean(b) => Some(if *b { "TRUE" } else { "FALSE" }.to_string()), + Value::Timestamp(ts) => Some(self.timestamp_literal(ts)), + } + } + + /// The SQL type a column of `kind` is read as, used to give an otherwise untyped `NULL` a + /// type in a `UNION ALL` branch. + pub(crate) fn kind_sql_type(self, kind: ValueKind) -> &'static str { + match self { + SqlDialect::DuckDb => match kind { + ValueKind::Text => "VARCHAR", + ValueKind::Integer => "BIGINT", + ValueKind::Float => "DOUBLE", + ValueKind::Boolean => "BOOLEAN", + ValueKind::Timestamp => "TIMESTAMPTZ", + }, + SqlDialect::Postgres => match kind { + ValueKind::Text => "TEXT", + ValueKind::Integer => "BIGINT", + ValueKind::Float => "DOUBLE PRECISION", + ValueKind::Boolean => "BOOLEAN", + ValueKind::Timestamp => "TIMESTAMPTZ", + }, + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/compile/differential.rs b/process_mining/src/core/event_data/object_centric/extraction/compile/differential.rs new file mode 100644 index 00000000..f72caa50 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/compile/differential.rs @@ -0,0 +1,2835 @@ +//! The differential harness: extract, compile, run the SQL, and compare the two logs. +//! +//! `compile::tests` checks the shape of emitted SQL; these tests check that the SQL means the same +//! thing the extractor does. +//! +//! Both sides are reduced to the same [`LogSets`] and compared field by field. Values are rendered +//! through one kind-tagged function used on both sides, so a `Text` `"1"` cannot pass for an +//! `Integer` `1`. The comparison is on sets, not multisets: every relation view is `DISTINCT`, +//! since the extractor's identity rules make a repeated `(event, object, qualifier)` +//! indistinguishable from a single one. +//! +//! The extractor runs against the fixture through `DuckDbRowProvider`, a test-only [`RowProvider`] +//! over the same connection the compiled SQL is executed on, so a divergence can only come from +//! the compiler. This also satisfies the compiler's catalog precondition, since `DuckDB` types its +//! columns rather than its cells. +//! +//! Under `IdRendering::Raw` two entities of different types can claim one id, where the extractor +//! reports `IdTypeCollision` and drops the loser while the compiled views keep both. +//! `assert_agrees` asserts the report contains no `IdTypeCollision` before comparing, so `Raw` +//! blueprints stay testable and a fixture that grows a collision fails with a named reason. +#![cfg(all(test, feature = "ocel-duckdb"))] + +use std::collections::{BTreeSet, HashMap}; +use std::ops::ControlFlow; + +use chrono::{DateTime, FixedOffset}; +use duckdb::types::ValueRef; +use duckdb::Connection; + +use super::{compile, CompiledOcel, EmissionShape, ProbeKind, RejectReason, SqlDialect}; +use crate::core::event_data::object_centric::extraction::blueprint::{ + Blueprint, DuplicateObjectPolicy, EventEndpoint, IdRendering, InlineObjectRef, Mapping, + MappingEntry, MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Target, +}; +use crate::core::event_data::object_centric::extraction::catalog::{ + ExtractionCatalog, TableSchema, +}; +use crate::core::event_data::object_centric::extraction::expr::{ + AttributeMapping, SplitKind, SplitSpec, TimestampSource, ValueExpression, +}; +use crate::core::event_data::object_centric::extraction::extract::extract; +use crate::core::event_data::object_centric::extraction::predicate::{ + CompareOp, Literal, Operand, Predicate, +}; +use crate::core::event_data::object_centric::extraction::provider::{ProviderError, RowProvider}; +use crate::core::event_data::object_centric::extraction::report::{ + ExtractionError, ExtractionReport, +}; +use crate::core::event_data::object_centric::extraction::slim_sink::SlimOcelSink; +use crate::core::event_data::object_centric::extraction::validate::validate; +use crate::core::event_data::object_centric::extraction::value::Value; +use crate::core::event_data::object_centric::ocel_sql::duckdb::schema::value::from_sql_value; +use crate::core::event_data::object_centric::readable::ReadableOCEL; +use crate::core::event_data::object_centric::{OCELAttributeType, OCELAttributeValue}; + +/// A `DuckDB` database in its own temporary directory. Every test gets its own, so a shared path +/// cannot corrupt under a concurrent run. +struct Fixture { + _dir: tempfile::TempDir, + con: Connection, +} + +impl Fixture { + /// Create the database and run `setup` (a `CREATE TABLE`/`INSERT` script) against it, with + /// the session time zone pinned to UTC. + /// + /// Pinning it makes every other test here deterministic, but also blind to a fragment that + /// reads a naive `TIMESTAMP`/`DATE` in the session's zone rather than anchoring it, since + /// under UTC the two readings coincide. Hence [`Self::with_time_zone`]. + fn new(setup: &str) -> Self { + Self::with_time_zone(setup, "UTC") + } + + /// [`Self::new`] under an explicit session time zone. + fn with_time_zone(setup: &str, zone: &str) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("fixture.duckdb"); + let con = Connection::open(&path).expect("open duckdb"); + con.execute_batch(&format!("SET TimeZone='{zone}';")) + .expect("set tz"); + con.execute_batch(setup).expect("fixture setup"); + Self { _dir: dir, con } + } +} + +/// A test-only [`RowProvider`] over a `DuckDB` connection, so the extractor and the compiled SQL +/// read the very same rows. +#[derive(Debug)] +struct DuckDbRowProvider<'a> { + con: &'a Connection, +} + +fn backend(table: &str, e: &duckdb::Error) -> ProviderError { + ProviderError::Backend { + table: table.to_string(), + message: e.to_string(), + } +} + +impl RowProvider for DuckDbRowProvider<'_> { + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError> { + let list = if columns.is_empty() { + "1".to_string() + } else { + columns + .iter() + .map(|c| format!("\"{}\"", c.replace('"', "\"\""))) + .collect::>() + .join(", ") + }; + let sql = format!("SELECT {list} FROM \"{}\"", table.replace('"', "\"\"")); + let mut stmt = self.con.prepare(&sql).map_err(|e| backend(table, &e))?; + let mut rows = stmt.query([]).map_err(|e| backend(table, &e))?; + let mut buf = vec![Value::Null; columns.len()]; + while let Some(row) = rows.next().map_err(|e| backend(table, &e))? { + for (i, slot) in buf.iter_mut().enumerate() { + *slot = value_of(row.get_ref(i).map_err(|e| backend(table, &e))?); + } + if f(&buf).is_break() { + return Ok(()); + } + } + Ok(()) + } +} + +/// One `DuckDB` cell as the extractor's own [`Value`]. +fn value_of(v: ValueRef<'_>) -> Value { + match v { + ValueRef::Null => Value::Null, + ValueRef::Boolean(b) => Value::Boolean(b), + ValueRef::TinyInt(i) => Value::Integer(i64::from(i)), + ValueRef::SmallInt(i) => Value::Integer(i64::from(i)), + ValueRef::Int(i) => Value::Integer(i64::from(i)), + ValueRef::BigInt(i) => Value::Integer(i), + ValueRef::UTinyInt(i) => Value::Integer(i64::from(i)), + ValueRef::USmallInt(i) => Value::Integer(i64::from(i)), + ValueRef::UInt(i) => Value::Integer(i64::from(i)), + ValueRef::Float(f) => Value::Float(f64::from(f)), + ValueRef::Double(f) => Value::Float(f), + ValueRef::Text(t) => Value::Text(String::from_utf8_lossy(t).into_owned()), + ValueRef::Timestamp(unit, raw) => Value::Timestamp( + DateTime::from_timestamp_micros(unit.to_micros(raw)) + .expect("a timestamp DuckDB produced is in range") + .fixed_offset(), + ), + ValueRef::Date32(d) => Value::Timestamp( + DateTime::from_timestamp(i64::from(d) * 86_400, 0) + .expect("a date DuckDB produced is in range") + .fixed_offset(), + ), + other => panic!("fixture used a column type the harness does not convert: {other:?}"), + } +} + +/// One log reduced to comparable sets. See the module docs for why sets. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct LogSets { + /// `(event id, event type, instant)`. + events: BTreeSet<(String, String, String)>, + /// `(event id, attribute name, rendered value)`. + event_attributes: BTreeSet<(String, String, String)>, + /// `(object id, object type)`. + objects: BTreeSet<(String, String)>, + /// `(object id, attribute name, instant, rendered value)`. + object_attributes: BTreeSet<(String, String, String, String)>, + /// `(event id, object id, qualifier)`. + e2o: BTreeSet<(String, String, String)>, + /// `(source object id, target object id, qualifier)`. + o2o: BTreeSet<(String, String, String)>, +} + +/// A value rendered with its kind, so a `Text` `"1"` cannot compare equal to an `Integer` `1`. +fn render(v: &Value) -> String { + match v { + Value::Null => "".to_string(), + Value::Text(s) => format!("s:{s}"), + Value::Integer(i) => format!("i:{i}"), + Value::Float(f) => format!("f:{f:?}"), + Value::Boolean(b) => format!("b:{b}"), + Value::Timestamp(t) => format!("t:{}", t.to_utc().to_rfc3339()), + } +} + +/// The same rendering for the extractor's own attribute values. +fn render_attr(v: &OCELAttributeValue) -> String { + match v { + OCELAttributeValue::Null => "".to_string(), + OCELAttributeValue::String(s) => format!("s:{s}"), + OCELAttributeValue::Integer(i) => format!("i:{i}"), + OCELAttributeValue::Float(f) => format!("f:{f:?}"), + OCELAttributeValue::Boolean(b) => format!("b:{b}"), + OCELAttributeValue::Time(t) => format!("t:{}", t.to_utc().to_rfc3339()), + } +} + +fn instant(t: DateTime) -> String { + t.to_utc().to_rfc3339() +} + +/// Reduce an extraction result to [`LogSets`]. +fn from_extractor(ocel: &O) -> LogSets { + let mut out = LogSets::default(); + for e in ocel.iter_events() { + out.events + .insert((e.id.clone(), e.event_type.clone(), instant(e.time))); + for a in &e.attributes { + out.event_attributes + .insert((e.id.clone(), a.name.clone(), render_attr(&a.value))); + } + for r in &e.relationships { + out.e2o + .insert((e.id.clone(), r.object_id.clone(), r.qualifier.clone())); + } + } + for o in ocel.iter_objects() { + out.objects.insert((o.id.clone(), o.object_type.clone())); + for a in &o.attributes { + out.object_attributes.insert(( + o.id.clone(), + a.name.clone(), + instant(a.time), + render_attr(&a.value), + )); + } + for r in &o.relationships { + out.o2o + .insert((o.id.clone(), r.object_id.clone(), r.qualifier.clone())); + } + } + out +} + +/// Run `sql` and return `(column names, rows)` with every cell as a [`Value`]. +fn query(con: &Connection, sql: &str) -> (Vec, Vec>) { + let mut stmt = con + .prepare(sql) + .unwrap_or_else(|e| panic!("preparing\n{sql}\nfailed: {e}")); + let mut rows = stmt + .query([]) + .unwrap_or_else(|e| panic!("running\n{sql}\nfailed: {e}")); + let mut names: Vec = Vec::new(); + let mut out = Vec::new(); + while let Some(row) = rows.next().expect("fetch row") { + if names.is_empty() { + names = row.as_ref().column_names(); + } + out.push( + (0..names.len()) + .map(|i| value_of(row.get_ref(i).expect("read cell"))) + .collect(), + ); + } + (names, out) +} + +fn text_of(v: &Value) -> String { + match v { + Value::Text(s) => s.clone(), + other => panic!("expected text, got {other:?}"), + } +} + +/// Reduce the compiled views to [`LogSets`], reading the same OCEL 2.0 relations an external +/// tool would. +fn from_sql(con: &Connection) -> LogSets { + let mut out = LogSets::default(); + + let (_, event_types) = query( + con, + "SELECT ocel_type, ocel_type_map FROM \"event_map_type\"", + ); + for row in &event_types { + let (ocel_type, suffix) = (text_of(&row[0]), text_of(&row[1])); + let (names, rows) = query(con, &format!("SELECT * FROM \"event_{suffix}\"")); + for r in rows { + let id = text_of(&r[0]); + let Value::Timestamp(t) = &r[1] else { + panic!("event_{suffix}.ocel_time is not a timestamp: {:?}", r[1]); + }; + out.events + .insert((id.clone(), ocel_type.clone(), instant(*t))); + for (i, name) in names.iter().enumerate().skip(2) { + out.event_attributes + .insert((id.clone(), name.clone(), render(&r[i]))); + } + } + } + + let (_, objects) = query(con, "SELECT ocel_id, ocel_type FROM \"object\""); + for r in &objects { + out.objects.insert((text_of(&r[0]), text_of(&r[1]))); + } + + let (_, object_types) = query( + con, + "SELECT ocel_type, ocel_type_map FROM \"object_map_type\"", + ); + for row in &object_types { + let suffix = text_of(&row[1]); + let (names, rows) = query(con, &format!("SELECT * FROM \"object_{suffix}\"")); + for r in rows { + // A row with no `ocel_changed_field` is the object's existence row and carries no + // observation. See `object_type_view`'s docs. + let Value::Text(changed) = &r[2] else { + continue; + }; + let id = text_of(&r[0]); + let Value::Timestamp(t) = &r[1] else { + panic!("object_{suffix}.ocel_time is not a timestamp: {:?}", r[1]); + }; + let column = names + .iter() + .position(|n| n == changed) + .unwrap_or_else(|| panic!("object_{suffix} has no column '{changed}'")); + out.object_attributes + .insert((id, changed.clone(), instant(*t), render(&r[column]))); + } + } + + let (_, e2o) = query( + con, + "SELECT ocel_event_id, ocel_object_id, ocel_qualifier FROM \"event_object\"", + ); + for r in &e2o { + out.e2o + .insert((text_of(&r[0]), text_of(&r[1]), text_of(&r[2]))); + } + let (_, o2o) = query( + con, + "SELECT ocel_source_id, ocel_target_id, ocel_qualifier FROM \"object_object\"", + ); + for r in &o2o { + out.o2o + .insert((text_of(&r[0]), text_of(&r[1]), text_of(&r[2]))); + } + out +} + +/// Reduce [`EmissionShape::Consolidated`]'s relations to [`LogSets`], reading them the way an +/// external reader would: `events` is a plain wide `SELECT *`, and +/// `object_attribute_changes.value`/`value_type` round-trip through [`from_sql_value`], the same +/// function +/// `DuckDbLinkedOCEL` uses. +fn from_sql_consolidated(con: &Connection) -> LogSets { + let mut out = LogSets::default(); + + let (names, events) = query(con, "SELECT * FROM \"events\""); + for r in &events { + let id = text_of(&r[0]); + let ocel_type = text_of(&r[1]); + let Value::Timestamp(t) = &r[2] else { + panic!("events.time is not a timestamp: {:?}", r[2]); + }; + out.events.insert((id.clone(), ocel_type, instant(*t))); + for (i, name) in names.iter().enumerate().skip(3) { + // A wide column an event's own type never declared reads back as `NULL`, not an + // observation. + if matches!(r[i], Value::Null) { + continue; + } + out.event_attributes + .insert((id.clone(), name.clone(), render(&r[i]))); + } + } + + let (_, objects) = query(con, "SELECT id, ocel_type FROM \"objects\""); + for r in &objects { + out.objects.insert((text_of(&r[0]), text_of(&r[1]))); + } + + let (_, attrs) = query( + con, + "SELECT id, name, \"time\", value, value_type FROM \"object_attribute_changes\"", + ); + for r in &attrs { + let id = text_of(&r[0]); + let name = text_of(&r[1]); + let Value::Timestamp(t) = &r[2] else { + panic!( + "object_attribute_changes.time is not a timestamp: {:?}", + r[2] + ); + }; + let value = from_sql_value(&text_of(&r[3]), &text_of(&r[4])); + out.object_attributes + .insert((id, name, instant(*t), render_attr(&value))); + } + + let (_, e2o) = query(con, "SELECT event_id, object_id, qualifier FROM \"e2o\""); + for r in &e2o { + out.e2o + .insert((text_of(&r[0]), text_of(&r[1]), text_of(&r[2]))); + } + let (_, o2o) = query(con, "SELECT source_id, target_id, qualifier FROM \"o2o\""); + for r in &o2o { + out.o2o + .insert((text_of(&r[0]), text_of(&r[1]), text_of(&r[2]))); + } + out +} + +/// What one differential run produced, for tests that want to assert more than agreement. +/// `compiled`/`sql` are `PerType`'s, `consolidated`/`consolidated_sql` are `Consolidated`'s. +struct Run { + report: ExtractionReport, + compiled: CompiledOcel, + extractor: LogSets, + sql: LogSets, + consolidated: CompiledOcel, + consolidated_sql: LogSets, +} + +/// Every entry of an `event_attributes`-shaped set except the `Null`-valued ones. See +/// [`assert_consolidated_agrees`] for why the carve-out exists. +fn drop_null_event_attrs( + set: &BTreeSet<(String, String, String)>, +) -> BTreeSet<(String, String, String)> { + set.iter() + .filter(|(_, _, v)| v != "") + .cloned() + .collect() +} + +/// The six [`LogSets`] fields, field by field, prefixing every failure with `shape` so a +/// disagreement names which emission surface produced it. `drop_null_event_attributes` is +/// documented on its one caller, [`assert_consolidated_agrees`]. +fn assert_log_sets_agree( + extractor: &LogSets, + sql: &LogSets, + shape: &str, + drop_null_event_attributes: bool, +) { + assert_eq!( + extractor.events, sql.events, + "[{shape}] events disagree (extractor left, SQL right)" + ); + if drop_null_event_attributes { + assert_eq!( + drop_null_event_attrs(&extractor.event_attributes), + drop_null_event_attrs(&sql.event_attributes), + "[{shape}] event attributes disagree" + ); + } else { + assert_eq!( + extractor.event_attributes, sql.event_attributes, + "[{shape}] event attributes disagree" + ); + } + assert_eq!(extractor.objects, sql.objects, "[{shape}] objects disagree"); + assert_eq!( + extractor.object_attributes, sql.object_attributes, + "[{shape}] object attribute observations disagree" + ); + assert_eq!(extractor.e2o, sql.e2o, "[{shape}] E2O relations disagree"); + assert_eq!(extractor.o2o, sql.o2o, "[{shape}] O2O relations disagree"); +} + +/// [`assert_log_sets_agree`] against `PerType`'s own output, with nothing dropped: `PerType` has +/// one view per event type, so a `NULL` cell there is unambiguously an observation of `Null`. +fn assert_per_type_agrees(extractor: &LogSets, sql: &LogSets) { + assert_log_sets_agree(extractor, sql, "PerType", false); +} + +/// [`assert_log_sets_agree`] against `Consolidated`'s own output, with every `Null`-valued event +/// attribute observation dropped from both sides first. +/// +/// A `NULL` cell in that shape's wide `events` table is ambiguous between "declared, with value +/// `Null`" and "belongs to a different event type", a limit of the schema that every reader +/// shares. Object attributes have no such gap, since `object_attribute_changes` is EAV. +fn assert_consolidated_agrees(extractor: &LogSets, sql: &LogSets) { + assert_log_sets_agree(extractor, sql, "Consolidated", true); +} + +/// Whether a differential run tolerates the compiler refusing a mapping. +/// +/// A mapping the compiler skipped is a relation neither side carries, so an unasserted +/// `RejectReason` turns a would-be divergence into a silent pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CompileErrors { + /// Neither shape may refuse anything: [`run_both`] asserts both error lists are empty. + None, + /// The test asserts the refusals itself. + Expected, +} + +/// Extract, compile both emission shapes, execute and compare. +/// +/// Asserts, in order, that the blueprint validates, that the extraction reports no +/// `IdTypeCollision` (see the module docs), that neither shape refused a mapping, that every +/// probe of both shapes returns zero rows, and that each shape's [`LogSets`] equals the +/// extractor's. +fn assert_agrees(fx: &Fixture, blueprint: &Blueprint, catalog: &ExtractionCatalog) -> Run { + let run = run_both(fx, blueprint, catalog); + assert_per_type_agrees(&run.extractor, &run.sql); + assert_consolidated_agrees(&run.extractor, &run.consolidated_sql); + run +} + +/// `probes()` paired with `probe_statements()`, checking the two lists have the same length so a +/// `zip` cannot silently drop the tail. +fn probes_with_sql(compiled: &CompiledOcel) -> Vec<(ProbeKind, String)> { + let statements = compiled.probe_statements(); + assert_eq!( + compiled.probes().len(), + statements.len(), + "every probe must have a statement, or zipping the two silently checks a prefix" + ); + compiled + .probes() + .iter() + .map(|p| p.kind.clone()) + .zip(statements) + .collect() +} + +/// Every probe a compile produced, run against `con` and asserted empty. +/// [`every_probe_kind_is_shown_to_fire`] separately drives each [`ProbeKind`] into returning rows, +/// since emptiness is satisfied vacuously by a probe that selects nothing. +fn assert_probes_hold(con: &Connection, compiled: &CompiledOcel, shape: &str) { + for (kind, sql) in probes_with_sql(compiled) { + let (_, rows) = query(con, &sql); + assert!( + rows.is_empty(), + "[{shape}] probe {kind:?} must hold before the views may be compared: {} rows\n{sql}", + rows.len() + ); + } +} + +/// Which [`ProbeKind`]s actually return rows against `con`, as their `Debug` spellings. +fn firing_probe_kinds(con: &Connection, compiled: &CompiledOcel) -> BTreeSet { + probes_with_sql(compiled) + .into_iter() + .filter(|(_, sql)| !query(con, sql).1.is_empty()) + .map(|(kind, _)| format!("{kind:?}")) + .collect() +} + +/// Unless the caller said it expects one, a [`CompileError`](super::CompileError) fails the run. +fn assert_no_unexpected_errors(compiled: &CompiledOcel, shape: &str, errors: CompileErrors) { + if errors == CompileErrors::None { + assert!( + compiled.errors().is_empty(), + "[{shape}] the compiler refused a mapping, so the comparison below would be between \ + an extraction and a partial compile: {:?}", + compiled.errors() + ); + } +} + +/// Whether every `Target::Event` in `blueprint` names its type with a `Constant`. +/// +/// Only a statically known event type contributes rows to `event_attr_meta`, so comparing that +/// relation against the extractor's declarations is exact only for such a blueprint. +fn every_event_type_is_static(blueprint: &Blueprint) -> bool { + fn is_static(m: &Mapping) -> bool { + match &m.target { + Target::Event { event_type, .. } => { + matches!(event_type, ValueExpression::Constant { .. }) + } + _ => true, + } + } + blueprint.mappings.iter().all(|entry| match entry { + MappingEntry::Single(m) => is_static(m), + MappingEntry::Ordered { mappings } => mappings.iter().all(is_static), + }) +} + +/// `event_attr_meta`, the one `Consolidated` relation [`from_sql_consolidated`] does not read, so +/// no [`LogSets`] comparison can catch it being wrong. Compared against the extractor's own +/// [`ReadableOCEL::event_types`]. +fn assert_event_attr_meta_agrees(con: &Connection, ocel: &O) { + let expected: BTreeSet<(String, String, String)> = ocel + .event_types() + .iter() + .flat_map(|t| { + t.attributes + .iter() + .map(|a| (t.name.clone(), a.name.clone(), a.value_type.clone())) + }) + .collect(); + let (_, rows) = query( + con, + "SELECT event_type, attr_name, attr_type FROM \"event_attr_meta\"", + ); + let actual: BTreeSet<(String, String, String)> = rows + .iter() + .map(|r| (text_of(&r[0]), text_of(&r[1]), text_of(&r[2]))) + .collect(); + assert_eq!( + expected, actual, + "[Consolidated] event_attr_meta disagrees with the extractor's event-type declarations \ + (extractor left, SQL right)" + ); +} + +/// Extract, compile [`EmissionShape::Consolidated`] alone and execute it, for the blueprints +/// [`run_both`] cannot take because `PerType` refuses them. Returns the compile, the extractor's +/// log and the SQL's, uncompared. +fn run_consolidated_only( + fx: &Fixture, + blueprint: &Blueprint, + catalog: &ExtractionCatalog, +) -> (CompiledOcel, LogSets, LogSets) { + assert_eq!( + validate(blueprint, catalog), + vec![], + "blueprint must validate" + ); + let provider = DuckDbRowProvider { con: &fx.con }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + let mut sink = SlimOcelSink::new(); + extract(blueprint, catalog, &providers, &mut sink).expect("extract"); + let extractor = from_extractor(sink.ocel()); + + let compiled = compile( + blueprint, + catalog, + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert!(compiled.errors().is_empty(), "{:?}", compiled.errors()); + let ddl = compiled.ddl(); + fx.con.execute_batch(&ddl).unwrap_or_else(|e| { + panic!("executing the emitted Consolidated DDL failed: {e}\n---\n{ddl}") + }); + assert_probes_hold(&fx.con, &compiled, "Consolidated"); + let sql = from_sql_consolidated(&fx.con); + (compiled, extractor, sql) +} + +/// The half of [`assert_agrees`] that produces every log without comparing them, with both +/// shapes required to compile whole. +fn run_both(fx: &Fixture, blueprint: &Blueprint, catalog: &ExtractionCatalog) -> Run { + run_both_with(fx, blueprint, catalog, CompileErrors::None) +} + +/// [`run_both`] for the tests that deliberately provoke a [`RejectReason`] and assert it +/// themselves. +fn run_both_expecting_errors( + fx: &Fixture, + blueprint: &Blueprint, + catalog: &ExtractionCatalog, +) -> Run { + run_both_with(fx, blueprint, catalog, CompileErrors::Expected) +} + +/// Produce every log without comparing them, for the tests that have to demonstrate an expected +/// difference (a skipped mapping). +/// +/// Both shapes' `CREATE VIEW`s go on the same connection: `PerType` and `Consolidated` never +/// share a relation name, so nothing here needs two databases. +fn run_both_with( + fx: &Fixture, + blueprint: &Blueprint, + catalog: &ExtractionCatalog, + errors: CompileErrors, +) -> Run { + assert_eq!( + validate(blueprint, catalog), + vec![], + "blueprint must validate" + ); + + let provider = DuckDbRowProvider { con: &fx.con }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(blueprint, catalog, &providers, &mut sink).expect("extract"); + assert!( + !report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::IdTypeCollision { .. })), + "harness precondition: a cross-type id collision makes the extractor drop an entity the \ + views keep, which is not the compiler's disagreement to answer for: {:?}", + report.errors + ); + let extractor = from_extractor(sink.ocel()); + + let compiled = compile( + blueprint, + catalog, + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert_no_unexpected_errors(&compiled, "PerType", errors); + let ddl = compiled.ddl(); + fx.con + .execute_batch(&ddl) + .unwrap_or_else(|e| panic!("executing the emitted PerType DDL failed: {e}\n---\n{ddl}")); + assert_probes_hold(&fx.con, &compiled, "PerType"); + let sql = from_sql(&fx.con); + + let consolidated = compile( + blueprint, + catalog, + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert_no_unexpected_errors(&consolidated, "Consolidated", errors); + let consolidated_ddl = consolidated.ddl(); + fx.con.execute_batch(&consolidated_ddl).unwrap_or_else(|e| { + panic!("executing the emitted Consolidated DDL failed: {e}\n---\n{consolidated_ddl}") + }); + assert_probes_hold(&fx.con, &consolidated, "Consolidated"); + let consolidated_sql = from_sql_consolidated(&fx.con); + + if errors == CompileErrors::None && every_event_type_is_static(blueprint) { + assert_event_attr_meta_agrees(&fx.con, sink.ocel()); + } + + Run { + report, + compiled, + extractor, + sql, + consolidated, + consolidated_sql, + } +} + +fn col(name: &str) -> ValueExpression { + ValueExpression::Column { + column: name.to_string(), + } +} + +fn constant(value: &str) -> ValueExpression { + ValueExpression::Constant { + value: value.to_string(), + } +} + +fn source(id: &str, table: &str) -> Node { + Node { + id: id.to_string(), + label: None, + op: NodeOp::Source { + source_id: "db".to_string(), + table: table.to_string(), + }, + } +} + +fn blueprint(rendering: IdRendering, nodes: Vec, mappings: Vec) -> Blueprint { + Blueprint { + version: crate::core::event_data::object_centric::extraction::MODEL_VERSION, + id_rendering: rendering, + nodes, + mappings, + on_missing_endpoint: MissingEndpointPolicy::Drop, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } +} + +fn single(node: &str, label: &str, when: Option, target: Target) -> MappingEntry { + MappingEntry::Single(Mapping { + node: node.to_string(), + label: Some(label.to_string()), + when, + target, + }) +} + +fn ts(column: &str) -> TimestampSource { + TimestampSource::column(column.to_string()) +} + +fn endpoint(id: &str, object_type: &str) -> ObjectEndpoint { + ObjectEndpoint { + id: col(id), + object_type: Some(constant(object_type)), + split: None, + } +} + +fn eq_text(column: &str, value: &str) -> Predicate { + Predicate::Compare { + left: Operand::Column { + column: column.to_string(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(value.to_string()), + }, + } +} + +const ORDERS: &str = " +CREATE TABLE orders (id BIGINT, cust VARCHAR, ts TIMESTAMP); +INSERT INTO orders VALUES + (1, 'ACME', TIMESTAMP '2020-01-01 08:00:00'), + (2, 'ACME', TIMESTAMP '2020-01-02 09:30:00'), + (3, 'GLOBEX', TIMESTAMP '2020-01-03 10:15:00'), + (4, NULL, TIMESTAMP '2020-01-04 11:00:00'), + (5, 'GLOBEX', NULL); +"; + +fn orders_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "orders", + [ + ("id", "BIGINT", false), + ("cust", "VARCHAR", true), + ("ts", "TIMESTAMP", true), + ], + ), + ) +} + +fn order_object() -> MappingEntry { + single( + "orders", + "order", + None, + Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ) +} + +fn customer_object() -> MappingEntry { + single( + "orders", + "customer", + None, + Target::Object { + object_type: constant("Customer"), + id: col("cust"), + timestamp: None, + attributes: vec![], + }, + ) +} + +fn placed_event() -> MappingEntry { + single( + "orders", + "placed", + None, + Target::Event { + event_type: constant("Placed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + ) +} + +#[test] +fn case_1_event_object_e2o_and_o2o_targets_each_agree() { + let fx = Fixture::new(ORDERS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("orders", "orders")], + vec![ + order_object(), + customer_object(), + placed_event(), + single( + "orders", + "placed-order", + None, + Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Placed")), + }, + object: endpoint("id", "Order"), + qualifier: Some(constant("order")), + }, + ), + single( + "orders", + "order-customer", + None, + Target::O2O { + source: endpoint("id", "Order"), + target: endpoint("cust", "Customer"), + qualifier: Some(constant("buyer")), + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &orders_catalog()); + // The comparison is only worth something if it compared something. + assert!(!run.extractor.objects.is_empty()); + assert!(!run.extractor.e2o.is_empty()); + assert!(!run.extractor.o2o.is_empty()); + // Row 5 has no timestamp, so its event is dropped on both sides. + assert_eq!(run.extractor.events.len(), 4, "{:?}", run.extractor.events); + assert!( + run.compiled.errors().is_empty(), + "{:?}", + run.compiled.errors() + ); +} + +#[test] +fn case_1_an_inline_object_reference_agrees() { + let fx = Fixture::new(ORDERS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("orders", "orders")], + vec![ + customer_object(), + single( + "orders", + "placed", + None, + Target::Event { + event_type: constant("Placed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![InlineObjectRef { + object: endpoint("cust", "Customer"), + qualifier: Some(constant("buyer")), + }], + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &orders_catalog()); + assert!(!run.extractor.e2o.is_empty()); + let _ = run.report; +} + +#[test] +fn case_1_an_o2o_creates_no_source_object_for_a_row_whose_target_id_is_absent() { + // `run_o2o` renders both endpoint ids before it resolves either, and returns as soon as one + // is absent, so row 4 (`cust` NULL) creates neither `Order-4` nor a customer. A compiled + // source branch built from the filters as they stood before the target's guards were added + // creates `Order-4` regardless, an object no extraction produces. + let fx = Fixture::new(ORDERS); + let mut bp = blueprint( + IdRendering::TypePrefixed, + vec![source("orders", "orders")], + vec![single( + "orders", + "order-customer", + None, + Target::O2O { + source: endpoint("id", "Order"), + target: endpoint("cust", "Customer"), + qualifier: Some(constant("buyer")), + }, + )], + ); + // Both endpoints exist only as relation ends, so they have to be created rather than dropped. + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let run = assert_agrees(&fx, &bp, &orders_catalog()); + assert!( + !run.extractor + .objects + .contains(&("Order-4".to_string(), "Order".to_string())), + "the row with no customer must create no order either: {:?}", + run.extractor.objects + ); + assert_eq!(run.extractor.o2o.len(), 4, "{:?}", run.extractor.o2o); +} + +const DOCS: &str = " +CREATE TABLE docs (id BIGINT, kind VARCHAR, ts TIMESTAMP); +INSERT INTO docs VALUES + (1, 'invoice', TIMESTAMP '2021-01-01 00:00:00'), + (2, 'credit', TIMESTAMP '2021-01-02 00:00:00'), + (3, 'invoice', TIMESTAMP '2021-01-03 00:00:00'), + (4, NULL, TIMESTAMP '2021-01-04 00:00:00'); +"; + +fn docs_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "docs", + [ + ("id", "BIGINT", false), + ("kind", "VARCHAR", true), + ("ts", "TIMESTAMP", false), + ], + ), + ) +} + +#[test] +fn case_2_when_discriminated_mappings_over_one_node_agree() { + let fx = Fixture::new(DOCS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("docs", "docs")], + vec![ + single( + "docs", + "invoices", + Some(eq_text("kind", "invoice")), + Target::Object { + object_type: constant("Invoice"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "docs", + "credits", + Some(eq_text("kind", "credit")), + Target::Object { + object_type: constant("CreditNote"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &docs_catalog()); + assert_eq!( + run.extractor.objects.len(), + 3, + "{:?}", + run.extractor.objects + ); +} + +#[test] +fn case_3_an_ordered_group_keeps_the_rows_whose_guard_column_is_null() { + // `desugar` rewrites the catch-all's guard to `Not(kind = 'invoice') AND Not(kind = + // 'credit')`. In the extractor's two-valued evaluation that is true for row 4, whose `kind` + // is NULL. A naive SQL compile makes `NOT (kind = 'invoice')` NULL there and drops exactly + // that row, which is why this fixture has a NULL in the guard column. + let fx = Fixture::new(DOCS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("docs", "docs")], + vec![MappingEntry::Ordered { + mappings: vec![ + Mapping { + node: "docs".into(), + label: Some("invoice".into()), + when: Some(eq_text("kind", "invoice")), + target: Target::Object { + object_type: constant("Invoice"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }, + Mapping { + node: "docs".into(), + label: Some("credit".into()), + when: Some(eq_text("kind", "credit")), + target: Target::Object { + object_type: constant("CreditNote"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }, + Mapping { + node: "docs".into(), + label: Some("other".into()), + when: None, + target: Target::Object { + object_type: constant("Other"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }, + ], + }], + ); + let run = assert_agrees(&fx, &bp, &docs_catalog()); + assert!( + run.extractor + .objects + .contains(&("Other-4".to_string(), "Other".to_string())), + "the NULL-guard row must reach the catch-all: {:?}", + run.extractor.objects + ); + assert!( + run.sql + .objects + .contains(&("Other-4".to_string(), "Other".to_string())), + "and the compiled view must keep it too: {:?}", + run.sql.objects + ); +} + +const JOINED: &str = " +CREATE TABLE headers (id BIGINT, ref VARCHAR, ts TIMESTAMP); +CREATE TABLE lines (id BIGINT, ref VARCHAR, sku VARCHAR); +CREATE TABLE more_lines (id BIGINT, ref VARCHAR); +INSERT INTO headers VALUES + (1, 'R1', TIMESTAMP '2022-01-01 00:00:00'), + (2, 'R2', TIMESTAMP '2022-01-02 00:00:00'), + (3, 'R3', TIMESTAMP '2022-01-03 00:00:00'); +INSERT INTO lines VALUES (10, 'R1', 'A'), (11, 'R1', 'B'), (12, 'R2', 'C'); +INSERT INTO more_lines VALUES (20, 'R3'), (21, 'R9'); +"; + +fn joined_catalog() -> ExtractionCatalog { + ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "headers", + [ + ("id", "BIGINT", false), + ("ref", "VARCHAR", false), + ("ts", "TIMESTAMP", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new( + "lines", + [ + ("id", "BIGINT", false), + ("ref", "VARCHAR", false), + ("sku", "VARCHAR", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new( + "more_lines", + [("id", "BIGINT", false), ("ref", "VARCHAR", false)], + ), + ) +} + +#[test] +fn case_4_a_join_agrees_including_the_right_prefixed_collision_column() { + let fx = Fixture::new(JOINED); + // `headers` and `lines` both have `id` and `ref`, so the join's output carries the left's + // under their own names and the right's as `right_id` / `right_ref`. + let bp = blueprint( + IdRendering::TypePrefixed, + vec![ + source("h", "headers"), + source("l", "lines"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "h".into(), + right: "l".into(), + on: vec![("ref".into(), "ref".into())], + }, + }, + ], + vec![ + single( + "j", + "header", + None, + Target::Object { + object_type: constant("Header"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "j", + "line", + None, + Target::Object { + object_type: constant("Line"), + id: col("right_id"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "j", + "header-line", + None, + Target::O2O { + source: endpoint("id", "Header"), + target: endpoint("right_id", "Line"), + qualifier: Some(col("sku")), + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &joined_catalog()); + assert_eq!(run.extractor.o2o.len(), 3, "{:?}", run.extractor.o2o); + assert!(run + .extractor + .objects + .contains(&("Line-10".to_string(), "Line".to_string()))); +} + +#[test] +fn case_4_a_union_agrees_with_the_absent_column_null_filled() { + let fx = Fixture::new(JOINED); + // `lines` has `sku`, `more_lines` does not, so the union null-fills it for the second + // branch. The object id reads `sku` through a Coalesce so the null-fill is observable. + let bp = blueprint( + IdRendering::TypePrefixed, + vec![ + source("l", "lines"), + source("m", "more_lines"), + Node { + id: "u".into(), + label: None, + op: NodeOp::Union { + inputs: vec!["l".into(), "m".into()], + }, + }, + ], + vec![single( + "u", + "line", + None, + Target::Object { + object_type: constant("Line"), + id: ValueExpression::Coalesce { + parts: vec![col("sku"), col("ref")], + }, + timestamp: None, + attributes: vec![], + }, + )], + ); + let run = assert_agrees(&fx, &bp, &joined_catalog()); + assert!( + run.extractor + .objects + .contains(&("Line-R3".to_string(), "Line".to_string())), + "the branch without `sku` must fall through to `ref`: {:?}", + run.extractor.objects + ); + assert_eq!( + run.extractor.objects.len(), + 5, + "{:?}", + run.extractor.objects + ); +} + +const SPLITS: &str = " +CREATE TABLE tickets (id BIGINT, parts VARCHAR, pairs VARCHAR, ts TIMESTAMP); +INSERT INTO tickets VALUES + (1, 'a, b ,,c', 'x=1;y=22', TIMESTAMP '2023-01-01 00:00:00'), + (2, 'd', 'z=3', TIMESTAMP '2023-01-02 00:00:00'), + (3, '', '', TIMESTAMP '2023-01-03 00:00:00'), + (4, NULL, NULL, TIMESTAMP '2023-01-04 00:00:00'); +"; + +fn tickets_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "tickets", + [ + ("id", "BIGINT", false), + ("parts", "VARCHAR", true), + ("pairs", "VARCHAR", true), + ("ts", "TIMESTAMP", false), + ], + ), + ) +} + +fn split_blueprint(column: &str, kind: SplitKind) -> Blueprint { + let mut bp = blueprint( + IdRendering::TypePrefixed, + vec![source("t", "tickets")], + vec![ + single( + "t", + "ticket", + None, + Target::Event { + event_type: constant("Ticket"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + ), + single( + "t", + "tags", + None, + Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Ticket")), + }, + object: ObjectEndpoint { + id: col(column), + object_type: Some(constant("Tag")), + split: Some(SplitSpec { kind, trim: true }), + }, + qualifier: Some(constant("tag")), + }, + ), + ], + ); + // Tags exist only as relation endpoints, so they have to be created rather than dropped. + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + bp +} + +#[test] +fn case_5_a_delimiter_split_agrees() { + let fx = Fixture::new(SPLITS); + let bp = split_blueprint( + "parts", + SplitKind::Delimiter { + delimiter: ",".into(), + }, + ); + let run = assert_agrees(&fx, &bp, &tickets_catalog()); + assert!( + run.extractor + .objects + .contains(&("Tag-b".to_string(), "Tag".to_string())), + "trimmed parts must survive: {:?}", + run.extractor.objects + ); + // 'a, b ,,c' yields a, b, c (the empty part is dropped) and 'd' yields d. + assert_eq!(run.extractor.e2o.len(), 4, "{:?}", run.extractor.e2o); +} + +#[test] +fn case_5_a_regex_split_agrees() { + let fx = Fixture::new(SPLITS); + let bp = split_blueprint( + "pairs", + SplitKind::Regex { + pattern: "([a-z])=([0-9]+)".into(), + }, + ); + let run = assert_agrees(&fx, &bp, &tickets_catalog()); + // 'x=1;y=22' yields x, 1, y, 22 and 'z=3' yields z, 3. + assert_eq!(run.extractor.e2o.len(), 6, "{:?}", run.extractor.e2o); +} + +#[test] +fn case_5_a_single_group_regex_split_agrees() { + // One group is its own emission path: the parts come from a single `regexp_extract_all` call + // rather than a concatenation of one per group. + let fx = Fixture::new(SPLITS); + let bp = split_blueprint( + "pairs", + SplitKind::Regex { + pattern: "=([0-9]+)".into(), + }, + ); + let run = assert_agrees(&fx, &bp, &tickets_catalog()); + // 'x=1;y=22' yields 1, 22 and 'z=3' yields 3. + assert_eq!(run.extractor.e2o.len(), 3, "{:?}", run.extractor.e2o); +} + +#[test] +fn case_5_a_group_free_regex_split_agrees() { + // No group at all is a third path: every whole match is a part. + let fx = Fixture::new(SPLITS); + let bp = split_blueprint( + "pairs", + SplitKind::Regex { + pattern: "[a-z]=[0-9]+".into(), + }, + ); + let run = assert_agrees(&fx, &bp, &tickets_catalog()); + // 'x=1;y=22' yields x=1, y=22 and 'z=3' yields z=3. + assert_eq!(run.extractor.e2o.len(), 3, "{:?}", run.extractor.e2o); +} + +const ATTRS: &str = " +CREATE TABLE items (id BIGINT, name VARCHAR, price DOUBLE, active BOOLEAN, ts TIMESTAMP); +INSERT INTO items VALUES + (1, 'widget', 9.5, true, TIMESTAMP '2024-01-01 00:00:00'), + (2, 'gadget', 12.0, false, TIMESTAMP '2024-01-02 00:00:00'), + (3, NULL, NULL, NULL, TIMESTAMP '2024-01-03 00:00:00'); +CREATE TABLE price_changes (item BIGINT, price DOUBLE, ts TIMESTAMP); +INSERT INTO price_changes VALUES + (1, 9.5, TIMESTAMP '2024-01-01 00:00:00'), + (1, 10.5, TIMESTAMP '2024-02-01 00:00:00'), + (2, 12.0, TIMESTAMP '2024-01-02 00:00:00'); +"; + +fn attrs_catalog() -> ExtractionCatalog { + ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "items", + [ + ("id", "BIGINT", false), + ("name", "VARCHAR", true), + ("price", "DOUBLE", true), + ("active", "BOOLEAN", true), + ("ts", "TIMESTAMP", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new( + "price_changes", + [ + ("item", "BIGINT", false), + ("price", "DOUBLE", false), + ("ts", "TIMESTAMP", false), + ], + ), + ) +} + +#[test] +fn case_6_static_object_attributes_and_typed_event_attributes_agree() { + let fx = Fixture::new(ATTRS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("items", "items")], + vec![ + single( + "items", + "item", + None, + Target::Object { + object_type: constant("Item"), + id: col("id"), + timestamp: None, + attributes: vec![ + AttributeMapping { + source_column: "name".into(), + name: "name".into(), + value_type: None, + }, + AttributeMapping { + source_column: "active".into(), + name: "active".into(), + value_type: Some(OCELAttributeType::Boolean), + }, + ], + }, + ), + single( + "items", + "listed", + None, + Target::Event { + event_type: constant("Listed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![ + AttributeMapping { + source_column: "price".into(), + name: "price".into(), + value_type: Some(OCELAttributeType::Float), + }, + AttributeMapping { + source_column: "name".into(), + name: "name".into(), + value_type: None, + }, + ], + objects: vec![], + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &attrs_catalog()); + assert!( + run.extractor + .object_attributes + .iter() + .any(|(id, name, _, v)| id == "Item-1" && name == "name" && v == "s:widget"), + "{:?}", + run.extractor.object_attributes + ); + assert!( + run.extractor + .event_attributes + .iter() + .any(|(id, name, v)| id == "Listed-1" && name == "price" && v == "f:9.5"), + "{:?}", + run.extractor.event_attributes + ); + // A NULL cell is a recorded observation of Null, not an absent attribute. + assert!(run.extractor.object_attributes.contains(&( + "Item-3".to_string(), + "name".to_string(), + "1970-01-01T00:00:00+00:00".to_string(), + "".to_string() + ))); +} + +#[test] +fn case_6_change_tracked_object_attributes_agree() { + let fx = Fixture::new(ATTRS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("pc", "price_changes")], + vec![single( + "pc", + "prices", + None, + Target::Object { + object_type: constant("Item"), + id: col("item"), + timestamp: Some(ts("ts")), + attributes: vec![AttributeMapping { + source_column: "price".into(), + name: "price".into(), + value_type: None, + }], + }, + )], + ); + let run = assert_agrees(&fx, &bp, &attrs_catalog()); + assert_eq!( + run.extractor.object_attributes.len(), + 3, + "every row is an observation: {:?}", + run.extractor.object_attributes + ); + assert!(run.extractor.object_attributes.contains(&( + "Item-1".to_string(), + "price".to_string(), + "2024-02-01T00:00:00+00:00".to_string(), + "f:10.5".to_string() + ))); +} + +#[test] +fn case_7_a_text_literal_against_an_integer_column_selects_the_same_rows() { + // An editor with a text input emits Literal::Text("2"), and `prepare` coerces it to the + // column's declared kind, so `id <= "2"` is a numeric comparison in the extractor. An + // uncoercible literal never reaches either side, since `validate` refuses the blueprint + // outright, as the assertion at the end of this test pins. + let fx = Fixture::new(ORDERS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("orders", "orders")], + vec![ + single( + "orders", + "coercible", + Some(Predicate::Compare { + left: Operand::Column { + column: "id".into(), + }, + op: CompareOp::Le, + right: Operand::Literal { + value: Literal::Text("2".into()), + }, + }), + Target::Object { + object_type: constant("Small"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "orders", + "uncoercible", + Some(Predicate::In { + column: "id".into(), + values: vec![Literal::Text("3".into())], + }), + Target::Object { + object_type: constant("Listed"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &orders_catalog()); + assert_eq!( + run.extractor + .objects + .iter() + .filter(|(_, t)| t == "Small") + .count(), + 2, + "id <= 2 must be a numeric comparison: {:?}", + run.extractor.objects + ); + assert_eq!( + run.extractor + .objects + .iter() + .filter(|(_, t)| t == "Listed") + .count(), + 1, + "the text member of the IN list must match the integer row: {:?}", + run.extractor.objects + ); + + let mut uncoercible = bp; + uncoercible.mappings.push(single( + "orders", + "uncoercible", + Some(Predicate::In { + column: "id".into(), + values: vec![Literal::Text("abc".into())], + }), + Target::Object { + object_type: constant("Other"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + )); + assert!( + !validate(&uncoercible, &orders_catalog()).is_empty(), + "a literal no cell of the column can equal is a validation error, not a compile one" + ); +} + +const ACTIVITIES: &str = " +CREATE TABLE log (id BIGINT, activity VARCHAR, ts TIMESTAMP); +INSERT INTO log VALUES + (1, 'Create', TIMESTAMP '2025-01-01 00:00:00'), + (2, 'Approve', TIMESTAMP '2025-01-02 00:00:00'), + (3, 'Create', TIMESTAMP '2025-01-03 00:00:00'); +"; + +fn log_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "log", + [ + ("id", "BIGINT", false), + ("activity", "VARCHAR", false), + ("ts", "TIMESTAMP", false), + ], + ), + ) +} + +fn dynamic_type_blueprint() -> Blueprint { + blueprint( + IdRendering::TypePrefixed, + vec![source("log", "log")], + vec![single( + "log", + "activities", + None, + Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + )], + ) +} + +#[test] +fn case_8_a_dynamic_event_type_with_a_supplied_domain_agrees() { + let fx = Fixture::new(ACTIVITIES); + let catalog = log_catalog().with_domain( + "db", + "log", + "activity", + ["Create".to_string(), "Approve".to_string()], + ); + let run = assert_agrees(&fx, &dynamic_type_blueprint(), &catalog); + assert!( + run.compiled.errors().is_empty(), + "{:?}", + run.compiled.errors() + ); + assert_eq!(run.extractor.events.len(), 3); + assert!(run + .compiled + .relations() + .iter() + .any(|v| v.name == "event_Create")); + assert!(run + .compiled + .relations() + .iter() + .any(|v| v.name == "event_Approve")); +} + +#[test] +fn case_8_the_staleness_probe_fires_once_a_value_outside_the_domain_is_inserted() { + let fx = Fixture::new(ACTIVITIES); + let catalog = log_catalog().with_domain( + "db", + "log", + "activity", + ["Create".to_string(), "Approve".to_string()], + ); + let bp = dynamic_type_blueprint(); + // Holds before, views included. + assert_agrees(&fx, &bp, &catalog); + + let compiled = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + let stale = probes_with_sql(&compiled) + .into_iter() + .find(|(kind, _)| matches!(kind, ProbeKind::StaleTypeDomain { .. })) + .map(|(_, sql)| sql) + .expect("a domain-derived type set must carry a staleness probe"); + + fx.con + .execute_batch("INSERT INTO log VALUES (4, 'Reject', TIMESTAMP '2025-01-04 00:00:00');") + .expect("insert an out-of-domain value"); + + let (_, rows) = query(&fx.con, &stale); + assert_eq!( + rows.len(), + 1, + "the probe must report exactly the value that appeared after compilation" + ); + assert_eq!(text_of(&rows[0][0]), "Reject"); + + // And the views really are missing those events now, which is what the probe guards. + let sql = from_sql(&fx.con); + assert_eq!(sql.events.len(), 3, "{:?}", sql.events); +} + +// The capability `PerType` cannot have at all: a type read from a column with no domain. +// +// `PerType` needs a domain to name each per-type view; with none supplied it is a +// `RejectReason::DynamicTypeName`. Under `Consolidated` the type is a column value, so there is +// no domain to need. + +#[test] +fn a_dynamic_type_with_no_domain_is_a_per_type_reject_but_a_consolidated_pass() { + let fx = Fixture::new(ACTIVITIES); + // Unlike case 8's, this catalog never gets a `.with_domain` call. + let catalog = log_catalog(); + let bp = dynamic_type_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![], "blueprint must validate"); + + let per_type = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert_eq!(per_type.errors().len(), 1, "{:?}", per_type.errors()); + assert!( + matches!( + per_type.errors()[0].reason, + RejectReason::DynamicTypeName { .. } + ), + "{:?}", + per_type.errors()[0] + ); + + let (consolidated, extractor, sql) = run_consolidated_only(&fx, &bp, &catalog); + assert_eq!(extractor.events.len(), 3, "{:?}", extractor.events); + assert!(extractor.events.iter().any(|(_, t, _)| t == "Create")); + assert!(extractor.events.iter().any(|(_, t, _)| t == "Approve")); + assert!( + !consolidated + .probes() + .iter() + .any(|p| matches!(p.kind, ProbeKind::StaleTypeDomain { .. })), + "no domain means nothing can go stale: {:?}", + consolidated.probes() + ); + assert_consolidated_agrees(&extractor, &sql); +} + +#[test] +fn a_dynamically_typed_events_attributes_reach_the_wide_events_table() { + // The attribute plan keys a declaration by its type name, which a `Consolidated` mapping + // reading its type from a column does not have, so `AttributePlan::dynamic_event_attrs` has + // to name those columns instead. + let fx = Fixture::new(ACTIVITIES); + let catalog = log_catalog(); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("log", "log")], + vec![single( + "log", + "activities", + None, + Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![AttributeMapping { + source_column: "activity".into(), + name: "activity".into(), + value_type: None, + }], + objects: vec![], + }, + )], + ); + let (_, extractor, sql) = run_consolidated_only(&fx, &bp, &catalog); + assert!( + extractor + .event_attributes + .iter() + .any(|(_, name, _)| name == "activity"), + "the fixture must actually declare an event attribute: {:?}", + extractor.event_attributes + ); + assert_consolidated_agrees(&extractor, &sql); +} + +#[test] +fn case_9_the_same_blueprint_agrees_under_both_emission_shapes() { + // Reuses case 6's mix of static object attributes, a typed event attribute and a `NULL` + // observation, where a shape-specific bug is most likely to show. + let fx = Fixture::new(ATTRS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("items", "items")], + vec![ + single( + "items", + "item", + None, + Target::Object { + object_type: constant("Item"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "name".into(), + name: "name".into(), + value_type: None, + }], + }, + ), + single( + "items", + "listed", + None, + Target::Event { + event_type: constant("Listed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![AttributeMapping { + source_column: "price".into(), + name: "price".into(), + value_type: Some(OCELAttributeType::Float), + }], + objects: vec![], + }, + ), + ], + ); + let run = assert_agrees(&fx, &bp, &attrs_catalog()); + assert!( + run.compiled.errors().is_empty(), + "{:?}", + run.compiled.errors() + ); + assert!( + run.consolidated.errors().is_empty(), + "{:?}", + run.consolidated.errors() + ); + // `Null`-valued event attributes are dropped from both sides for the same reason + // `assert_consolidated_agrees` drops them. + assert_eq!(run.sql.events, run.consolidated_sql.events); + assert_eq!( + drop_null_event_attrs(&run.sql.event_attributes), + drop_null_event_attrs(&run.consolidated_sql.event_attributes), + "the two shapes must agree with each other, not merely with the extractor separately" + ); + assert_eq!(run.sql.objects, run.consolidated_sql.objects); + assert_eq!( + run.sql.object_attributes, + run.consolidated_sql.object_attributes + ); + assert_eq!(run.sql.e2o, run.consolidated_sql.e2o); + assert_eq!(run.sql.o2o, run.consolidated_sql.o2o); + // The relation names genuinely differ, so this is not two views of the same SQL. + let per_type_names: BTreeSet<&str> = run + .compiled + .relations() + .iter() + .map(|v| v.name.as_str()) + .collect(); + let consolidated_names: BTreeSet<&str> = run + .consolidated + .relations() + .iter() + .map(|v| v.name.as_str()) + .collect(); + assert!( + per_type_names.is_disjoint(&consolidated_names), + "PerType: {per_type_names:?}, Consolidated: {consolidated_names:?}" + ); +} + +fn case_11_blueprint() -> Blueprint { + blueprint( + IdRendering::TypePrefixed, + vec![source("items", "items")], + vec![ + single( + "items", + "item", + None, + Target::Object { + object_type: constant("Item"), + id: col("id"), + timestamp: None, + attributes: vec![ + AttributeMapping { + source_column: "name".into(), + name: "name".into(), + value_type: None, + }, + AttributeMapping { + source_column: "active".into(), + name: "active".into(), + value_type: Some(OCELAttributeType::Boolean), + }, + ], + }, + ), + single( + "items", + "listed", + None, + Target::Event { + event_type: constant("Listed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![AttributeMapping { + source_column: "price".into(), + name: "price".into(), + value_type: Some(OCELAttributeType::Float), + }], + objects: vec![], + }, + ), + single( + "items", + "listed-item", + None, + Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Listed")), + }, + object: endpoint("id", "Item"), + qualifier: Some(constant("subject")), + }, + ), + ], + ) +} + +#[test] +fn case_11_all_four_consolidated_emission_paths_agree() { + let bp = case_11_blueprint(); + let catalog = attrs_catalog(); + + let fx_extract = Fixture::new(ATTRS); + assert_eq!(validate(&bp, &catalog), vec![], "blueprint must validate"); + let provider = DuckDbRowProvider { + con: &fx_extract.con, + }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let extractor = from_extractor(sink.ocel()); + // Sanity: every relation this blueprint touches actually has rows, or the four-way + // comparison below would be vacuous. + assert!(!extractor.events.is_empty()); + assert!(!extractor.objects.is_empty()); + assert!(!extractor.e2o.is_empty()); + assert!(!extractor.object_attributes.is_empty()); + + let compiled = compile( + &bp, + &catalog, + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert!(compiled.errors().is_empty(), "{:?}", compiled.errors()); + + // Path 1: `ddl` (`CREATE VIEW`). + let fx_ddl = Fixture::new(ATTRS); + let ddl = compiled.ddl(); + fx_ddl + .con + .execute_batch(&ddl) + .unwrap_or_else(|e| panic!("ddl failed: {e}\n---\n{ddl}")); + assert_probes_hold(&fx_ddl.con, &compiled, "Consolidated ddl"); + assert_consolidated_agrees(&extractor, &from_sql_consolidated(&fx_ddl.con)); + + // Path 2: `materialize_ddl` (`CREATE TABLE ... AS`), against a fresh database with write + // rights, so a relation referenced more than once is computed once instead of re-inlined. + let fx_mat = Fixture::new(ATTRS); + let materialize = compiled.materialize_ddl(); + fx_mat + .con + .execute_batch(&materialize) + .unwrap_or_else(|e| panic!("materialize_ddl failed: {e}\n---\n{materialize}")); + assert_consolidated_agrees(&extractor, &from_sql_consolidated(&fx_mat.con)); + + // Path 3: `with_prelude`, every relation bound as a CTE in front of a read-only analysis + // query, so it runs with no `CREATE` right at all. + let fx_pre = Fixture::new(ATTRS); + let analysis = "SELECT \ + (SELECT count(*) FROM events) AS n_events, \ + (SELECT count(*) FROM objects) AS n_objects, \ + (SELECT count(*) FROM e2o) AS n_e2o, \ + (SELECT count(*) FROM object_attribute_changes) AS n_object_attrs"; + let (_, rows) = query(&fx_pre.con, &compiled.with_prelude(analysis)); + assert_eq!(rows.len(), 1); + let as_i64 = |v: &Value| match v { + Value::Integer(i) => *i, + other => panic!("expected an integer count, got {other:?}"), + }; + assert_eq!( + as_i64(&rows[0][0]), + extractor.events.len() as i64, + "with_prelude: n_events" + ); + assert_eq!( + as_i64(&rows[0][1]), + extractor.objects.len() as i64, + "with_prelude: n_objects" + ); + assert_eq!( + as_i64(&rows[0][2]), + extractor.e2o.len() as i64, + "with_prelude: n_e2o" + ); + assert_eq!( + as_i64(&rows[0][3]), + extractor.object_attributes.len() as i64, + "with_prelude: n_object_attrs" + ); + + // Path 4: the probes, run view-free (`probe_statements`) against yet another fresh + // connection, so an already-materialized relation cannot be hiding a probe bug. + let fx_probes = Fixture::new(ATTRS); + for (kind, sql) in probes_with_sql(&compiled) { + let (_, rows) = query(&fx_probes.con, &sql); + assert!( + rows.is_empty(), + "probe {kind:?} must hold view-free too: {} rows\n{sql}", + rows.len() + ); + } +} + +#[test] +fn case_10_a_skipped_mapping_is_reported_and_its_entities_are_the_only_difference() { + let fx = Fixture::new(ORDERS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("orders", "orders")], + vec![ + order_object(), + // No `id`, so the extractor mints a UUID per row: nondeterministic, and refused. + single( + "orders", + "minted", + None, + Target::Event { + event_type: constant("Placed"), + id: None, + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + ), + ], + ); + let run = run_both_expecting_errors(&fx, &bp, &orders_catalog()); + + assert_eq!( + run.compiled.errors().len(), + 1, + "{:?}", + run.compiled.errors() + ); + let err = &run.compiled.errors()[0]; + assert!(matches!( + err.reason, + RejectReason::SynthesizedId { field: "id" } + )); + assert_eq!( + err.mapping.as_ref().and_then(|m| m.label.clone()), + Some("minted".to_string()) + ); + + // The difference is exactly the skipped mapping's events, demonstrated rather than hidden. + assert_eq!(run.extractor.events.len(), 4, "{:?}", run.extractor.events); + assert!(run.extractor.events.iter().all(|(_, t, _)| t == "Placed")); + assert!( + run.sql.events.is_empty(), + "the skipped mapping's events must be absent from the SQL: {:?}", + run.sql.events + ); + // Everything else still agrees. + assert_eq!(run.extractor.objects, run.sql.objects); + assert_eq!(run.extractor.e2o, run.sql.e2o); + assert_eq!(run.extractor.o2o, run.sql.o2o); + + // `Consolidated` skips the same mapping for the same reason: `SynthesizedId` is decided + // before any shape-specific logic runs, so it shows the identical difference. + assert_eq!( + run.consolidated.errors().len(), + 1, + "{:?}", + run.consolidated.errors() + ); + assert!(matches!( + run.consolidated.errors()[0].reason, + RejectReason::SynthesizedId { field: "id" } + )); + assert!( + run.consolidated_sql.events.is_empty(), + "{:?}", + run.consolidated_sql.events + ); + assert_eq!(run.extractor.objects, run.consolidated_sql.objects); + assert_eq!(run.extractor.e2o, run.consolidated_sql.e2o); + assert_eq!(run.extractor.o2o, run.consolidated_sql.o2o); +} + +const CROSS_KIND: &str = " +CREATE TABLE left_side (k VARCHAR); +CREATE TABLE right_side (k BIGINT); +CREATE TABLE right_text (k VARCHAR); +INSERT INTO left_side VALUES ('1'), ('2'); +INSERT INTO right_side VALUES (1), (2); +INSERT INTO right_text VALUES ('1'), ('2'); +"; + +fn cross_kind_catalog() -> ExtractionCatalog { + ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("left_side", [("k", "VARCHAR", false)]), + ) + .with_table( + "db", + TableSchema::new("right_side", [("k", "BIGINT", false)]), + ) + .with_table( + "db", + TableSchema::new("right_text", [("k", "VARCHAR", false)]), + ) +} + +/// `left_side` joined to `right` on `k`, with every matched row becoming a `Joined` object. +fn join_blueprint(right: &str) -> Blueprint { + blueprint( + IdRendering::TypePrefixed, + vec![ + source("l", "left_side"), + source("r", right), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("k".into(), "k".into())], + }, + }, + ], + vec![single( + "j", + "joined", + None, + Target::Object { + object_type: constant("Joined"), + id: col("k"), + timestamp: None, + attributes: vec![], + }, + )], + ) +} + +#[test] +fn a_same_kind_join_key_does_produce_rows() { + // The control for `a_text_versus_number_join_key_produces_no_rows_on_both_sides`, whose + // emptiness assertions would also hold against a compiler emitting `ON FALSE` for every join. + let fx = Fixture::new(CROSS_KIND); + let run = assert_agrees(&fx, &join_blueprint("right_text"), &cross_kind_catalog()); + assert_eq!( + run.extractor.objects.len(), + 2, + "{:?}", + run.extractor.objects + ); + assert_eq!(run.sql.objects.len(), 2, "{:?}", run.sql.objects); +} + +#[test] +fn a_text_versus_number_join_key_produces_no_rows_on_both_sides() { + // `graph.rs` keys each join column through `Value::join_key_part`, which tags the runtime + // value's kind: `s:1` never matches `n:1`. DuckDB would implicit-cast and join them, so a + // bare `l.k = r.k` would make the view keep two rows the extractor refuses. + let fx = Fixture::new(CROSS_KIND); + let run = assert_agrees(&fx, &join_blueprint("right_side"), &cross_kind_catalog()); + assert!( + run.extractor.objects.is_empty(), + "the extractor's kind-tagged keys must not match: {:?}", + run.extractor.objects + ); + assert!( + run.sql.objects.is_empty(), + "and the compiled join must say so rather than inherit DuckDB's implicit cast: {:?}", + run.sql.objects + ); +} + +/// Serialises the panic-hook swap [`comparison_rejects`] performs, so two of these running +/// concurrently cannot leave the suppressing hook installed for a genuine failure elsewhere. +static HOOK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Whether `compare`, one of the real comparison entry points, rejects a deliberately tampered +/// log. +/// +/// The default panic hook is suppressed for the duration: these panics are the expected result, +/// and printing six backtraces during a passing run would make a green suite look broken. +fn comparison_rejects(compare: impl FnOnce() + std::panic::UnwindSafe) -> bool { + let _guard = HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let outcome = std::panic::catch_unwind(compare); + std::panic::set_hook(previous); + outcome.is_err() +} + +/// One deliberate corruption of a compiled log, applied to a single [`LogSets`] field. +type Perturbation = fn(&mut LogSets); + +/// A blueprint filling all six [`LogSets`] fields at once, so a tampering test can perturb each +/// one in turn against a log that genuinely carries it. +fn every_relation_blueprint() -> Blueprint { + blueprint( + IdRendering::TypePrefixed, + vec![source("items", "items")], + vec![ + single( + "items", + "item", + None, + Target::Object { + object_type: constant("Item"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "name".into(), + name: "name".into(), + value_type: None, + }], + }, + ), + single( + "items", + "label", + None, + Target::Object { + object_type: constant("Label"), + id: col("name"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "items", + "listed", + None, + Target::Event { + event_type: constant("Listed"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![AttributeMapping { + source_column: "price".into(), + name: "price".into(), + value_type: Some(OCELAttributeType::Float), + }], + objects: vec![], + }, + ), + single( + "items", + "listed-item", + None, + Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Listed")), + }, + object: endpoint("id", "Item"), + qualifier: Some(constant("subject")), + }, + ), + single( + "items", + "item-label", + None, + Target::O2O { + source: endpoint("id", "Item"), + target: endpoint("name", "Label"), + qualifier: Some(constant("named")), + }, + ), + ], + ) +} + +#[test] +fn the_harness_notices_when_the_two_logs_disagree() { + // Drives the real comparison entry points against a log tampered in one field at a time, so + // every `assert_eq!` inside `assert_log_sets_agree` is individually load-bearing. + let fx = Fixture::new(ATTRS); + let run = assert_agrees(&fx, &every_relation_blueprint(), &attrs_catalog()); + + // Tampering only means something against a field that actually carries rows. + assert!(!run.extractor.events.is_empty()); + assert!(!run.extractor.event_attributes.is_empty()); + assert!(!run.extractor.objects.is_empty()); + assert!(!run.extractor.object_attributes.is_empty()); + assert!(!run.extractor.e2o.is_empty()); + assert!(!run.extractor.o2o.is_empty()); + + let extractor = &run.extractor; + let perturbations: Vec<(&str, Perturbation)> = vec![ + ("events", |s| { + s.events.insert(( + "Listed-999".into(), + "Listed".into(), + "2024-09-09T00:00:00+00:00".into(), + )); + }), + ("event attributes", |s| { + s.event_attributes + .insert(("Listed-1".into(), "price".into(), "f:999.0".into())); + }), + ("objects", |s| { + s.objects.insert(("Item-999".into(), "Item".into())); + }), + ("object attributes", |s| { + s.object_attributes.insert(( + "Item-1".into(), + "name".into(), + "1970-01-01T00:00:00+00:00".into(), + "s:tampered".into(), + )); + }), + ("E2O", |s| { + s.e2o + .insert(("Listed-1".into(), "Item-999".into(), "subject".into())); + }), + ("O2O", |s| { + s.o2o + .insert(("Item-1".into(), "Label-999".into(), "named".into())); + }), + ]; + for (field, perturb) in perturbations { + let mut tampered = run.sql.clone(); + perturb(&mut tampered); + assert!( + comparison_rejects(|| assert_per_type_agrees(extractor, &tampered)), + "assert_per_type_agrees accepted a log whose {field} were tampered with" + ); + } + + // `assert_consolidated_agrees` compares event attributes through a different `assert_eq!`, + // the `drop_null_event_attributes` branch, so it needs its own case. + let mut tampered = run.consolidated_sql.clone(); + tampered + .event_attributes + .insert(("Listed-1".into(), "price".into(), "f:999.0".into())); + assert!( + comparison_rejects(|| assert_consolidated_agrees(extractor, &tampered)), + "assert_consolidated_agrees accepted a tampered non-null event attribute" + ); + + // And the carve-out really is a carve-out, not a hole: a `Null`-valued extra observation is + // rejected by `PerType`'s comparison and deliberately tolerated by `Consolidated`'s. + let mut null_valued = run.sql.clone(); + null_valued + .event_attributes + .insert(("Listed-1".into(), "invented".into(), "".into())); + assert!( + comparison_rejects(|| assert_per_type_agrees(extractor, &null_valued)), + "PerType compares event attributes in full, `Null` observations included" + ); +} + +const AMBIGUOUS: &str = " +CREATE TABLE dup (id BIGINT, ts TIMESTAMP, note VARCHAR); +INSERT INTO dup VALUES + (1, TIMESTAMP '2020-01-01 00:00:00', 'first'), + (1, TIMESTAMP '2020-01-02 00:00:00', 'second'); +"; + +fn ambiguous_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "dup", + [ + ("id", "BIGINT", false), + ("ts", "TIMESTAMP", false), + ("note", "VARCHAR", true), + ], + ), + ) +} + +#[test] +fn every_probe_kind_is_shown_to_fire() { + // `assert_probes_hold` only asserts a probe returns nothing, which a probe selecting nothing + // at all satisfies vacuously. This drives each `ProbeKind` into returning rows on a fixture + // that genuinely violates what it guards. + // + // Not `run_both`: the extractor answers these inputs with an `IdTypeCollision` or a + // first-row-wins choice, which `run_both` refuses to compare across. + let fx = Fixture::new(AMBIGUOUS); + let bp = blueprint( + // `Raw`, so the two object mappings really do claim the same id under two types. + IdRendering::Raw, + vec![source("dup", "dup")], + vec![ + single( + "dup", + "as-a", + None, + Target::Object { + object_type: constant("A"), + id: col("id"), + timestamp: None, + // Two rows, one id, different values: `AmbiguousStaticObjectAttributes`. + attributes: vec![AttributeMapping { + source_column: "note".into(), + name: "note".into(), + value_type: Some(OCELAttributeType::String), + }], + }, + ), + single( + "dup", + "as-b", + None, + Target::Object { + object_type: constant("B"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + ), + single( + "dup", + "e1", + None, + Target::Event { + event_type: constant("E1"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + ), + single( + "dup", + "e2", + None, + Target::Event { + event_type: constant("E2"), + id: Some(col("id")), + timestamp: ts("ts"), + attributes: vec![], + objects: vec![], + }, + ), + ], + ); + let catalog = ambiguous_catalog(); + assert_eq!(validate(&bp, &catalog), vec![], "blueprint must validate"); + + let compiled = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert!(compiled.errors().is_empty(), "{:?}", compiled.errors()); + let ddl = compiled.ddl(); + fx.con + .execute_batch(&ddl) + .unwrap_or_else(|e| panic!("ddl failed: {e}\n---\n{ddl}")); + + // The fourth `ProbeKind`, `StaleTypeDomain`, is driven into firing by + // `case_8_the_staleness_probe_fires_once_a_value_outside_the_domain_is_inserted`. + let firing = firing_probe_kinds(&fx.con, &compiled); + for expected in [ + "AmbiguousObjectIdentity", + "AmbiguousEventIdentity", + "AmbiguousStaticObjectAttributes", + ] { + assert!( + firing.contains(expected), + "probe {expected} never fires, so asserting it returns zero rows proves nothing: \ + firing = {firing:?}" + ); + } + + // The extractor really does disagree here: the probes announce a genuine divergence. + let provider = DuckDbRowProvider { con: &fx.con }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert!( + report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::IdTypeCollision { .. })), + "{:?}", + report.errors + ); + assert_ne!( + from_extractor(sink.ocel()).objects, + from_sql(&fx.con).objects, + "the extractor drops the losing object and the views keep it, which is what \ + AmbiguousObjectIdentity announces" + ); +} + +const BOOL_FLAGS: &str = " +CREATE TABLE flags (order_id INTEGER, is_cancelled BOOLEAN); +INSERT INTO flags VALUES + (5, NULL), + (6, false), + (7, true); +"; + +fn bool_flags_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "flags", + [ + ("order_id", "INTEGER", false), + ("is_cancelled", "BOOLEAN", true), + ], + ), + ) +} + +#[test] +fn a_null_boolean_in_a_template_id_drops_the_row_rather_than_rendering_false() { + // `render_template` returns `None` as soon as a placeholder has no `Value::canonical_string`, + // and `Value::Null` has none, so row 5 has no id and the extractor drops it. A + // `CASE WHEN THEN 'true' ELSE 'false' END` would take the ELSE branch for `NULL` and + // mint the id `5-false`, past the caller's `IS NOT NULL` and `<> ''` guards. + let fx = Fixture::new(BOOL_FLAGS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("flags", "flags")], + vec![single( + "flags", + "order", + None, + Target::Object { + object_type: constant("Order"), + id: ValueExpression::Template { + template: "{order_id}-{is_cancelled}".to_string(), + }, + timestamp: None, + attributes: vec![], + }, + )], + ); + let run = assert_agrees(&fx, &bp, &bool_flags_catalog()); + let ids: BTreeSet<&str> = run + .extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect(); + assert_eq!( + ids, + BTreeSet::from(["Order-6-false", "Order-7-true"]), + "row 5's NULL flag must render no id at all" + ); +} + +#[test] +fn matches_against_a_null_boolean_column_is_false_rather_than_matching_false() { + // `Predicate::Matches` reads the cell through `Value::display_string().is_some_and(..)`, and + // `Value::Null`'s is `None`, so row 5 does not match `^false$`. Rendering the column as the + // literal text `false` first would make it match, keeping a row the extractor drops. + let fx = Fixture::new(BOOL_FLAGS); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![source("flags", "flags")], + vec![single( + "flags", + "cancelled", + Some(Predicate::Matches { + column: "is_cancelled".to_string(), + regex: "^false$".to_string(), + }), + Target::Object { + object_type: constant("NotCancelled"), + id: col("order_id"), + timestamp: None, + attributes: vec![], + }, + )], + ); + let run = assert_agrees(&fx, &bp, &bool_flags_catalog()); + assert_eq!( + run.extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + BTreeSet::from(["NotCancelled-6"]), + "only the genuine `false` row matches" + ); +} + +// C2: a naive TIMESTAMP/DATE column read in a predicate or a join key, under a session zone +// that is not UTC. +// +// Every other fixture pins `TimeZone='UTC'`, under which an unanchored `src."ts"` and an anchored +// `timezone('UTC', CAST(src."ts" AS TIMESTAMP))` agree by accident. These run at +// `America/New_York` so that accident is removed. + +const NAIVE_TIMES: &str = " +CREATE TABLE evs (id BIGINT, ts TIMESTAMP, d DATE); +INSERT INTO evs VALUES + (1, TIMESTAMP '2020-01-01 12:00:00', DATE '2020-01-01'), + (2, TIMESTAMP '2020-01-02 12:00:00', DATE '2020-01-02'); +"; + +fn naive_times_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "BIGINT", false), + ("ts", "TIMESTAMP", false), + ("d", "DATE", false), + ], + ), + ) +} + +/// One object per row of `evs` surviving `when`. +fn naive_times_blueprint(when: Predicate) -> Blueprint { + blueprint( + IdRendering::TypePrefixed, + vec![source("evs", "evs")], + vec![single( + "evs", + "kept", + Some(when), + Target::Object { + object_type: constant("Kept"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + )], + ) +} + +#[test] +fn a_naive_timestamp_compared_in_a_predicate_is_read_as_utc_not_in_the_session_zone() { + // The extractor's providers report a naive column as a `Value::Timestamp` at UTC, so + // `2020-01-01 12:00:00` is 12:00Z and is not after 12:30Z. Emitting a bare `src."ts"` + // hands DuckDB a naive TIMESTAMP next to a TIMESTAMPTZ, which it promotes using the session + // zone: at America/New_York that reads 17:00Z, which is after 12:30Z, and the view keeps a + // row the extractor drops. + let fx = Fixture::with_time_zone(NAIVE_TIMES, "America/New_York"); + let bp = naive_times_blueprint(Predicate::Compare { + left: Operand::Column { + column: "ts".to_string(), + }, + op: CompareOp::Gt, + right: Operand::Literal { + value: Literal::Text("2020-01-01T12:30:00Z".to_string()), + }, + }); + let run = assert_agrees(&fx, &bp, &naive_times_catalog()); + assert_eq!( + run.extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + BTreeSet::from(["Kept-2"]), + "row 1 is 12:00Z, which is not after 12:30Z" + ); +} + +#[test] +fn a_naive_timestamp_in_an_in_list_is_read_as_utc_not_in_the_session_zone() { + // `Predicate::In` has its own emission path (`in_sql`), which read the column just as bare as + // `operand_sql` did. + let fx = Fixture::with_time_zone(NAIVE_TIMES, "America/New_York"); + let bp = naive_times_blueprint(Predicate::In { + column: "ts".to_string(), + values: vec![Literal::Text("2020-01-01T12:00:00Z".to_string())], + }); + let run = assert_agrees(&fx, &bp, &naive_times_catalog()); + assert_eq!( + run.extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + BTreeSet::from(["Kept-1"]), + "row 1's instant is exactly the literal" + ); +} + +#[test] +fn a_naive_date_compared_in_a_predicate_is_read_as_utc_not_in_the_session_zone() { + // `ColumnSchema::declared_kind` maps DATE to `ValueKind::Timestamp` too, so a DATE column + // reaches the identical emission path and shifts the identical way. + let fx = Fixture::with_time_zone(NAIVE_TIMES, "America/New_York"); + let bp = naive_times_blueprint(Predicate::Compare { + left: Operand::Column { + column: "d".to_string(), + }, + op: CompareOp::Lt, + right: Operand::Literal { + value: Literal::Text("2020-01-01T02:00:00Z".to_string()), + }, + }); + let run = assert_agrees(&fx, &bp, &naive_times_catalog()); + assert_eq!( + run.extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + BTreeSet::from(["Kept-1"]), + "row 1's date is UTC midnight, which is before 02:00Z. Read in the session zone it \ + would be 05:00Z and the view would drop the row. The boundary sits deliberately \ + between the two readings" + ); +} + +const MIXED_TZ_JOIN: &str = " +CREATE TABLE naive_side (k TIMESTAMP, tag VARCHAR); +CREATE TABLE tz_side (k TIMESTAMPTZ, label VARCHAR); +INSERT INTO naive_side VALUES (TIMESTAMP '2020-06-01 12:00:00', 'n'); +INSERT INTO tz_side VALUES (TIMESTAMPTZ '2020-06-01 12:00:00+00', 'utc-noon'); +"; + +#[test] +fn a_naive_to_tz_join_key_matches_on_the_instant_not_on_the_session_reading() { + // `graph.rs` keys both sides through `Value::join_key_part`, which sees two `Value::Timestamp` + // at the same instant and pairs the rows. A bare `l.k = r.k` promotes the naive side using + // the session zone (16:00Z at America/New_York in June) and the join finds nothing. + let fx = Fixture::with_time_zone(MIXED_TZ_JOIN, "America/New_York"); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "naive_side", + [("k", "TIMESTAMP", false), ("tag", "VARCHAR", false)], + ), + ) + .with_table( + "db", + TableSchema::new( + "tz_side", + [("k", "TIMESTAMPTZ", false), ("label", "VARCHAR", false)], + ), + ); + let bp = blueprint( + IdRendering::TypePrefixed, + vec![ + source("n", "naive_side"), + source("t", "tz_side"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "n".into(), + right: "t".into(), + on: vec![("k".into(), "k".into())], + }, + }, + ], + vec![single( + "j", + "paired", + None, + Target::Object { + object_type: constant("Paired"), + id: col("label"), + timestamp: None, + attributes: vec![], + }, + )], + ); + let run = assert_agrees(&fx, &bp, &catalog); + assert_eq!( + run.extractor + .objects + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + BTreeSet::from(["Paired-utc-noon"]), + "the two keys are the same instant, so the join must pair them" + ); +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/compile/emit.rs b/process_mining/src/core/event_data/object_centric/extraction/compile/emit.rs new file mode 100644 index 00000000..836b0201 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/compile/emit.rs @@ -0,0 +1,969 @@ +//! Row-level emission: a node's rows as SQL, and one row's expressions, predicates, timestamps +//! and splits as SQL fragments. +//! +//! Everything here mirrors a specific piece of the extractor, named in the implementing function's +//! doc comment. Where the two could disagree the function returns a [`RejectReason`] rather than +//! guessing. +//! +//! The extractor decides literal coercion, identity rendering and join-key matching from the +//! runtime [`Value`] in a cell, while a compiler only has [`ColumnSchema::declared_kind`]. Every +//! kind-dependent rule below therefore assumes the catalog describes the kinds a source's values +//! actually have, which holds for statically typed engines but not for `SQLite`. + +use std::collections::HashMap; + +use super::dialect::SqlDialect; +use super::RejectReason; +use crate::core::event_data::object_centric::extraction::blueprint::{Blueprint, NodeOp}; +use crate::core::event_data::object_centric::extraction::catalog::{ColumnSchema, TableSchema}; +use crate::core::event_data::object_centric::extraction::expr::{ + SplitKind, SplitSpec, TimestampFormat, TimestampSource, ValueExpression, +}; +use crate::core::event_data::object_centric::extraction::predicate::{ + prepare_literal, CompareOp, Literal, Operand, Predicate, +}; +use crate::core::event_data::object_centric::extraction::row::{build_column_index, Row}; +use crate::core::event_data::object_centric::extraction::schema::{join_column_source, JoinSide}; +use crate::core::event_data::object_centric::extraction::value::ValueKind; + +/// The alias every emitted fragment qualifies its column references with. +pub(crate) const ROW_ALIAS: &str = "src"; + +/// A node's rows as SQL, plus the declared shape those rows have. +/// +/// Holds the same per-node column resolution the extractor uses +/// ([`full_node_schemas`](crate::core::event_data::object_centric::extraction::schema::full_node_schemas)), +/// so a `Join`'s `right_` columns and a `Union`'s null-filled +/// ones carry the identical names on both sides. +#[derive(Debug)] +pub(crate) struct Emitter<'a, 'b> { + pub(crate) dialect: SqlDialect, + blueprint: &'a Blueprint, + full: &'b HashMap<&'a str, TableSchema>, +} + +impl<'a, 'b> Emitter<'a, 'b> { + /// Borrow a caller's already-resolved + /// [`full_node_schemas`](crate::core::event_data::object_centric::extraction::schema::full_node_schemas) + /// map. Borrowed rather than owned because the push-down path builds one emitter per join per + /// consumer per phase, and the map holds every node's whole column list. + pub(crate) fn from_schemas( + blueprint: &'a Blueprint, + full: &'b HashMap<&'a str, TableSchema>, + dialect: SqlDialect, + ) -> Self { + Self { + dialect, + blueprint, + full, + } + } + + /// The declared shape of `node_id`'s rows, or `None` when the node does not exist or its + /// schema could not be resolved (an unknown source table, say). + pub(crate) fn schema_of(&self, node_id: &str) -> Option<&TableSchema> { + self.full.get(node_id) + } + + /// A bare `SELECT` producing every column of `node_id`, in the same order and under the same + /// names the extractor's own row layout uses. + pub(crate) fn node_sql(&self, node_id: &str) -> Result { + self.node_sql_inner(node_id, &mut Vec::new()) + } + + fn node_sql_inner( + &self, + node_id: &str, + visiting: &mut Vec, + ) -> Result { + if visiting.iter().any(|v| v == node_id) { + return Err(RejectReason::NodeCycle { + node: node_id.to_string(), + }); + } + let node = self + .blueprint + .node(node_id) + .ok_or_else(|| RejectReason::UnknownNode { + node: node_id.to_string(), + })?; + let schema = self + .schema_of(node_id) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: node_id.to_string(), + })?; + let columns: Vec<&str> = schema.columns.keys().map(String::as_str).collect(); + if columns.is_empty() { + return Err(RejectReason::EmptyProjection { + node: node_id.to_string(), + }); + } + + visiting.push(node_id.to_string()); + let sql = match &node.op { + NodeOp::Source { table, .. } => { + let cols: Vec = columns + .iter() + .map(|c| self.dialect.quote_ident(c)) + .collect(); + Ok(format!( + "SELECT {} FROM {}", + cols.join(", "), + self.dialect.quote_ident(table) + )) + } + NodeOp::Filter { input, condition } => self.filter_sql(input, condition, visiting), + NodeOp::Union { inputs } => self.union_sql(node_id, inputs, &columns, visiting), + NodeOp::Join { left, right, on } => { + self.join_sql(node_id, left, right, on, &columns, visiting) + } + }; + visiting.pop(); + sql + } + + /// A `Filter` narrows rows and never columns, exactly as `WHERE` does. + fn filter_sql( + &self, + input: &str, + condition: &Predicate, + visiting: &mut Vec, + ) -> Result { + let inner = self.node_sql_inner(input, visiting)?; + let input_schema = + self.schema_of(input) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: input.to_string(), + })?; + let cols: Vec = input_schema + .columns + .keys() + .map(|c| format!("{ROW_ALIAS}.{}", self.dialect.quote_ident(c))) + .collect(); + let where_sql = predicate_sql(self.dialect, condition, input_schema, ROW_ALIAS)?; + Ok(format!( + "SELECT {} FROM {} WHERE {where_sql}", + cols.join(", "), + self.dialect.derived_table(&inner, ROW_ALIAS) + )) + } + + /// `UNION ALL` with the absent columns explicitly null-filled, as [`NodeOp::Union`] specifies. + fn union_sql( + &self, + node_id: &str, + inputs: &[String], + columns: &[&str], + visiting: &mut Vec, + ) -> Result { + if inputs.is_empty() { + return Err(RejectReason::EmptyUnion { + node: node_id.to_string(), + }); + } + let out_schema = self.schema_of(node_id).expect("checked by the caller"); + let mut branches = Vec::with_capacity(inputs.len()); + for input in inputs { + let inner = self.node_sql_inner(input, visiting)?; + let input_schema = + self.schema_of(input) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: input.to_string(), + })?; + let cols: Vec = columns + .iter() + .map(|c| { + let quoted = self.dialect.quote_ident(c); + if input_schema.columns.contains_key(*c) { + format!("{ROW_ALIAS}.{quoted} AS {quoted}") + } else { + let kind = out_schema + .columns + .get(*c) + .and_then(ColumnSchema::declared_kind); + match kind { + Some(k) => format!( + "CAST(NULL AS {}) AS {quoted}", + self.dialect.kind_sql_type(k) + ), + None => format!("NULL AS {quoted}"), + } + } + }) + .collect(); + branches.push(format!( + "SELECT {} FROM {}", + cols.join(", "), + self.dialect.derived_table(&inner, ROW_ALIAS) + )); + } + Ok(self.dialect.union_all(&branches)) + } + + /// An inner join whose output columns are routed by [`join_column_source`], the same rule the + /// extractor's own `GraphExecutor` uses, so `right_` cannot mean two things. + /// + /// The key comparison is not a bare `l.k = r.k`: [`Value::join_key_part`] tags each key with + /// its kind, so a `Text` `"1"` never matches an `Integer` `1` where `DuckDB` would + /// implicit-cast and join them. Under the module's catalog precondition, two declared kinds + /// that are equal or both numeric compile to an equality, any other pair to a constant false, + /// and an undeclared kind is rejected. + fn join_sql( + &self, + node_id: &str, + left: &str, + right: &str, + on: &[(String, String)], + columns: &[&str], + visiting: &mut Vec, + ) -> Result { + let left_sql = self.node_sql_inner(left, visiting)?; + let right_sql = self.node_sql_inner(right, visiting)?; + let l_schema = self + .schema_of(left) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: left.to_string(), + })?; + let r_schema = self + .schema_of(right) + .ok_or_else(|| RejectReason::UnresolvedNodeSchema { + node: right.to_string(), + })?; + + let mut conds: Vec = Vec::with_capacity(on.len()); + for (l, r) in on { + let (l_col, lk) = key_column(l_schema, l, node_id, "left")?; + let (r_col, rk) = key_column(r_schema, r, node_id, "right")?; + if lk == rk || (is_numeric(lk) && is_numeric(rk)) { + conds.push(format!( + "{} = {}", + column_read_sql(self.dialect, l_col, l, "l"), + column_read_sql(self.dialect, r_col, r, "r") + )); + } else { + conds.push(self.dialect.false_predicate().to_string()); + } + } + if conds.is_empty() { + // An empty `on` gives every left row the empty key, which every right row also has: + // a full cross product in the extractor, and `ON TRUE` here. + conds.push(self.dialect.true_predicate().to_string()); + } + + let mut select_cols = Vec::with_capacity(columns.len()); + for c in columns { + let quoted = self.dialect.quote_ident(c); + let projection = match join_column_source(c, l_schema, r_schema) { + Some((JoinSide::Left, source)) => { + format!("l.{} AS {quoted}", self.dialect.quote_ident(source)) + } + Some((JoinSide::Right, source)) => { + format!("r.{} AS {quoted}", self.dialect.quote_ident(source)) + } + // The executor raises a hard error here, so a null-filled column would be a view + // carrying rows no extraction can produce. + None => { + return Err(RejectReason::UnknownColumn { + column: (*c).to_string(), + field: "join output column", + }) + } + }; + select_cols.push(projection); + } + + Ok(format!( + "SELECT {} FROM {} INNER JOIN {} ON {}", + select_cols.join(", "), + self.dialect.derived_table(&left_sql, "l"), + self.dialect.derived_table(&right_sql, "r"), + conds.join(" AND ") + )) + } +} + +fn is_numeric(k: ValueKind) -> bool { + matches!(k, ValueKind::Integer | ValueKind::Float) +} + +fn key_column<'s>( + schema: &'s TableSchema, + column: &str, + node: &str, + side: &'static str, +) -> Result<(&'s ColumnSchema, ValueKind), RejectReason> { + let col = schema + .columns + .get(column) + .ok_or_else(|| RejectReason::UnknownColumn { + column: column.to_string(), + field: "join key", + })?; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndecidableJoinKey { + node: node.to_string(), + side, + column: column.to_string(), + col_type: col.col_type.clone(), + })?; + Ok((col, kind)) +} + +/// Whether a timestamp column carries no offset of its own (`TIMESTAMP`, `DATE`, `DATETIME`) and +/// so needs the explicit UTC anchor [`SqlDialect::timestamp_column`] applies. +fn is_naive_timestamp(col: &ColumnSchema) -> bool { + let lowered = col.col_type.to_ascii_lowercase(); + !(lowered.contains("tz") || lowered.contains("with time zone")) +} + +/// One column read as the extractor's providers report it. +/// +/// Everything except a timestamp is the bare qualified reference. A timestamp is an instant to +/// the extractor, so an offset-less column is anchored rather than compared as a bare `TIMESTAMP`, +/// which `DuckDB` promotes using the session time zone as soon as the other side is a +/// `TIMESTAMPTZ`. +fn column_read_sql(dialect: SqlDialect, col: &ColumnSchema, column: &str, alias: &str) -> String { + let q = format!("{alias}.{}", dialect.quote_ident(column)); + if col.declared_kind() == Some(ValueKind::Timestamp) { + dialect.timestamp_column(&q, is_naive_timestamp(col)) + } else { + q + } +} + +/// A [`Predicate`] as a SQL boolean that is never `NULL`. +/// +/// The extractor evaluates predicates in two-valued logic: an unresolvable comparison is `false`, +/// not "unknown". SQL is three-valued, so every leaf that can yield `NULL` is wrapped through +/// [`SqlDialect::total_bool`] before `AND`/`OR`/`NOT` ever see it. +pub(crate) fn predicate_sql( + dialect: SqlDialect, + predicate: &Predicate, + schema: &TableSchema, + alias: &str, +) -> Result { + match predicate { + Predicate::And { conditions } => { + if conditions.is_empty() { + return Ok(dialect.true_predicate().to_string()); + } + let parts = conditions + .iter() + .map(|c| predicate_sql(dialect, c, schema, alias)) + .collect::, _>>()?; + Ok(format!("({})", parts.join(" AND "))) + } + Predicate::Or { conditions } => { + if conditions.is_empty() { + return Ok(dialect.false_predicate().to_string()); + } + let parts = conditions + .iter() + .map(|c| predicate_sql(dialect, c, schema, alias)) + .collect::, _>>()?; + Ok(format!("({})", parts.join(" OR "))) + } + // Safe because the operand is total: see this function's docs. + Predicate::Not { condition } => Ok(format!( + "(NOT {})", + predicate_sql(dialect, condition, schema, alias)? + )), + Predicate::Compare { left, op, right } => { + compare_sql(dialect, left, *op, right, schema, alias) + } + Predicate::IsNull { column } => match schema.columns.get(column) { + // A column the row does not carry reads as `None`, which `IsNull` calls true. + None => Ok(dialect.true_predicate().to_string()), + Some(_) => Ok(format!("({alias}.{} IS NULL)", dialect.quote_ident(column))), + }, + Predicate::IsEmpty { column } => is_empty_sql(dialect, column, schema, alias), + Predicate::Matches { column, regex } => matches_sql(dialect, column, regex, schema, alias), + Predicate::In { column, values } => in_sql(dialect, column, values, schema, alias), + } +} + +/// `IsEmpty` is `NULL`, absent, or a [`Value::canonical_string`] of `""`. Only `Text` has a +/// canonical rendering that can be empty, which makes every other kind exactly "is null". +fn is_empty_sql( + dialect: SqlDialect, + column: &str, + schema: &TableSchema, + alias: &str, +) -> Result { + let Some(col) = schema.columns.get(column) else { + return Ok(dialect.true_predicate().to_string()); + }; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndeclaredColumnKind { + column: column.to_string(), + col_type: col.col_type.clone(), + field: "is-empty", + })?; + let q = format!("{alias}.{}", dialect.quote_ident(column)); + Ok(match kind { + ValueKind::Text => format!("({q} IS NULL OR {q} = '')"), + _ => format!("({q} IS NULL)"), + }) +} + +/// `Matches` reads the column through [`Value::display_string`]. +/// +/// That rendering is only reproducible in SQL for `Text`, `Integer` and `Boolean`: Rust's +/// `f64::to_string` writes `1` where `DuckDB` writes `1.0`, and `DateTime::to_rfc3339` keeps the +/// original offset where a SQL cast does not. +fn matches_sql( + dialect: SqlDialect, + column: &str, + regex: &str, + schema: &TableSchema, + alias: &str, +) -> Result { + let Some(col) = schema.columns.get(column) else { + // `row.get` is `None`, so `is_some_and` is false for every row. + return Ok(dialect.false_predicate().to_string()); + }; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndeclaredColumnKind { + column: column.to_string(), + col_type: col.col_type.clone(), + field: "matches", + })?; + let q = format!("{alias}.{}", dialect.quote_ident(column)); + let text = match kind { + ValueKind::Text => q, + ValueKind::Integer => dialect.cast_to_text(&q), + ValueKind::Boolean => dialect.bool_to_text(&q), + ValueKind::Float | ValueKind::Timestamp => { + return Err(RejectReason::UnstableDisplayRendering { + column: column.to_string(), + col_type: col.col_type.clone(), + field: "matches", + }) + } + }; + Ok(dialect.total_bool(&dialect.regex_match(&text, regex))) +} + +/// `In` coerces every literal to the column's declared kind independently, exactly as +/// [`Predicate::prepare`] does, and drops the ones that then cannot compare equal to a value of +/// that kind. An uncoercible literal matches nothing there either. +fn in_sql( + dialect: SqlDialect, + column: &str, + values: &[Literal], + schema: &TableSchema, + alias: &str, +) -> Result { + let Some(col) = schema.columns.get(column) else { + return Ok(dialect.false_predicate().to_string()); + }; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndeclaredColumnKind { + column: column.to_string(), + col_type: col.col_type.clone(), + field: "in", + })?; + let literals: Vec = values + .iter() + .map(|l| prepare_literal(l, Some(kind))) + .filter(|v| comparable(v.kind(), Some(kind))) + .filter_map(|v| dialect.value_literal(&v)) + .collect(); + if literals.is_empty() { + return Ok(dialect.false_predicate().to_string()); + } + let q = column_read_sql(dialect, col, column, alias); + Ok(dialect.total_bool(&format!("{q} IN ({})", literals.join(", ")))) +} + +/// Whether [`Value::compare`] can order these two kinds: identical kinds, or both numeric. +/// Anything else, and anything involving `Null`, makes the comparing predicate false. +fn comparable(a: Option, b: Option) -> bool { + match (a, b) { + (Some(a), Some(b)) => a == b || (is_numeric(a) && is_numeric(b)), + _ => false, + } +} + +fn op_sql(op: CompareOp) -> &'static str { + match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + } +} + +fn apply_op(op: CompareOp, ord: std::cmp::Ordering) -> bool { + match op { + CompareOp::Eq => ord.is_eq(), + CompareOp::Ne => ord.is_ne(), + CompareOp::Lt => ord.is_lt(), + CompareOp::Le => ord.is_le(), + CompareOp::Gt => ord.is_gt(), + CompareOp::Ge => ord.is_ge(), + } +} + +/// A typed comparison, reproducing [`Predicate::prepare`]'s coercion and +/// [`Value::compare`]'s "mismatched kinds do not order, so the predicate is false" rule. +fn compare_sql( + dialect: SqlDialect, + left: &Operand, + op: CompareOp, + right: &Operand, + schema: &TableSchema, + alias: &str, +) -> Result { + let lk = operand_kind(left, schema)?; + let rk = operand_kind(right, schema)?; + match (left, right) { + // Two literals fold at compile time: no column is read, so the answer is the same on + // every row. + (Operand::Literal { value: l }, Operand::Literal { value: r }) => { + let answer = l + .as_value() + .compare(&r.as_value()) + .is_some_and(|ord| apply_op(op, ord)); + Ok(if answer { + dialect.true_predicate().to_string() + } else { + dialect.false_predicate().to_string() + }) + } + _ => { + // `prepare` coerces each literal against the other side's declared column kind. + let (l_kind, r_kind) = ( + operand_value_kind(left, rk, schema), + operand_value_kind(right, lk, schema), + ); + if !comparable(l_kind, r_kind) { + // `Value::compare` returns `None`, and `CompareOp` turns that into false. Decided + // before either side is rendered, so a column the node does not carry degrades to + // a false predicate rather than refusing the whole mapping. + return Ok(dialect.false_predicate().to_string()); + } + let l_sql = operand_sql(dialect, left, rk, schema, alias)?; + let r_sql = operand_sql(dialect, right, lk, schema, alias)?; + Ok(dialect.total_bool(&format!("{l_sql} {} {r_sql}", op_sql(op)))) + } + } +} + +/// A column operand's declared kind, or `None` for a literal. Mirrors `column_kind` in +/// `predicate.rs`, which decides whether the other side's literal is coerced. +fn operand_kind( + operand: &Operand, + schema: &TableSchema, +) -> Result, RejectReason> { + match operand { + Operand::Literal { .. } => Ok(None), + Operand::Column { column } => { + match schema.columns.get(column) { + None => Ok(None), + Some(col) => col.declared_kind().map(Some).ok_or_else(|| { + RejectReason::UndeclaredColumnKind { + column: column.to_string(), + col_type: col.col_type.clone(), + field: "compare", + } + }), + } + } + } +} + +/// The kind the value on this side has once coercion has run. +fn operand_value_kind( + operand: &Operand, + other_side_kind: Option, + schema: &TableSchema, +) -> Option { + match operand { + Operand::Column { column } => schema + .columns + .get(column) + .and_then(ColumnSchema::declared_kind), + Operand::Literal { value } => prepare_literal(value, other_side_kind).kind(), + } +} + +fn operand_sql( + dialect: SqlDialect, + operand: &Operand, + other_side_kind: Option, + schema: &TableSchema, + alias: &str, +) -> Result { + match operand { + Operand::Column { column } => { + let col = schema + .columns + .get(column) + .ok_or_else(|| RejectReason::UnknownColumn { + column: column.clone(), + field: "compare", + })?; + Ok(column_read_sql(dialect, col, column, alias)) + } + Operand::Literal { value } => { + let v = prepare_literal(value, other_side_kind); + Ok(dialect + .value_literal(&v) + .unwrap_or_else(|| "NULL".to_string())) + } + } +} + +/// A [`ValueExpression`] as text, at an identity position. +/// +/// Mirrors [`ValueExpression::evaluate`], which reads every column through +/// [`Value::canonical_string`]: only `Text`, `Integer` and `Boolean` have one, and every variant +/// propagates absence. `NULL` propagation through `||` and `COALESCE` reproduces that exactly, so +/// the expression is `NULL` on precisely the rows the extractor drops. The caller adds the +/// `IS NOT NULL` filter. +pub(crate) fn identity_sql( + dialect: SqlDialect, + expr: &ValueExpression, + schema: &TableSchema, + alias: &str, + field: &'static str, +) -> Result { + match expr { + ValueExpression::Constant { value } => Ok(dialect.string_literal(value)), + ValueExpression::Column { column } => { + column_identity_sql(dialect, column, schema, alias, field) + } + ValueExpression::Template { template } => { + template_sql(dialect, template, schema, alias, field) + } + ValueExpression::Coalesce { parts } => { + if parts.is_empty() { + // `find_map` over nothing is `None`. + return Ok(dialect.null_text()); + } + let rendered = parts + .iter() + .map(|p| identity_sql(dialect, p, schema, alias, field)) + .collect::, _>>()?; + Ok(dialect.coalesce(&rendered)) + } + } +} + +fn column_identity_sql( + dialect: SqlDialect, + column: &str, + schema: &TableSchema, + alias: &str, + field: &'static str, +) -> Result { + let col = schema + .columns + .get(column) + .ok_or_else(|| RejectReason::UnknownColumn { + column: column.to_string(), + field, + })?; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndeclaredColumnKind { + column: column.to_string(), + col_type: col.col_type.clone(), + field, + })?; + let q = format!("{alias}.{}", dialect.quote_ident(column)); + match kind { + ValueKind::Text => Ok(q), + ValueKind::Integer => Ok(dialect.cast_to_text(&q)), + ValueKind::Boolean => Ok(dialect.bool_to_text(&q)), + // The extractor renders a whole-number Float as its integer and drops a fractional one, + // which no single SQL expression reproduces. Timestamp offset rendering varies the same + // way. + ValueKind::Float | ValueKind::Timestamp => Err(RejectReason::UnstableIdentityRendering { + column: column.to_string(), + col_type: col.col_type.clone(), + field, + }), + } +} + +/// `render_template` scans the template, substituting each `{name}` and returning `None` as +/// soon as one placeholder has no [`Value::canonical_string`]. `||` propagates `NULL` the same +/// way, so no per-placeholder fallback is emitted. +fn template_sql( + dialect: SqlDialect, + template: &str, + schema: &TableSchema, + alias: &str, + field: &'static str, +) -> Result { + let mut parts: Vec = Vec::new(); + let mut rest = template; + while let Some(open) = rest.find('{') { + let literal = &rest[..open]; + if !literal.is_empty() { + parts.push(dialect.string_literal(literal)); + } + let after = &rest[open + 1..]; + let Some(close) = after.find('}') else { + // `render_template`'s `?` on the missing '}': the whole expression is always `None`. + return Err(RejectReason::InvalidTemplate { + template: template.to_string(), + reason: "unterminated placeholder".to_string(), + }); + }; + let name = &after[..close]; + if name.is_empty() { + return Err(RejectReason::InvalidTemplate { + template: template.to_string(), + reason: "empty placeholder".to_string(), + }); + } + parts.push(column_identity_sql(dialect, name, schema, alias, field)?); + rest = &after[close + 1..]; + } + if !rest.is_empty() { + parts.push(dialect.string_literal(rest)); + } + match parts.len() { + 0 => Ok(dialect.string_literal("")), + 1 => Ok(parts.remove(0)), + _ => Ok(dialect.concat(&parts)), + } +} + +/// A compiled timestamp, carrying whether the extractor can still drop the row. +#[derive(Debug)] +pub(crate) enum TimeSql { + /// A constant folded at compile time: present on every row. + Literal(String), + /// A native timestamp column: `NULL` parses to `None`, dropping the row. + Column(String), + /// A constant that did not parse, so no row survives. + Never, +} + +impl TimeSql { + pub(crate) fn sql(&self, dialect: SqlDialect) -> String { + match self { + TimeSql::Literal(s) | TimeSql::Column(s) => s.clone(), + TimeSql::Never => dialect.null_timestamp(), + } + } + + /// The filter that keeps exactly the rows whose timestamp parsed. + pub(crate) fn filter(&self, dialect: SqlDialect) -> Option { + match self { + TimeSql::Literal(_) => None, + TimeSql::Column(s) => Some(format!("{s} IS NOT NULL")), + TimeSql::Never => Some(dialect.false_predicate().to_string()), + } + } +} + +fn reads_no_column(expr: &ValueExpression) -> bool { + let mut columns = std::collections::HashSet::new(); + expr.referenced_columns(&mut columns); + columns.is_empty() +} + +/// [`TimestampSource::parse`] as SQL. +/// +/// Two shapes compile, both inside [`TimestampSource::Value`]. A source that reads nothing from +/// the row folds at compile time, running the whole chrono cascade here for any format. A plain +/// `Column` whose declared type is already a timestamp is read directly, because `parse` +/// short-circuits on a [`Value::Timestamp`] before any string parsing. +/// +/// Everything else is a [`RejectReason::ResidualTimestamp`]: chrono's format cascade has no +/// SQL translation that can be proved identical. +pub(crate) fn timestamp_sql( + dialect: SqlDialect, + ts: &TimestampSource, + schema: &TableSchema, + alias: &str, +) -> Result { + match ts { + // Reads no column, so every row gets the same instant: fold it now. + TimestampSource::Value(part) if reads_no_column(&part.source) => { + let index = build_column_index(&[]); + let row = Row { + values: &[], + index: &index, + }; + Ok(match ts.parse(&row) { + Some(folded) => TimeSql::Literal(dialect.timestamp_literal(&folded)), + None => TimeSql::Never, + }) + } + TimestampSource::Value(part) => { + let ValueExpression::Column { column } = &part.source else { + return Err(RejectReason::ResidualTimestamp { + detail: "a timestamp composed from columns is parsed by chrono, not SQL" + .to_string(), + }); + }; + let col = schema + .columns + .get(column) + .ok_or_else(|| RejectReason::UnknownColumn { + column: column.to_string(), + field: "timestamp", + })?; + if col.declared_kind() == Some(ValueKind::Timestamp) { + return Ok(TimeSql::Column(column_read_sql( + dialect, col, column, alias, + ))); + } + let format = part.format.as_ref().unwrap_or(&TimestampFormat::Auto); + Err(RejectReason::ResidualTimestamp { + detail: format!( + "column '{column}' is declared {} rather than a timestamp, so the value goes \ + through chrono's {format:?} string parsing", + col.col_type + ), + }) + } + TimestampSource::Components { .. } => Err(RejectReason::ResidualTimestamp { + detail: "Components tries three chrono strategies in order".to_string(), + }), + } +} + +/// One endpoint's split, as a `FROM` target that yields one row per part plus the expression +/// naming that part. +#[derive(Debug)] +pub(crate) struct SplitSql { + /// The `FROM` target replacing the mapping's own, already aliased. + pub(crate) from: String, + /// The expression for one part, valid against `from`. + pub(crate) part: String, + /// Extra filters the split needs, in addition to the caller's own. + pub(crate) filters: Vec, +} + +/// `split_or_single` as SQL: the split parts, or the raw cell itself when there is no split. +/// +/// The caller has already filtered the raw cell to non-`NULL` and non-empty. The filters +/// returned here are the per-part ones on top. The `unnest` goes into a derived table so the +/// per-part column can be trimmed, filtered and referenced more than once, and so a second split +/// can nest over the first. +pub(crate) fn split_sql( + dialect: SqlDialect, + from: &str, + raw_expr: &str, + split: Option<&SplitSpec>, + part_column: &str, +) -> Result { + let Some(split) = split else { + return Ok(SplitSql { + from: from.to_string(), + part: raw_expr.to_string(), + filters: Vec::new(), + }); + }; + let ident = dialect.quote_ident(part_column); + let expanded = match &split.kind { + SplitKind::Delimiter { delimiter } => { + if delimiter.is_empty() { + // `PreparedSplit::split` returns the whole raw value for an empty delimiter. + let part = maybe_trim(dialect, raw_expr, split.trim); + return Ok(SplitSql { + from: from.to_string(), + filters: vec![format!("{part} <> ''")], + part, + }); + } + dialect.split_to_rows(raw_expr, delimiter) + } + SplitKind::Regex { pattern } => { + let compiled = regex::Regex::new(pattern).map_err(|e| RejectReason::InvalidRegex { + pattern: pattern.clone(), + message: e.to_string(), + })?; + // `captures_len` counts the implicit whole-match group, which `split` only reads + // when the pattern has no other one. + let groups = compiled.captures_len().saturating_sub(1); + dialect.regex_split_to_rows(raw_expr, pattern, groups) + } + }; + let part = maybe_trim(dialect, &format!("{ROW_ALIAS}.{ident}"), split.trim); + Ok(SplitSql { + // Reusing the input's own alias keeps a nested second split naming `src` too. + from: dialect.derived_table( + &format!("SELECT {ROW_ALIAS}.*, {expanded} AS {ident} FROM {from}"), + ROW_ALIAS, + ), + // `regexp_extract_all` yields `[NULL]` for a group that did not participate, where + // Rust's `caps.get(i)` yields nothing; and `split` drops empty parts after trimming. + filters: vec![ + format!("{ROW_ALIAS}.{ident} IS NOT NULL"), + format!("{part} <> ''"), + ], + part, + }) +} + +fn maybe_trim(dialect: SqlDialect, expr: &str, trim: bool) -> String { + if trim { + dialect.trim(expr) + } else { + expr.to_string() + } +} + +/// A source column read as an attribute value of declared type `declared`. +/// +/// `attribute_value` falls back to the cell's natural rendering when coercion fails, which a +/// typed SQL column cannot hold. So only the two combinations where coercion is provably a no-op +/// compile: the column already has the declared kind, or an `Integer` column widening to `Float`. +pub(crate) fn attribute_sql( + dialect: SqlDialect, + source_column: &str, + attribute: &str, + declared: crate::core::event_data::object_centric::OCELAttributeType, + schema: &TableSchema, + alias: &str, +) -> Result { + use crate::core::event_data::object_centric::OCELAttributeType as A; + let Some(col) = schema.columns.get(source_column) else { + // A column the row does not carry is `OCELAttributeValue::Null` on every row. + return Ok(dialect.null_attribute(declared)); + }; + let kind = col + .declared_kind() + .ok_or_else(|| RejectReason::UndeclaredColumnKind { + column: source_column.to_string(), + col_type: col.col_type.clone(), + field: "attribute", + })?; + let q = format!("{alias}.{}", dialect.quote_ident(source_column)); + let ok = matches!( + (kind, declared), + (ValueKind::Text, A::String) + | (ValueKind::Integer, A::Integer) + | (ValueKind::Float, A::Float) + | (ValueKind::Boolean, A::Boolean) + | (ValueKind::Timestamp, A::Time) + | (ValueKind::Integer, A::Float) + ); + if !ok { + return Err(RejectReason::AttributeCoercion { + attribute: attribute.to_string(), + column: source_column.to_string(), + col_type: col.col_type.clone(), + declared: declared.as_type_str(), + }); + } + Ok(match (kind, declared) { + (ValueKind::Integer, A::Float) => format!("CAST({q} AS DOUBLE)"), + (ValueKind::Timestamp, A::Time) => dialect.timestamp_column(&q, is_naive_timestamp(col)), + _ => q, + }) +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/compile/tests.rs b/process_mining/src/core/event_data/object_centric/extraction/compile/tests.rs new file mode 100644 index 00000000..8383e95c --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/compile/tests.rs @@ -0,0 +1,1276 @@ +//! Unit tests for the dialect layer and the assembly path. +//! +//! Nothing here executes SQL: these check the shape of what is emitted and which mappings are +//! refused. That the SQL agrees with the extractor is checked in `compile::differential`. +#![cfg(test)] + +use std::collections::{BTreeMap, BTreeSet}; + +use super::{compile, EmissionShape, ProbeKind, RejectReason, SqlDialect}; +use crate::core::event_data::object_centric::extraction::blueprint::{ + Blueprint, DuplicateObjectPolicy, EventEndpoint, IdRendering, Mapping, MappingEntry, + MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Target, +}; +use crate::core::event_data::object_centric::extraction::catalog::{ + Catalog, ExtractionCatalog, TableSchema, +}; +use crate::core::event_data::object_centric::extraction::expr::{ + AttributeMapping, TimestampSource, ValueExpression, +}; + +fn col(name: &str) -> ValueExpression { + ValueExpression::Column { + column: name.to_string(), + } +} + +fn constant(value: &str) -> ValueExpression { + ValueExpression::Constant { + value: value.to_string(), + } +} + +fn source(id: &str, table: &str) -> Node { + Node { + id: id.to_string(), + label: None, + op: NodeOp::Source { + source_id: "db".to_string(), + table: table.to_string(), + }, + } +} + +fn blueprint(nodes: Vec, mappings: Vec) -> Blueprint { + Blueprint { + version: crate::core::event_data::object_centric::extraction::MODEL_VERSION, + id_rendering: IdRendering::Raw, + nodes, + mappings, + on_missing_endpoint: MissingEndpointPolicy::Drop, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } +} + +fn orders_catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "orders", + [ + ("id", "INTEGER", false), + ("kind", "TEXT", true), + ("at", "TIMESTAMP", true), + ], + ), + ) +} + +fn object_mapping() -> MappingEntry { + MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("orders".into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }) +} + +fn event_mapping() -> MappingEntry { + MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("placed".into()), + when: None, + target: Target::Event { + event_type: constant("Placed"), + id: Some(col("id")), + timestamp: TimestampSource::column("at"), + attributes: vec![], + objects: vec![], + }, + }) +} + +fn view_bodies(c: &super::CompiledOcel) -> BTreeMap { + c.relations() + .iter() + .map(|v| (v.name.clone(), v.body.clone())) + .collect() +} + +#[test] +fn identifiers_and_literals_are_quoted_with_embedded_delimiters_doubled() { + let d = SqlDialect::DuckDb; + assert_eq!(d.quote_ident(r#"we"ird"#), r#""we""ird""#); + assert_eq!(d.string_literal("it's"), "'it''s'"); +} + +#[test] +fn the_six_relations_are_always_emitted_even_for_an_empty_blueprint() { + let bp = blueprint(vec![], vec![]); + let c = compile( + &bp, + &ExtractionCatalog::new(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + let names: Vec<&str> = c.relations().iter().map(|v| v.name.as_str()).collect(); + assert_eq!( + names, + vec![ + "object", + "event", + "event_map_type", + "object_map_type", + "event_object", + "object_object" + ] + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); +} + +#[test] +fn a_per_type_view_is_emitted_for_each_declared_type() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![object_mapping(), event_mapping()], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let views = view_bodies(&c); + assert!(views.contains_key("object_Order"), "{:?}", views.keys()); + assert!(views.contains_key("event_Placed"), "{:?}", views.keys()); +} + +#[test] +fn the_three_emission_paths_share_one_relation_body_set() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![object_mapping(), event_mapping()], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + let ddl = c.ddl(); + let materialize = c.materialize_ddl(); + let prelude = c.with_prelude("SELECT 1"); + for v in c.relations() { + assert!(ddl.contains(&v.body), "ddl is missing {}", v.name); + assert!( + materialize.contains(&v.body), + "materialize_ddl is missing {}", + v.name + ); + assert!( + prelude.contains(&v.body), + "with_prelude is missing {}", + v.name + ); + } + assert!(ddl.contains("CREATE VIEW \"object\" AS")); + assert!(materialize.contains("CREATE TABLE \"object\" AS")); + assert!(prelude.starts_with("WITH \"object\" AS (")); + assert!(prelude.ends_with("SELECT 1")); +} + +/// An `E2O` mapping whose object endpoint is created rather than dropped +/// ([`MissingEndpointPolicy::Create`]) inherits the event endpoint's own "does this event +/// actually exist" check, so the row `assemble` adds to `object` semi-joins `event` and `object` +/// must be emitted after `event`. +fn object_created_by_e2o_depends_on_event() -> Blueprint { + let mut bp = blueprint( + vec![source("orders", "orders")], + vec![ + event_mapping(), + MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("tag".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Placed")), + }, + object: ObjectEndpoint { + id: col("kind"), + object_type: Some(constant("Tag")), + split: None, + }, + qualifier: None, + }, + }), + ], + ); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + bp +} + +#[test] +fn every_emission_path_emits_the_event_relation_before_the_object_one_that_reads_it() { + let bp = object_created_by_e2o_depends_on_event(); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + // Confirm this blueprint actually exercises the dependency the test is pinning, not just + // that some unrelated ordering happens to work out. + let object_body = &view_bodies(&c)["object"]; + assert!( + object_body.contains("FROM \"event\" AS e"), + "fixture no longer creates the object-depends-on-event edge this test pins: {object_body}" + ); + + let paths = [ + ( + "ddl", + c.ddl(), + "CREATE VIEW \"event\" AS", + "CREATE VIEW \"object\" AS", + ), + ( + "materialize_ddl", + c.materialize_ddl(), + "CREATE TABLE \"event\" AS", + "CREATE TABLE \"object\" AS", + ), + ( + "with_prelude", + c.with_prelude("SELECT 1"), + "\"event\" AS (", + "\"object\" AS (", + ), + ]; + for (path, sql, event_marker, object_marker) in paths { + let event_pos = sql + .find(event_marker) + .unwrap_or_else(|| panic!("{path}: '{event_marker}' is missing:\n{sql}")); + let object_pos = sql + .find(object_marker) + .unwrap_or_else(|| panic!("{path}: '{object_marker}' is missing:\n{sql}")); + assert!( + event_pos < object_pos, + "{path}: 'object' semi-joins 'event' and must come after it:\n{sql}" + ); + } +} + +#[test] +fn with_prelude_returns_the_analysis_query_untouched_when_there_is_nothing_to_bind() { + let c = super::CompiledOcel { + dialect: SqlDialect::DuckDb, + shape: EmissionShape::PerType, + views: Vec::new(), + probes: Vec::new(), + errors: Vec::new(), + }; + assert_eq!(c.with_prelude("SELECT 1"), "SELECT 1"); +} + +#[test] +fn probe_statements_carry_the_relation_ctes_so_they_run_view_free() { + let bp = blueprint(vec![source("orders", "orders")], vec![object_mapping()]); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + let statements = c.probe_statements(); + assert_eq!(statements.len(), c.probes().len()); + assert!( + statements.iter().all(|s| s.starts_with("WITH ")), + "every probe must be self-contained: {statements:?}" + ); + assert!(c + .probes() + .iter() + .any(|p| p.kind == ProbeKind::AmbiguousObjectIdentity)); +} + +#[test] +fn a_union_is_union_all_with_the_absent_column_null_filled() { + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("a", [("id", "INTEGER", false), ("extra", "TEXT", true)]), + ) + .with_table("db", TableSchema::new("b", [("id", "INTEGER", false)])); + let bp = blueprint( + vec![ + source("a", "a"), + source("b", "b"), + Node { + id: "u".into(), + label: None, + op: NodeOp::Union { + inputs: vec!["a".into(), "b".into()], + }, + }, + ], + vec![MappingEntry::Single(Mapping { + node: "u".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let c = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["object"]; + assert!(body.contains("UNION ALL"), "{body}"); + assert!( + !body.replace("UNION ALL", "").contains("UNION"), + "plain UNION would drop rows the extractor keeps: {body}" + ); + assert!( + body.contains("CAST(NULL AS VARCHAR) AS \"extra\""), + "the branch without 'extra' must null-fill it: {body}" + ); +} + +#[test] +fn a_cross_kind_join_key_compiles_to_a_join_that_matches_nothing() { + // `join_key_part` tags each key with the runtime value's kind, so a Text "1" never matches + // an Integer 1, where DuckDB would implicit-cast and join them. + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("l", [("k", "TEXT", false)])) + .with_table("db", TableSchema::new("r", [("k", "INTEGER", false)])); + let bp = blueprint( + vec![ + source("l", "l"), + source("r", "r"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("k".into(), "k".into())], + }, + }, + ], + vec![MappingEntry::Single(Mapping { + node: "j".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("k"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let c = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["object"]; + assert!( + body.contains("INNER JOIN") && body.contains("ON FALSE"), + "a cross-kind key must be said to match nothing, not left to the engine: {body}" + ); +} + +#[test] +fn a_join_key_whose_kind_the_catalog_does_not_declare_is_refused() { + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("l", [("k", "GEOMETRY", false)])) + .with_table("db", TableSchema::new("r", [("k", "GEOMETRY", false)])); + let bp = blueprint( + vec![ + source("l", "l"), + source("r", "r"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("k".into(), "k".into())], + }, + }, + ], + vec![MappingEntry::Single(Mapping { + node: "j".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("k"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let c = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::UndecidableJoinKey { .. }) + ), + "{:?}", + c.errors() + ); +} + +#[test] +fn an_event_without_an_id_expression_is_reported_and_the_rest_still_compiles() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![ + object_mapping(), + MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("minted".into()), + when: None, + target: Target::Event { + event_type: constant("Placed"), + id: None, + timestamp: TimestampSource::column("at"), + attributes: vec![], + objects: vec![], + }, + }), + ], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert_eq!(c.errors().len(), 1, "{:?}", c.errors()); + let err = &c.errors()[0]; + assert!(matches!( + err.reason, + RejectReason::SynthesizedId { field: "id" } + )); + assert_eq!( + err.mapping.as_ref().map(|m| m.path.as_str()), + Some("mappings[1]") + ); + // The object mapping is untouched. + assert!(view_bodies(&c).contains_key("object_Order")); +} + +#[test] +fn a_type_read_from_a_column_without_a_domain_is_a_reject_not_a_wrong_view() { + let bp = dynamic_type_blueprint(); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::DynamicTypeName { .. }) + ), + "{:?}", + c.errors() + ); + assert!(!view_bodies(&c) + .keys() + .any(|k| k.starts_with("object_") && k != "object_map_type" && k != "object_object")); +} + +#[test] +fn a_supplied_domain_names_one_view_per_value_and_gets_a_staleness_probe() { + let catalog = orders_catalog().with_domain( + "db", + "orders", + "kind", + ["retail".to_string(), "wholesale".to_string()], + ); + let c = compile( + &dynamic_type_blueprint(), + &catalog, + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let views = view_bodies(&c); + assert!(views.contains_key("object_retail"), "{:?}", views.keys()); + assert!(views.contains_key("object_wholesale"), "{:?}", views.keys()); + let stale = c + .probes() + .iter() + .find(|p| matches!(&p.kind, ProbeKind::StaleTypeDomain { column } if column == "kind")) + .expect("a domain-derived type set must get a staleness probe"); + assert!( + stale.sql.contains("NOT IN ('retail', 'wholesale')"), + "{}", + stale.sql + ); +} + +#[test] +fn a_domain_above_the_cardinality_cap_is_an_error_naming_the_column() { + let domain: Vec = (0..=super::MAX_TYPE_DOMAIN) + .map(|i| format!("t{i}")) + .collect(); + let catalog = orders_catalog().with_domain("db", "orders", "kind", domain); + let c = compile( + &dynamic_type_blueprint(), + &catalog, + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::TypeDomainTooLarge { column, .. }) if column == "kind" + ), + "{:?}", + c.errors() + ); +} + +#[test] +fn a_recorded_but_empty_domain_is_a_reject_rather_than_a_view_set_with_nothing_in_it() { + // Distinct from no domain at all, which `Catalog::column_domain` reports as `None`. With no + // names there is no per-type view to emit and the probe would read `NOT IN ()`. + let catalog = orders_catalog().with_domain("db", "orders", "kind", Vec::::new()); + let c = compile( + &dynamic_type_blueprint(), + &catalog, + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::DynamicTypeName { detail, .. }) if detail.contains("empty") + ), + "{:?}", + c.errors() + ); + assert!(c.probes().is_empty(), "{:?}", c.probes()); +} + +#[test] +fn a_dynamically_typed_events_attributes_are_columns_of_the_wide_events_table() { + // The attribute plan keys a declaration by its type name, and a `Consolidated` mapping whose + // type is read from a column has none. `events` still needs the column, or every attribute of + // every dynamically-typed event is dropped from the table. + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("dynamic".into()), + when: None, + target: Target::Event { + event_type: col("kind"), + id: Some(col("id")), + timestamp: TimestampSource::column("at"), + attributes: vec![AttributeMapping { + source_column: "kind".into(), + name: "kind".into(), + value_type: None, + }], + objects: vec![], + }, + })], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let events = &view_bodies(&c)["events"]; + assert!( + events.starts_with("SELECT DISTINCT id, ocel_type, \"time\", \"kind\" FROM"), + "the attribute must have a column of its own: {events}" + ); + assert!( + events.contains("src.\"kind\" AS \"kind\""), + "and the branch must project the value into it: {events}" + ); +} + +#[test] +fn an_o2o_creates_its_source_object_only_where_the_target_id_is_there_too() { + // `run_o2o` renders both endpoint ids before it resolves either and returns as soon as one is + // absent, so a row with no target creates no source object either. + let mut bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("order-customer".into()), + when: None, + target: Target::O2O { + source: ObjectEndpoint { + id: col("id"), + object_type: Some(constant("Order")), + split: None, + }, + target: ObjectEndpoint { + id: col("kind"), + object_type: Some(constant("Customer")), + split: None, + }, + qualifier: None, + }, + })], + ); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["object"]; + let created_order = body + .split("UNION ALL") + .find(|branch| branch.contains("'Order' AS ocel_type")) + .unwrap_or_else(|| panic!("no branch creates the source object: {body}")); + assert!( + created_order.contains("src.\"kind\" IS NOT NULL"), + "the created source object must inherit the target's id guard: {created_order}" + ); +} + +#[test] +fn a_created_endpoint_whose_type_is_read_from_the_data_is_refused_by_a_per_type_shape() { + // `validate` asks only that the endpoint declares a type, so a `Column` passes it. Emitting + // the branch anyway puts objects in `object` that no `object_` view and no + // `object_map_type` row names. + let mut bp = blueprint( + vec![source("orders", "orders")], + vec![ + event_mapping(), + MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("tag".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("id"), + event_type: Some(constant("Placed")), + }, + object: ObjectEndpoint { + id: col("id"), + object_type: Some(col("kind")), + split: None, + }, + qualifier: None, + }, + }), + ], + ); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::DynamicTypeName { field, .. }) if *field == "object" + ), + "{:?}", + c.errors() + ); + // And nothing of that mapping reached the relations. + let object_body = &view_bodies(&c)["object"]; + assert!(!object_body.contains("\"kind\""), "{object_body}"); + + // The same blueprint is fine under `Consolidated`, where the type is a column value. + let consolidated = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert!( + consolidated.errors().is_empty(), + "{:?}", + consolidated.errors() + ); +} + +#[test] +fn a_type_named_after_a_relation_the_compiler_defines_is_refused() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("map_type"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::ReservedTypeName { name }) if name == "map_type" + ), + "{:?}", + c.errors() + ); +} + +#[test] +fn a_negated_guard_is_forced_false_rather_than_left_null() { + use crate::core::event_data::object_centric::extraction::predicate::{ + CompareOp, Literal, Operand, Predicate, + }; + let guard = Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("retail".into()), + }, + }; + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Ordered { + mappings: vec![ + Mapping { + node: "orders".into(), + label: Some("retail".into()), + when: Some(guard), + target: Target::Object { + object_type: constant("Retail"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }, + Mapping { + node: "orders".into(), + label: Some("other".into()), + when: None, + target: Target::Object { + object_type: constant("Other"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }, + ], + }], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["object"]; + assert!( + body.contains("(NOT COALESCE("), + "the catch-all's negated guard must be total before NOT sees it: {body}" + ); +} + +#[test] +fn the_consolidated_shape_emits_its_six_fixed_relations_even_for_an_empty_blueprint() { + let bp = blueprint(vec![], vec![]); + let c = compile( + &bp, + &ExtractionCatalog::new(), + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + let mut names: Vec<&str> = c.relations().iter().map(|v| v.name.as_str()).collect(); + names.sort_unstable(); + assert_eq!( + names, + vec![ + "e2o", + "event_attr_meta", + "events", + "o2o", + "object_attribute_changes", + "objects", + ] + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); +} + +/// A [`Catalog`] that panics if [`Catalog::column_domain`] is ever called, so a test compiling +/// under it is itself the proof that [`EmissionShape::Consolidated`] never consults a domain -- +/// not just that it tolerates a missing one. +#[derive(Debug)] +struct PanicOnDomainCatalog(ExtractionCatalog); + +impl Catalog for PanicOnDomainCatalog { + fn has_source(&self, source_id: &str) -> bool { + self.0.has_source(source_id) + } + + fn table(&self, source_id: &str, table: &str) -> Option<&TableSchema> { + self.0.table(source_id, table) + } + + fn column_domain( + &self, + _source_id: &str, + _table: &str, + _column: &str, + ) -> Option<&BTreeSet> { + panic!( + "EmissionShape::Consolidated must never call Catalog::column_domain: the type is a \ + column value, not a view name" + ); + } +} + +fn dynamic_type_blueprint() -> Blueprint { + blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: Some("dynamic".into()), + when: None, + target: Target::Object { + object_type: col("kind"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ) +} + +#[test] +fn consolidated_compiles_a_dynamic_type_with_no_domain_without_ever_consulting_it() { + let bp = dynamic_type_blueprint(); + let catalog = PanicOnDomainCatalog(orders_catalog()); + let c = compile( + &bp, + &catalog, + SqlDialect::DuckDb, + EmissionShape::Consolidated, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + assert!( + !c.probes() + .iter() + .any(|p| matches!(p.kind, ProbeKind::StaleTypeDomain { .. })), + "no domain means nothing can go stale: {:?}", + c.probes() + ); + let objects_body = &view_bodies(&c)["objects"]; + assert!( + objects_body.contains("src.\"kind\" AS ocel_type"), + "the type must still be read straight off the column: {objects_body}" + ); +} + +fn id_equals_text( + value: &str, +) -> crate::core::event_data::object_centric::extraction::predicate::Predicate { + use crate::core::event_data::object_centric::extraction::predicate::{ + CompareOp, Literal, Operand, Predicate, + }; + Predicate::Compare { + left: Operand::Column { + column: "id".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(value.into()), + }, + } +} + +#[test] +fn a_literal_is_coerced_to_the_operand_columns_declared_type() { + // `id = "1"` against an INTEGER column: `prepare` coerces the text literal, so the emitted + // comparison must be against an integer, not a string. + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: None, + when: Some(id_equals_text("1")), + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["object"]; + assert!( + body.contains("CAST(1 AS BIGINT)"), + "the literal must be coerced to the column's kind: {body}" + ); + assert!(!body.contains("= '1'"), "{body}"); +} + +#[test] +fn an_uncoercible_literal_compiles_to_a_predicate_that_matches_nothing() { + // Against the emitter rather than through `compile`: `validate` refuses a blueprint whose + // literal cannot be read as its column's type, so the whole compile stops before this + // predicate is reached. The emitter still has to answer for it, since a `Filter` node's + // condition reaches the same code from the push-down path. + let catalog = orders_catalog(); + let schema = catalog.table("db", "orders").expect("the fixture table"); + let sql = super::emit::predicate_sql( + SqlDialect::DuckDb, + &id_equals_text("abc"), + schema, + super::emit::ROW_ALIAS, + ) + .expect("the predicate itself compiles"); + assert!( + sql.contains("FALSE"), + "an uncoercible literal must match nothing, exactly as `prepare` leaves it: {sql}" + ); + // And it must not have been handed to the engine to implicit-cast instead, which is the one + // way this could quietly select rows the extractor refuses. + assert!( + !sql.contains("'abc'"), + "the literal must not reach the emitted SQL at all: {sql}" + ); +} + +#[test] +fn a_timestamp_parsed_by_the_chrono_cascade_is_reported_as_residual() { + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("orders", [("id", "INTEGER", false), ("at", "TEXT", true)]), + ); + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: None, + when: None, + target: Target::Event { + event_type: constant("Placed"), + id: Some(col("id")), + timestamp: TimestampSource::column("at"), + attributes: vec![], + objects: vec![], + }, + })], + ); + let c = compile(&bp, &catalog, SqlDialect::DuckDb, EmissionShape::PerType); + assert!( + matches!( + c.errors().first().map(|e| &e.reason), + Some(RejectReason::ResidualTimestamp { .. }) + ), + "{:?}", + c.errors() + ); +} + +#[test] +fn a_constant_timestamp_folds_at_compile_time() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![MappingEntry::Single(Mapping { + node: "orders".into(), + label: None, + when: None, + target: Target::Event { + event_type: constant("Placed"), + id: Some(col("id")), + timestamp: TimestampSource::constant("2020-01-01T00:00:00Z"), + attributes: vec![], + objects: vec![], + }, + })], + ); + let c = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(c.errors().is_empty(), "{:?}", c.errors()); + let body = &view_bodies(&c)["event_Placed"]; + assert!( + body.contains("CAST('2020-01-01T00:00:00+00:00' AS TIMESTAMPTZ)"), + "the fold must produce the constant's own instant: {body}" + ); +} + +#[test] +fn every_reject_reason_renders_a_message_naming_what_it_is_about() { + // Each reason is paired with something out of its own payload, so an arm that renders a + // fixed sentence and drops what it is about fails here. + let reasons = [ + (RejectReason::SynthesizedId { field: "id" }, "id"), + ( + RejectReason::DynamicTypeName { + field: "type", + detail: "no domain".into(), + }, + "no domain", + ), + ( + RejectReason::TypeDomainTooLarge { + column: "kind".into(), + size: 9, + cap: 8, + }, + "kind", + ), + ( + RejectReason::ReservedTypeName { + name: "reserved".into(), + }, + "reserved", + ), + (RejectReason::UnknownNode { node: "n".into() }, "n"), + (RejectReason::UnresolvedNodeSchema { node: "n".into() }, "n"), + (RejectReason::NodeCycle { node: "n".into() }, "n"), + (RejectReason::EmptyProjection { node: "n".into() }, "n"), + (RejectReason::EmptyUnion { node: "n".into() }, "n"), + ( + RejectReason::UnknownColumn { + column: "c".into(), + field: "id", + }, + "c", + ), + ( + RejectReason::UndeclaredColumnKind { + column: "c".into(), + col_type: "GEOMETRY".into(), + field: "id", + }, + "GEOMETRY", + ), + ( + RejectReason::UnstableIdentityRendering { + column: "c".into(), + col_type: "DOUBLE".into(), + field: "id", + }, + "DOUBLE", + ), + ( + RejectReason::UnstableDisplayRendering { + column: "c".into(), + col_type: "DOUBLE".into(), + field: "matches", + }, + "matches", + ), + ( + RejectReason::ResidualTimestamp { + detail: "cascade".into(), + }, + "cascade", + ), + ( + RejectReason::UndecidableJoinKey { + node: "j".into(), + side: "left", + column: "k".into(), + col_type: "GEOMETRY".into(), + }, + "GEOMETRY", + ), + ( + RejectReason::InvalidRegex { + pattern: "(".into(), + message: "unclosed".into(), + }, + "unclosed", + ), + ( + RejectReason::InvalidTemplate { + template: "a{".into(), + reason: "unterminated".into(), + }, + "unterminated", + ), + ( + RejectReason::AttributeCoercion { + attribute: "a".into(), + column: "c".into(), + col_type: "TEXT".into(), + declared: "integer", + }, + "integer", + ), + ( + RejectReason::DynamicTypeAttributeConflict { + attribute: "conflicted".into(), + }, + "conflicted", + ), + ( + RejectReason::ViewCycle { + view: "object".into(), + }, + "object", + ), + ]; + for (r, payload) in reasons { + let message = r.to_string(); + assert!( + message.contains(payload), + "{r:?} rendered {message:?}, which does not name {payload:?}" + ); + } +} + +// The differential suite runs against DuckDB only, so nothing executes the PostgreSQL output. These +// pin the spellings that actually differ between the two, which is what a differential run would +// otherwise have caught, plus the one construct the dialect deliberately refuses. + +#[test] +fn postgres_emits_the_same_relations_as_duckdb() { + let bp = blueprint( + vec![source("orders", "orders")], + vec![object_mapping(), event_mapping()], + ); + let pg = compile( + &bp, + &orders_catalog(), + SqlDialect::Postgres, + EmissionShape::PerType, + ); + let duck = compile( + &bp, + &orders_catalog(), + SqlDialect::DuckDb, + EmissionShape::PerType, + ); + assert!(pg.errors().is_empty(), "{:?}", pg.errors()); + let pg_names: Vec<&str> = pg.relations().iter().map(|v| v.name.as_str()).collect(); + let duck_names: Vec<&str> = duck.relations().iter().map(|v| v.name.as_str()).collect(); + assert_eq!( + pg_names, duck_names, + "the two dialects must describe the same OCEL surface" + ); +} + +/// The type names, which are the most pervasive difference: `VARCHAR`/`DOUBLE` are `DuckDB` +/// spellings, and a `CAST(.. AS DOUBLE)` is a syntax error in `PostgreSQL`. +#[test] +fn postgres_uses_its_own_type_names() { + use crate::core::event_data::object_centric::extraction::value::ValueKind; + let pg = SqlDialect::Postgres; + assert_eq!(pg.cast_to_text("x"), "CAST(x AS TEXT)"); + assert_eq!(pg.kind_sql_type(ValueKind::Text), "TEXT"); + assert_eq!(pg.kind_sql_type(ValueKind::Float), "DOUBLE PRECISION"); + assert_eq!( + pg.attribute_sql_type(crate::core::event_data::object_centric::OCELAttributeType::Float), + "DOUBLE PRECISION" + ); + assert_eq!( + pg.attribute_sql_type(crate::core::event_data::object_centric::OCELAttributeType::String), + "TEXT" + ); + + let bp = blueprint( + vec![source("orders", "orders")], + vec![object_mapping(), event_mapping()], + ); + let sql = compile( + &bp, + &orders_catalog(), + SqlDialect::Postgres, + EmissionShape::PerType, + ) + .ddl(); + assert!( + !sql.contains("AS VARCHAR"), + "PostgreSQL output must not carry DuckDB's VARCHAR spelling:\n{sql}" + ); + assert!( + !sql.contains("AS DOUBLE)"), + "PostgreSQL output must not carry DuckDB's DOUBLE spelling:\n{sql}" + ); +} + +/// The three function names with no shared spelling. +#[test] +fn postgres_uses_its_own_function_names() { + let pg = SqlDialect::Postgres; + // `string_split` is DuckDB-only. + assert!(pg + .split_to_rows("x", ",") + .starts_with("unnest(string_to_array(x, ")); + // `strftime` is DuckDB-only, and `%f` is not a PostgreSQL pattern. + let iso = pg.timestamptz_to_iso_text("x"); + assert!(iso.starts_with("to_char(timezone('UTC', x), "), "{iso}"); + assert!(iso.contains("US"), "microseconds, six digits: {iso}"); + // In PostgreSQL `regexp_matches` is set-returning, so a predicate has to use `~`. + assert_eq!(pg.regex_match("x", "a.c"), "(x ~ 'a.c')"); +} + +/// A split goes into a `SELECT` list, where a bare `(SELECT .. FROM regexp_matches(..))` is a +/// scalar subquery: any value with more than one match raises "more than one row returned by a +/// subquery used as an expression". The array constructor is what keeps it set-returning. +#[test] +fn postgres_regex_splitting_stays_set_returning() { + let pg = SqlDialect::Postgres; + for groups in [0, 1, 2] { + let sql = pg.regex_split_to_rows("x", "a(b)(c)", groups); + assert!(sql.starts_with("unnest(ARRAY("), "{groups} groups: {sql}"); + } +} + +/// `list_concat` takes exactly two lists, so the number of capture groups decides how many calls +/// there are, not how many arguments one call has. +#[test] +fn duckdb_regex_splitting_never_passes_list_concat_a_third_list() { + let duck = SqlDialect::DuckDb; + let calls = |groups| { + duck.regex_split_to_rows("x", "(a)(b)(c)", groups) + .matches("list_concat") + .count() + }; + assert_eq!(calls(1), 0); + assert_eq!(calls(2), 1); + assert_eq!(calls(3), 2); +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/dbcon_provider.rs b/process_mining/src/core/event_data/object_centric/extraction/dbcon_provider.rs new file mode 100644 index 00000000..12a3430e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/dbcon_provider.rs @@ -0,0 +1,714 @@ +//! A [`dbcon`]-backed [`RowProvider`] for `PostgreSQL`, `SQLite`, CSV and Parquet sources, plus +//! catalog discovery so a caller has a schema to compile or validate against before opening one. +//! +//! `dbcon`'s row-reading API is synchronous, and so is [`RowProvider`]: [`RowProvider::scan`] is a +//! direct, blocking call into [`DataSource::scan`] on the calling thread. +//! +//! [`DbconRowProvider::connect`] and [`discover_catalog`] take an arbitrary connection string and +//! use `dbcon`'s `new_any_without_discovery` / `new_any`, which are blocking. Only a `PostgreSQL` +//! source needs a runtime at all, and `dbcon` drives that one on a short-lived thread of its own, +//! so this module never touches Tokio and both are safe to call from any thread, including a +//! Tokio worker. +//! [`DbconRowProvider::from_bytes`] uses `dbcon`'s synchronous byte constructors directly. +//! +//! `PostgreSQL` is async underneath, so `dbcon` drives its own runtime once per scan and, from +//! inside another runtime, returns an error naming `spawn_blocking` rather than panicking. An +//! async caller with a `PostgreSQL` source must wrap its scans in `tokio::task::spawn_blocking`. + +use std::fmt; +use std::ops::ControlFlow; + +use dbcon::{DataSource, NormalizedType, NormalizedValue}; + +use super::catalog::{ + ColumnSchema, ExtractionCatalog, TablePreview, TableSchema, UNTYPED_COL_TYPE, +}; +use super::provider::{preview_rows, ProviderError, RowProvider}; +use super::value::{Value, ValueKind}; + +/// Why connecting through [`DbconRowProvider`] or one of this module's free functions failed. +/// +/// Distinct from [`ProviderError`], which covers a scan against an already-open connection. +/// `dbcon` reports connection failures as an unstructured `anyhow::Error`, so this carries a +/// message rather than inventing categories. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DbconProviderError { + /// Connecting to the source, or discovering its schema, failed. Carries `dbcon`'s message. + Connect(String), + /// A query against an already-open connection failed outside of [`RowProvider::scan`] + /// (currently only [`DbconRowProvider::distinct_values`]). Carries `dbcon`'s message. + Query(String), +} + +impl fmt::Display for DbconProviderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DbconProviderError::Connect(message) => write!(f, "connecting failed: {message}"), + DbconProviderError::Query(message) => write!(f, "query failed: {message}"), + } + } +} + +impl std::error::Error for DbconProviderError {} + +/// A [`RowProvider`] backed by [`dbcon`](https://github.com/aarkue/dbcon), giving [`scan`] a +/// connection to whatever `dbcon`'s connection string dispatch recognises: `PostgreSQL`, +/// `SQLite`, a CSV file or a Parquet file. +/// +/// [`scan`]: RowProvider::scan +/// +/// [`DataSource::scan`] abandons the query on [`ControlFlow::Break`], so breaking early also saves +/// the query's own time and transfer. +pub struct DbconRowProvider { + ds: DataSource, +} + +impl fmt::Debug for DbconRowProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DbconRowProvider").finish_non_exhaustive() + } +} + +impl DbconRowProvider { + /// Connect to `connection_string` without discovering its schema, giving the fast, + /// query-only connection [`RowProvider::scan`] needs. Use [`discover_catalog`] separately to + /// get a schema to compile or validate against. + /// + /// `connection_string` is a `postgres://`, `postgresql://`, `sqlite:`, `csv://` or + /// `parquet://` URL, or a bare path ending in `.csv` or `.parquet` (see + /// `dbcon::DataSource::new_any`'s dispatch). `name` only labels the connection in `dbcon`'s + /// bookkeeping and has no effect on what `scan` returns. + /// + /// Blocks the calling thread until the connection is established or fails. Safe from any + /// thread, including a Tokio worker: `dbcon`'s `new_any_without_discovery` drives the + /// connection on a short-lived thread of its own when the source needs a runtime at all. + /// + /// # Errors + /// Returns [`DbconProviderError::Connect`] if `dbcon` cannot open the connection. + pub fn connect(name: &str, connection_string: &str) -> Result { + let ds = + DataSource::new_any_without_discovery(name.to_string(), connection_string.to_string()) + .map_err(|e| DbconProviderError::Connect(e.to_string()))?; + Ok(Self { ds }) + } + + /// Open a source from bytes already in memory, rather than from a path. + /// + /// `format` is a file extension: `csv`, `tsv`, or `parquet`. Use this for contents with no + /// name to open, e.g. a browser upload, a `wasm32` build with no filesystem, or bytes fetched + /// over a network. + /// + /// This never touches `PostgreSQL`, so `dbcon`'s byte-backed constructors are called + /// synchronously and no runtime is involved. + /// + /// # Errors + /// Returns [`DbconProviderError::Connect`] if the bytes cannot be read as `format`. + pub fn from_bytes( + name: &str, + format: &str, + bytes: std::sync::Arc<[u8]>, + ) -> Result { + let name = name.to_string(); + let ds = match format.to_ascii_lowercase().as_str() { + "csv" | "tsv" => DataSource::new_csv_bytes(name, bytes).map_err(|e| e.to_string()), + "parquet" => DataSource::new_parquet_bytes(name, bytes).map_err(|e| e.to_string()), + "xlsx" => DataSource::new_xlsx_bytes(name, bytes).map_err(|e| e.to_string()), + other => Err(format!( + "cannot read '{other}' from memory; expected csv, tsv, parquet or xlsx" + )), + } + .map_err(DbconProviderError::Connect)?; + Ok(Self { ds }) + } + + /// This source's schema as an [`ExtractionCatalog`] under `source_id`. + /// + /// Reads the tables `dbcon` discovered when the source was opened, rather than reconnecting + /// as the free [`discover_catalog`] does. A byte-backed source has no connection string to + /// reopen from at all. + #[must_use] + pub fn discover_catalog(&self, source_id: &str) -> ExtractionCatalog { + let mut catalog = ExtractionCatalog::new(); + for (table_name, info) in &self.ds.tables { + let columns = info + .columns + .values() + .map(|c| (c.name.clone(), col_type_of(&c.col_type), c.is_nullable)); + catalog = catalog.with_table(source_id, TableSchema::new(table_name, columns)); + } + catalog + } + + /// Run `SELECT DISTINCT` on `table.column` over this provider's already-open connection + /// and return the distinct values as text, for populating + /// [`ExtractionCatalog::with_domain`]. + /// + /// Reuses the connection [`DbconRowProvider::connect`] already opened, unlike the free + /// function [`discover_catalog`]. + /// + /// # Errors + /// Returns [`DbconProviderError::Query`] if the query fails. + pub fn distinct_values( + &self, + table: &str, + column: &str, + ) -> Result, DbconProviderError> { + self.ds + .get_distinct_values(table, column) + .map_err(|e| DbconProviderError::Query(e.to_string())) + } + + /// Read at most `limit` rows of `table`, for showing a person what the data looks like. + /// + /// [`preview_rows`] over this provider's own connection. `dbcon` exposes no `LIMIT`, so + /// unlike [`distinct_values`](Self::distinct_values) this is no faster than the generic + /// function. + /// + /// # Errors + /// Returns whatever [`RowProvider::scan`] reports for an unknown table or column. + pub fn table_preview( + &self, + table: &str, + columns: &[&str], + limit: usize, + ) -> Result { + preview_rows(self, table, columns, limit) + } +} + +impl RowProvider for DbconRowProvider { + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError> { + // One buffer for the whole scan, so a full-table scan allocates per scan, not per row. + let mut row: Vec = Vec::with_capacity(columns.len()); + self.ds + .scan(table, columns, None, &mut |values| { + row.clear(); + row.extend(values.iter().map(convert)); + f(&row) + }) + .map_err(|e| map_err(table, &e.to_string())) + } +} + +/// Convert one `dbcon` cell into this crate's [`Value`], mapping every variant explicitly. +/// +/// - [`NormalizedValue::Null`]/`Text`/`Integer`/`Float`/`Boolean` map to the matching [`Value`] +/// variant. +/// - [`NormalizedValue::Timestamp`] becomes [`Value::Timestamp`], not text: every predicate +/// reading a timestamp column checks `Value::kind()` first, and a `Text` cell would fall back to +/// the slower, lossier multi-format parse cascade. +/// - [`NormalizedValue::Json`] and [`NormalizedValue::Unknown`] become [`Value::Text`]: both carry +/// real data with no dedicated `Value` case, and `Null` would leave a column a blueprint reads +/// permanently empty. `Json` becomes its compact serialisation, `Unknown` the string `dbcon` +/// already produced, and a `Matches` predicate can search either. Extracting one JSON field +/// belongs in a source-side view. +/// +/// A SQL compiler must render a JSON column through an explicit textual cast, or `Matches` would +/// see different text on the two paths. +fn convert(v: &NormalizedValue) -> Value { + match v { + NormalizedValue::Null => Value::Null, + NormalizedValue::Text(s) | NormalizedValue::Unknown(s) => Value::Text(s.clone()), + NormalizedValue::Integer(i) => Value::Integer(*i), + NormalizedValue::Float(f) => Value::Float(*f), + NormalizedValue::Boolean(b) => Value::Boolean(*b), + NormalizedValue::Timestamp(t) => Value::Timestamp(*t), + NormalizedValue::Json(j) => Value::Text(j.to_string()), + } +} + +/// [`ColumnSchema::declared_kind`] applied to a bare `col_type` string. +fn declared_kind_of(col_type: &str) -> Option { + ColumnSchema { + name: String::new(), + col_type: col_type.to_string(), + nullable: true, + } + .declared_kind() +} + +/// Map a [`NormalizedType`] to the `col_type` spelling [`ColumnSchema::declared_kind`] already +/// recognises, so a column [`discover_catalog`] finds still gets literal coercion. +/// +/// The `col_type` this produces must never make `declared_kind` claim a [`ValueKind`] that +/// `dbcon` itself declined to claim. The six classified variants map to spellings `declared_kind` +/// reads back as the matching kind, and [`NormalizedType::Json`]/[`NormalizedType::Unknown`] both +/// read back as `None`. +/// +/// `Unknown` covers every binary, array and range type and every `SQLite` NUMERIC-affinity column, +/// all of which decode per value. Its `declared` spelling is forwarded only when `declared_kind` +/// says `None` for it, since an array's element type is a word of its own and `int4[]` would +/// otherwise read back as an integer. Everything else becomes +/// [`UNTYPED_COL_TYPE`](super::catalog::UNTYPED_COL_TYPE), the original spellings still being +/// available from `dbcon::DataSource::unknown_column_types`. +fn col_type_of(t: &NormalizedType) -> String { + match t { + NormalizedType::Text => "TEXT".to_string(), + NormalizedType::Integer => "INTEGER".to_string(), + NormalizedType::Float => "DOUBLE PRECISION".to_string(), + NormalizedType::Boolean => "BOOLEAN".to_string(), + NormalizedType::Timestamp => "TIMESTAMP".to_string(), + NormalizedType::Json => "JSON".to_string(), + NormalizedType::Unknown(declared) => { + if declared_kind_of(declared).is_none() { + declared.clone() + } else { + UNTYPED_COL_TYPE.to_string() + } + } + } +} + +/// Connect to `connection_string`, discover its schema, and record every table `dbcon` finds +/// under `source_id` in a fresh [`ExtractionCatalog`], giving the schema needed to +/// [`compile`](super::compile::compile) or [`validate`](super::validate::validate) a blueprint +/// against this source. +/// +/// This opens a one-off connection, separate from any [`DbconRowProvider`], and walks every table +/// `dbcon` can see. Run it once ahead of execution, or cache the result. +/// +/// Blocks the calling thread. See the module docs for the `spawn_blocking` obligation on an async +/// caller. +/// +/// # Errors +/// Returns [`DbconProviderError::Connect`] if `dbcon` cannot connect or discover the schema. +pub fn discover_catalog( + source_id: &str, + connection_string: &str, +) -> Result { + let tables = DataSource::new_any(source_id.to_string(), connection_string.to_string()) + .map_err(|e| DbconProviderError::Connect(e.to_string()))? + .tables; + + let mut catalog = ExtractionCatalog::new(); + for (table_name, info) in tables { + let columns = info + .columns + .into_values() + .map(|c| (c.name, col_type_of(&c.col_type), c.is_nullable)); + catalog = catalog.with_table(source_id, TableSchema::new(&table_name, columns)); + } + Ok(catalog) +} + +/// Translate one of `dbcon`'s `anyhow::Error` messages into a [`ProviderError`], recognising +/// the patterns its backends (`SQLite` via `rusqlite`, `PostgreSQL` via `sqlx`, and CSV) are +/// each known to emit for an unknown table or column, and falling back to +/// [`ProviderError::Backend`] for everything else. +/// +/// Best-effort: `dbcon` reports failures as a stringly-typed `anyhow::Error`. An unrecognised +/// message still reaches the caller, as [`ProviderError::Backend`]. +fn map_err(table: &str, message: &str) -> ProviderError { + let lower = message.to_ascii_lowercase(); + // SQLite: "no such table: foo" / "no such column: bar". + if let Some(e) = super::provider::sqlite_message_error(table, message) { + return e; + } + // PostgreSQL: `relation "foo" does not exist` / `column "bar" does not exist`. + if lower.contains("relation") && lower.contains("does not exist") { + return ProviderError::UnknownTable { + table: table.to_string(), + }; + } + if lower.contains("column") && lower.contains("does not exist") { + if let Some(column) = extract_quoted(message, '"') { + return ProviderError::UnknownColumn { + table: table.to_string(), + column, + }; + } + } + // CSV: `Column 'foo' not found in CSV`. + if lower.contains("not found in csv") { + if let Some(column) = extract_quoted(message, '\'') { + return ProviderError::UnknownColumn { + table: table.to_string(), + column, + }; + } + } + ProviderError::Backend { + table: table.to_string(), + message: message.to_string(), + } +} + +/// The first substring of `message` delimited by a pair of `quote` characters. +/// +/// `None` when nothing closes the quote: returning the rest of the message would name the +/// driver's own error text as the missing column. +fn extract_quoted(message: &str, quote: char) -> Option { + let mut parts = message.splitn(3, quote); + parts.next()?; + let quoted = parts.next()?; + parts.next()?; + Some(quoted.to_string()) +} + +#[cfg(test)] +mod tests { + use super::super::catalog::Catalog; + use super::*; + use std::io::Write; + + // NormalizedValue -> Value conversion: exhaustive, no database needed. + + #[test] + fn null_converts_to_null() { + assert_eq!(convert(&NormalizedValue::Null), Value::Null); + } + + #[test] + fn text_converts_to_text() { + assert_eq!( + convert(&NormalizedValue::Text("hi".to_string())), + Value::Text("hi".to_string()) + ); + } + + #[test] + fn integer_converts_to_integer() { + assert_eq!(convert(&NormalizedValue::Integer(42)), Value::Integer(42)); + } + + #[test] + fn float_converts_to_float() { + assert_eq!(convert(&NormalizedValue::Float(1.5)), Value::Float(1.5)); + } + + #[test] + fn boolean_converts_to_boolean() { + assert_eq!( + convert(&NormalizedValue::Boolean(true)), + Value::Boolean(true) + ); + } + + #[test] + fn timestamp_converts_to_timestamp_not_text() { + let ts = chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap(); + let converted = convert(&NormalizedValue::Timestamp(ts)); + assert_eq!(converted, Value::Timestamp(ts)); + // Must carry `ValueKind::Timestamp`, not `Text`, so the fast path in + // `TimestampSource::parse` and `Compare`'s literal coercion both see it. + assert_eq!(converted.kind(), Some(ValueKind::Timestamp)); + } + + #[test] + fn json_converts_to_its_compact_text_form() { + let json = serde_json::json!({"a": 1, "b": [true, null]}); + let expected = json.to_string(); + assert_eq!(convert(&NormalizedValue::Json(json)), Value::Text(expected)); + } + + #[test] + fn unknown_converts_to_its_carried_text() { + assert_eq!( + convert(&NormalizedValue::Unknown("raw-driver-text".to_string())), + Value::Text("raw-driver-text".to_string()) + ); + } + + // col_type mapping, round-tripped through `ColumnSchema::declared_kind`. + + #[test] + fn col_type_mapping_round_trips_through_declared_kind_for_real_engine_spellings() { + let cases: &[(&str, ValueKind)] = &[ + ("int4", ValueKind::Integer), + ("INTEGER", ValueKind::Integer), + ("BIGINT", ValueKind::Integer), + ("VARCHAR", ValueKind::Text), + ("TEXT", ValueKind::Text), + ("timestamp", ValueKind::Timestamp), + ("TIMESTAMPTZ", ValueKind::Timestamp), + ("DOUBLE PRECISION", ValueKind::Float), + ("REAL", ValueKind::Float), + ("NUMERIC", ValueKind::Float), + ("BOOLEAN", ValueKind::Boolean), + ("bool", ValueKind::Boolean), + ]; + for &(spelling, expected) in cases { + let normalized = NormalizedType::from_raw(spelling); + let col_type = col_type_of(&normalized); + let schema = ColumnSchema { + name: "c".to_string(), + col_type, + nullable: true, + }; + assert_eq!( + schema.declared_kind(), + Some(expected), + "spelling {spelling:?} (normalized to {normalized:?}) did not declare {expected:?}" + ); + } + } + + #[test] + fn json_col_type_declares_no_kind_same_as_the_value_side_has_no_json_kind() { + let schema = ColumnSchema { + name: "c".to_string(), + col_type: col_type_of(&NormalizedType::Json), + nullable: true, + }; + assert_eq!(schema.declared_kind(), None); + } + + // `NormalizedType::Unknown`: dbcon's "do not assume" signal must not become a claim. + + /// Declared spellings `dbcon` reports as `Unknown`, taken from its own documented list: + /// binary types, array types, range types, `SQLite` BLOB/NUMERIC-affinity columns, and a + /// column declared with no type at all. + const UNKNOWN_SPELLINGS: &[&str] = &[ + "blob", + "bytea", + "varbinary", + "image", + "raw", + "int4[]", + "_int4", + "text[]", + "int4range", + "daterange", + "numrange", + "tsvector", + "point", + "bit varying", + "blah", + "", + ]; + + #[test] + fn unknown_col_type_never_declares_a_kind_dbcon_declined_to_declare() { + for spelling in UNKNOWN_SPELLINGS { + let unknown = NormalizedType::Unknown((*spelling).to_string()); + let col_type = col_type_of(&unknown); + assert_eq!( + declared_kind_of(&col_type), + None, + "Unknown({spelling:?}) produced col_type {col_type:?}, which declares a kind; \ + dbcon said it could not classify this column, so nothing downstream may coerce \ + literals against it" + ); + } + } + + #[test] + fn unknown_keeps_an_inert_declared_spelling_but_replaces_a_misleading_one() { + // Inert: `declared_kind` already says `None`, so the spelling survives as a diagnostic. + for inert in ["blob", "bytea", "point", "daterange", "int4range"] { + assert_eq!( + col_type_of(&NormalizedType::Unknown(inert.to_string())), + inert, + "`declared_kind` reads a col_type word by word, so a range type names no kind \ + and its spelling is worth keeping" + ); + } + // Still misleading: an array's element type is a word of its own, so `int4[]` and + // `_int4` do name a kind. + for misleading in ["int4[]", "_int4", "text[]"] { + assert_eq!( + col_type_of(&NormalizedType::Unknown(misleading.to_string())), + UNTYPED_COL_TYPE + ); + } + } + + #[test] + fn a_binary_column_is_unknown_not_text_and_so_gets_no_coercion() { + // dbcon used to guess `Text` here (its old `ends_with("char")|ends_with("text")` + // heuristic), which made a BLOB column look safe to compare as a string. It is `Unknown` + // now, and must stay uncoerced. + let blob = NormalizedType::from_sqlite_declared("BLOB"); + assert_eq!(blob, NormalizedType::Unknown("blob".to_string())); + assert_eq!(declared_kind_of(&col_type_of(&blob)), None); + } + + #[test] + fn every_classified_normalized_type_round_trips_to_its_own_kind() { + let cases: &[(NormalizedType, Option)] = &[ + (NormalizedType::Text, Some(ValueKind::Text)), + (NormalizedType::Integer, Some(ValueKind::Integer)), + (NormalizedType::Float, Some(ValueKind::Float)), + (NormalizedType::Boolean, Some(ValueKind::Boolean)), + (NormalizedType::Timestamp, Some(ValueKind::Timestamp)), + // `ValueKind` has no JSON case; `convert` lands JSON as `Value::Text`, but claiming + // `Text` here would let a `Compare` literal coerce against the raw serialisation. + (NormalizedType::Json, None), + ]; + for (normalized, expected) in cases { + assert_eq!( + declared_kind_of(&col_type_of(normalized)), + *expected, + "{normalized:?} did not round-trip" + ); + } + } + + // End-to-end `scan` over a temporary CSV file: no server required. + + fn write_temp_csv(contents: &str) -> tempfile::NamedTempFile { + let mut file = tempfile::Builder::new().suffix(".csv").tempfile().unwrap(); + file.write_all(contents.as_bytes()).unwrap(); + file.flush().unwrap(); + file + } + + #[test] + fn scan_streams_rows_from_a_csv_file() { + let file = write_temp_csv("id,name,active\n1,alice,true\n2,bob,false\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let mut rows: Vec> = Vec::new(); + provider + .scan("main", &["id", "name"], &mut |row| { + rows.push(row.to_vec()); + ControlFlow::Continue(()) + }) + .unwrap(); + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0], + vec![ + Value::Text("1".to_string()), + Value::Text("alice".to_string()) + ] + ); + assert_eq!( + rows[1], + vec![Value::Text("2".to_string()), Value::Text("bob".to_string())] + ); + } + + #[test] + fn table_preview_stops_at_the_limit_and_keeps_column_order() { + let file = write_temp_csv("id,name\n1,alice\n2,bob\n3,carol\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let preview = provider.table_preview("main", &["id", "name"], 2).unwrap(); + + assert_eq!(preview.columns, vec!["id".to_string(), "name".to_string()]); + assert_eq!(preview.rows.len(), 2, "the limit bounds the scan"); + assert_eq!( + preview.rows[0], + vec![Some("1".to_string()), Some("alice".to_string())] + ); + } + + #[test] + fn table_preview_with_a_zero_limit_reads_nothing() { + let file = write_temp_csv("id\n1\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let preview = provider.table_preview("main", &["id"], 0).unwrap(); + + // Still reports its shape: the path a "previews off" setting takes. + assert_eq!(preview.columns, vec!["id".to_string()]); + assert!(preview.rows.is_empty()); + } + + #[test] + fn table_preview_projects_per_column_examples_deduplicated() { + let file = write_temp_csv("status\ndraft\ndraft\ndone\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let preview = provider.table_preview("main", &["status"], 5).unwrap(); + + assert_eq!(preview.column_values("status", 5), vec!["draft", "done"]); + assert!(preview.column_values("nonexistent", 5).is_empty()); + } + + #[test] + fn scan_breaks_early_without_calling_f_again() { + let file = write_temp_csv("id\n1\n2\n3\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let mut seen = 0; + provider + .scan("main", &["id"], &mut |_row| { + seen += 1; + ControlFlow::Break(()) + }) + .unwrap(); + + assert_eq!(seen, 1, "f must not be called again after it breaks"); + } + + #[test] + fn scan_with_empty_columns_calls_f_once_per_row_with_an_empty_slice() { + let file = write_temp_csv("id,name\n1,alice\n2,bob\n3,carol\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let mut row_count = 0; + provider + .scan("main", &[], &mut |row| { + assert!( + row.is_empty(), + "empty columns must yield an empty row, not every column" + ); + row_count += 1; + ControlFlow::Continue(()) + }) + .unwrap(); + + assert_eq!(row_count, 3); + } + + #[test] + fn scan_of_an_unknown_column_is_a_provider_error() { + let file = write_temp_csv("id\n1\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let err = provider + .scan("main", &["nope"], &mut |_| ControlFlow::Continue(())) + .unwrap_err(); + assert!( + matches!( + err, + ProviderError::UnknownColumn { .. } | ProviderError::Backend { .. } + ), + "unexpected error variant: {err:?}" + ); + } + + // Catalog discovery and the distinct-values helper, both over the same CSV file. + + #[test] + fn discover_catalog_finds_the_csv_s_single_table_as_all_text_columns() { + let file = write_temp_csv("id,name\n1,alice\n2,bob\n"); + let catalog = discover_catalog("erp", file.path().to_str().unwrap()).unwrap(); + + let table = catalog + .table("erp", "main") + .expect("csv table 'main' present"); + assert_eq!(table.columns.len(), 2); + for col in table.columns.values() { + // CSV carries no type information; dbcon reports every column as Text. + assert_eq!(col.declared_kind(), Some(ValueKind::Text)); + assert!(col.nullable); + } + } + + #[test] + fn distinct_values_reuses_the_open_connection() { + let file = write_temp_csv("status\ndraft\nsale\ndraft\n"); + let provider = + DbconRowProvider::connect("csv-test", file.path().to_str().unwrap()).unwrap(); + + let mut values = provider.distinct_values("main", "status").unwrap(); + values.sort(); + assert_eq!(values, vec!["draft".to_string(), "sale".to_string()]); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/desugar.rs b/process_mining/src/core/event_data/object_centric/extraction/desugar.rs new file mode 100644 index 00000000..7380e7e9 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/desugar.rs @@ -0,0 +1,208 @@ +//! Rewrite surface sugar into the plain mappings everything downstream works with. + +use super::blueprint::{Blueprint, Mapping, MappingEntry}; +use super::predicate::Predicate; + +/// Flatten a blueprint's mappings, rewriting ordered groups into independent mappings. +/// +/// A group's `k`th mapping keeps its own guard and gains a negation of every earlier guard, so +/// first-match-wins holds without the evaluator needing to know about ordering. A member with no +/// guard of its own becomes the conjunction of those negations alone, which is the catch-all, and +/// ends the group: nothing written after a catch-all can match a row it left over. +#[must_use] +pub fn desugar(blueprint: &Blueprint) -> Vec { + desugar_with_paths(blueprint) + .into_iter() + .map(|(_, m)| m) + .collect() +} + +/// Like [`desugar`], but paired with the JSON path of the authored entry each output mapping +/// came from, e.g. `mappings[1]` for a top-level `Single` or `mappings[0].mappings[2]` for the +/// third member of the first `Ordered` group. A diagnostic can then point back at what the author +/// wrote instead of a position in the flattened list. +pub(crate) fn desugar_with_paths(blueprint: &Blueprint) -> Vec<(String, Mapping)> { + let mut out = Vec::with_capacity(blueprint.mappings.len()); + for (i, entry) in blueprint.mappings.iter().enumerate() { + match entry { + MappingEntry::Single(m) => out.push((format!("mappings[{i}]"), m.clone())), + MappingEntry::Ordered { mappings } => { + let mut earlier: Vec = Vec::new(); + for (j, m) in mappings.iter().enumerate() { + let mut conditions: Vec = Vec::with_capacity(earlier.len() + 1); + conditions.extend(m.when.clone()); + conditions.extend(earlier.iter().map(|p| Predicate::Not { + condition: Box::new(p.clone()), + })); + let when = match conditions.len() { + 0 => None, + 1 => conditions.pop(), + _ => Some(Predicate::And { conditions }), + }; + out.push(( + format!("mappings[{i}].mappings[{j}]"), + Mapping { when, ..m.clone() }, + )); + let Some(own) = &m.when else { + // An unguarded member matches every row the earlier ones did not, so no + // later member can ever fire. + break; + }; + earlier.push(own.clone()); + } + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::blueprint::*; + use crate::core::event_data::object_centric::extraction::expr::*; + use crate::core::event_data::object_centric::extraction::predicate::*; + use crate::core::event_data::object_centric::extraction::row::with_row; + use crate::core::event_data::object_centric::extraction::value::Value; + + fn eq(column: &str, value: &str) -> Predicate { + Predicate::Compare { + left: Operand::Column { + column: column.into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(value.into()), + }, + } + } + + fn mapping(label: &str, when: Option) -> Mapping { + Mapping { + node: "n".into(), + label: Some(label.into()), + when, + target: Target::Object { + object_type: ValueExpression::Constant { + value: label.into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + } + } + + fn blueprint(mappings: Vec) -> Blueprint { + Blueprint { + version: 1, + id_rendering: IdRendering::Raw, + nodes: vec![], + mappings, + on_missing_endpoint: MissingEndpointPolicy::Drop, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } + } + + #[test] + fn single_mappings_pass_through_untouched() { + let bp = blueprint(vec![MappingEntry::Single(mapping("a", Some(eq("s", "x"))))]); + let out = desugar(&bp); + assert_eq!(out.len(), 1); + assert_eq!(out[0].when, Some(eq("s", "x"))); + } + + #[test] + fn ordered_mappings_become_mutually_exclusive_guards() { + let bp = blueprint(vec![MappingEntry::Ordered { + mappings: vec![ + mapping("completed", Some(eq("new", "C"))), + mapping("changed", None), + ], + }]); + let out = desugar(&bp); + assert_eq!(out.len(), 2); + + // Row where the first rule matches: only the first mapping fires. + let first = out[0].when.clone().unwrap().prepare(None).unwrap(); + let second = out[1].when.clone().unwrap().prepare(None).unwrap(); + with_row(&[("new", Value::Text("C".into()))], |row| { + assert!(first.evaluate(row)); + assert!( + !second.evaluate(row), + "second must not fire when the first matched" + ); + }); + // Row where it does not: the unguarded second mapping takes over. + with_row(&[("new", Value::Text("B".into()))], |row| { + assert!(!first.evaluate(row)); + assert!(second.evaluate(row)); + }); + } + + #[test] + fn a_third_ordered_mapping_excludes_both_predecessors() { + let bp = blueprint(vec![MappingEntry::Ordered { + mappings: vec![ + mapping("a", Some(eq("s", "1"))), + mapping("b", Some(eq("s", "2"))), + mapping("c", None), + ], + }]); + let out = desugar(&bp); + let third = out[2].when.clone().unwrap().prepare(None).unwrap(); + for v in ["1", "2"] { + with_row(&[("s", Value::Text(v.into()))], |row| { + assert!(!third.evaluate(row)) + }); + } + with_row(&[("s", Value::Text("3".into()))], |row| { + assert!(third.evaluate(row)) + }); + } + + #[test] + fn a_group_ends_at_its_catch_all() { + let bp = blueprint(vec![MappingEntry::Ordered { + mappings: vec![ + mapping("a", Some(eq("s", "1"))), + mapping("catch-all", None), + mapping("unreachable", Some(eq("s", "2"))), + ], + }]); + let out = desugar(&bp); + assert_eq!( + out.iter().map(|m| m.label.clone()).collect::>(), + vec![Some("a".to_string()), Some("catch-all".to_string())] + ); + } + + #[test] + fn paths_point_back_at_the_authored_entry_not_the_flattened_position() { + // With mappings = [Ordered{3 members}, Single], the Single is at flattened index 3 but + // its authored JSON path is mappings[1]. + let bp = blueprint(vec![ + MappingEntry::Ordered { + mappings: vec![ + mapping("a", Some(eq("s", "1"))), + mapping("b", Some(eq("s", "2"))), + mapping("c", None), + ], + }, + MappingEntry::Single(mapping("d", None)), + ]); + let out = desugar_with_paths(&bp); + let paths: Vec<&str> = out.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!( + paths, + vec![ + "mappings[0].mappings[0]", + "mappings[0].mappings[1]", + "mappings[0].mappings[2]", + "mappings[1]", + ] + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/differential.rs b/process_mining/src/core/event_data/object_centric/extraction/differential.rs new file mode 100644 index 00000000..2dcfe96f --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/differential.rs @@ -0,0 +1,418 @@ +//! Reusable harness for comparing two extraction results content for content, shared by the +//! sink-agreement tests and the SQL compiler's differential tests. +//! +//! [`snapshot`] builds a canonical, [`PartialEq`]-comparable [`OcelSnapshot`] from anything +//! implementing [`ReadableOCEL`], so `assert_eq!` on two snapshots compares every id, timestamp, +//! attribute value and relation instead of just counts. +//! +//! Two ways of running one blueprint through two sinks are available: +//! +//! [`run_against_both`] fans a single `extract` call out to both sinks. This is required for a +//! [`Target::Event`](super::blueprint::Target::Event) without an `id` expression, where `extract` +//! mints a fresh UUID per row and two separate runs would not be comparable by id. In exchange, +//! [`FanOutSink`] relays only the stronger resolution answer, so the extractor never takes a +//! [`Resolution::Deferred`] branch if one sink is eager. This compares write-side behaviour only. +//! +//! [`extract_separately`] runs `extract` once per sink, so each sink answers `resolve_event`/ +//! `resolve_object` its own way. This is the only variant that catches resolution divergences, but +//! is limited to blueprints whose entity ids are all author-given. +#![cfg(test)] + +use std::collections::BTreeMap; + +use chrono::{DateTime, FixedOffset}; + +// `snapshot`/`OcelSnapshot` are pure and are what the ordering tests compare with. Everything +// that runs a blueprint through two sinks needs a second sink to compare against, which is the +// `DuckDbSink`, so it and its imports are gated together. +#[cfg(feature = "ocel-duckdb")] +use super::{ + blueprint::Blueprint, + catalog::Catalog, + extract::extract, + provider::RowProvider, + report::{ExtractionError, ExtractionReport}, + sink::{EventRef, ExtractionSink, FinalizeReport, ObjectRef, Resolution, SinkError}, +}; +use crate::core::event_data::object_centric::readable::ReadableOCEL; +#[cfg(feature = "ocel-duckdb")] +use crate::core::event_data::object_centric::OCELTypeAttribute; +use crate::core::event_data::object_centric::{OCELAttributeValue, OCELType}; +#[cfg(feature = "ocel-duckdb")] +use std::collections::HashMap; + +/// `(attribute name, attribute type string)` for `t`, sorted by name: two sinks must agree on the +/// declared set, not on declaration order. +fn sorted_attrs(t: &OCELType) -> Vec<(String, String)> { + let mut attrs: Vec<(String, String)> = t + .attributes + .iter() + .map(|a| (a.name.clone(), a.value_type.clone())) + .collect(); + attrs.sort(); + attrs +} + +/// A canonical, comparable snapshot of an OCEL's declared types, events, objects and relations. +/// Build with [`snapshot`]; compare two with `assert_eq!` (or `PartialEq`) directly. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct OcelSnapshot { + /// Declared event types: name -> `{(attribute name, attribute type string)}`. + pub(crate) event_types: BTreeMap>, + /// Declared object types. See [`Self::event_types`]. + pub(crate) object_types: BTreeMap>, + /// Every event, keyed by id. + pub(crate) events: BTreeMap, + /// Every object, keyed by id. + pub(crate) objects: BTreeMap, +} + +/// One event's comparable content. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct EventSnapshot { + pub(crate) event_type: String, + pub(crate) time: DateTime, + /// `(name, value)`, sorted by name. + pub(crate) attributes: Vec<(String, OCELAttributeValue)>, + /// `(qualifier, object_id)`, sorted. A multiset, not deduplicated: a relation repeated twice + /// must appear twice on both sides. + pub(crate) e2o: Vec<(String, String)>, +} + +/// One object's comparable content. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ObjectSnapshot { + pub(crate) object_type: String, + /// `(name, time, value)`, sorted by `(name, time)`. + pub(crate) attributes: Vec<(String, DateTime, OCELAttributeValue)>, + /// `(qualifier, target_object_id)`, sorted. See [`EventSnapshot::e2o`]. + pub(crate) o2o: Vec<(String, String)>, +} + +/// Build an [`OcelSnapshot`] from any [`ReadableOCEL`] implementor, for a content-for-content +/// `assert_eq!` between two extraction results, or between the reference executor and the SQL +/// compiler's output. +pub(crate) fn snapshot(ocel: &O) -> OcelSnapshot { + let event_types = ocel + .event_types() + .iter() + .map(|t| (t.name.clone(), sorted_attrs(t))) + .collect(); + let object_types = ocel + .object_types() + .iter() + .map(|t| (t.name.clone(), sorted_attrs(t))) + .collect(); + + let mut events = BTreeMap::new(); + for e in ocel.iter_events() { + let mut attributes: Vec<(String, OCELAttributeValue)> = e + .attributes + .iter() + .map(|a| (a.name.clone(), a.value.clone())) + .collect(); + attributes.sort_by(|a, b| a.0.cmp(&b.0)); + let mut e2o: Vec<(String, String)> = e + .relationships + .iter() + .map(|r| (r.qualifier.clone(), r.object_id.clone())) + .collect(); + e2o.sort(); + events.insert( + e.id.clone(), + EventSnapshot { + event_type: e.event_type.clone(), + time: e.time, + attributes, + e2o, + }, + ); + } + + let mut objects = BTreeMap::new(); + for o in ocel.iter_objects() { + let mut attributes: Vec<(String, DateTime, OCELAttributeValue)> = o + .attributes + .iter() + .map(|a| (a.name.clone(), a.time, a.value.clone())) + .collect(); + attributes.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); + let mut o2o: Vec<(String, String)> = o + .relationships + .iter() + .map(|r| (r.qualifier.clone(), r.object_id.clone())) + .collect(); + o2o.sort(); + objects.insert( + o.id.clone(), + ObjectSnapshot { + object_type: o.object_type.clone(), + attributes, + o2o, + }, + ); + } + + OcelSnapshot { + event_types, + object_types, + events, + objects, + } +} + +/// Run one `extract` per sink, so each sink answers endpoint resolution its own way and the +/// extractor takes the branch that answer selects, including the deferral path that +/// [`run_against_both`] cannot reach. See the module docs for when to use which. +/// +/// Only meaningful for a blueprint whose entity ids are all author-given: a minted event id +/// differs between the two runs by design. +/// +/// # Errors +/// Returns whichever run's [`ExtractionError`] came first. +#[cfg(feature = "ocel-duckdb")] +pub(crate) fn extract_separately( + blueprint: &Blueprint, + catalog: &dyn Catalog, + providers: &HashMap, + a: &mut dyn ExtractionSink, + b: &mut dyn ExtractionSink, +) -> Result<(ExtractionReport, ExtractionReport), ExtractionError> { + let ra = extract(blueprint, catalog, providers, a)?; + let rb = extract(blueprint, catalog, providers, b)?; + Ok((ra, rb)) +} + +/// Run `f` (an `extract` call, typically) against `a` and `b` simultaneously through a fan-out +/// [`ExtractionSink`], so both receive the exact same sequence of declarations, entities and +/// relations, including any id `extract` mints itself. Two separate `extract` calls would each +/// mint their own. See the module docs. +#[cfg(feature = "ocel-duckdb")] +pub(crate) fn run_against_both( + a: &mut dyn ExtractionSink, + b: &mut dyn ExtractionSink, + f: impl FnOnce(&mut dyn ExtractionSink) -> T, +) -> T { + let mut fan_out = FanOutSink { + a, + b, + event_handles: HashMap::new(), + object_handles: HashMap::new(), + }; + f(&mut fan_out) +} + +/// Forwards every [`ExtractionSink`] call to both `a` and `b`, verbatim and in order. Always +/// hands back [`EventRef::Id`]/[`ObjectRef::Id`] itself (echoing the id it was given), regardless +/// of what either inner sink's own handle looks like, and translates an incoming ref back to each +/// inner sink's own handle before forwarding a relation call, so the two inner sinks never see +/// each other's ref type. Handles are remembered from each inner sink's `add_*`/`resolve_*` +/// answers rather than re-asked: a second `resolve_*` call per endpoint would double-stage +/// `DuckDbSink`'s deferred-endpoint bookkeeping and break its `MissingEndpointPolicy::Create` +/// gating. +#[cfg(feature = "ocel-duckdb")] +#[derive(Debug)] +struct FanOutSink<'a> { + a: &'a mut dyn ExtractionSink, + b: &'a mut dyn ExtractionSink, + event_handles: HashMap, + object_handles: HashMap, +} + +#[cfg(feature = "ocel-duckdb")] +impl FanOutSink<'_> { + fn event_id(r: &EventRef) -> Result<&str, SinkError> { + match r { + EventRef::Id(s) => Ok(s.as_str()), + EventRef::Index(_) => Err(SinkError::InvalidRef), + } + } + + fn object_id(r: &ObjectRef) -> Result<&str, SinkError> { + match r { + ObjectRef::Id(s) => Ok(s.as_str()), + ObjectRef::Index(_) => Err(SinkError::InvalidRef), + } + } + + /// Both inner sinks' remembered handles for an object id. + fn object_handles(&self, id: &str) -> Result<(ObjectRef, ObjectRef), SinkError> { + self.object_handles + .get(id) + .cloned() + .ok_or(SinkError::InvalidRef) + } + + /// Both inner sinks' remembered handles for an event id. + fn event_handles(&self, id: &str) -> Result<(EventRef, EventRef), SinkError> { + self.event_handles + .get(id) + .cloned() + .ok_or(SinkError::InvalidRef) + } +} + +/// The handle inside a resolution, if it carries one. +#[cfg(feature = "ocel-duckdb")] +fn resolution_ref(r: &Resolution) -> Option { + match r { + Resolution::Exists(h) | Resolution::Deferred(h) => Some(h.clone()), + Resolution::Missing => None, + } +} + +/// The more informative of two resolutions: a definite `Exists`/`Missing` beats a `Deferred`, +/// which is a sink declining to answer and therefore compatible with either. +/// +/// It has to be this way round: an eager sink handed a relation against an entity it does not +/// have would answer [`SinkError::InvalidRef`]. That is also why a fan-out run never exercises a +/// deferral branch, which [`extract_separately`] covers instead. +#[cfg(feature = "ocel-duckdb")] +fn stronger(a: Resolution, b: Resolution) -> Resolution { + match (a, b) { + (Resolution::Deferred(_), other) | (other, Resolution::Deferred(_)) => other, + (definite, _) => definite, + } +} + +#[cfg(feature = "ocel-duckdb")] +impl Resolution { + /// Replace the handle, keeping which of the three answers this is. + fn map_ref(self, f: impl FnOnce(R) -> T) -> Resolution { + match self { + Resolution::Exists(r) => Resolution::Exists(f(r)), + Resolution::Missing => Resolution::Missing, + Resolution::Deferred(r) => Resolution::Deferred(f(r)), + } + } +} + +#[cfg(feature = "ocel-duckdb")] +impl ExtractionSink for FanOutSink<'_> { + fn declare_event_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + self.a.declare_event_type(name, attrs)?; + self.b.declare_event_type(name, attrs)?; + Ok(()) + } + + fn declare_object_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + self.a.declare_object_type(name, attrs)?; + self.b.declare_object_type(name, attrs)?; + Ok(()) + } + + fn add_event( + &mut self, + event_type: &str, + time: DateTime, + id: &str, + attributes: &[(String, OCELAttributeValue)], + ) -> Result { + let ra = self.a.add_event(event_type, time, id, attributes)?; + let rb = self.b.add_event(event_type, time, id, attributes)?; + self.event_handles.insert(id.to_string(), (ra, rb)); + Ok(EventRef::Id(id.to_string())) + } + + fn add_object( + &mut self, + object_type: &str, + id: &str, + attributes: &[(String, DateTime, OCELAttributeValue)], + ) -> Result { + let ra = self.a.add_object(object_type, id, attributes)?; + let rb = self.b.add_object(object_type, id, attributes)?; + self.object_handles.insert(id.to_string(), (ra, rb)); + Ok(ObjectRef::Id(id.to_string())) + } + + fn add_object_attribute( + &mut self, + object: &ObjectRef, + name: &str, + time: DateTime, + value: OCELAttributeValue, + ) -> Result<(), SinkError> { + let id = Self::object_id(object)?; + let (ra, rb) = self.object_handles(id)?; + self.a + .add_object_attribute(&ra, name, time, value.clone())?; + self.b.add_object_attribute(&rb, name, time, value)?; + Ok(()) + } + + fn set_missing_endpoint_policy( + &mut self, + policy: crate::core::event_data::object_centric::extraction::MissingEndpointPolicy, + ) -> Result<(), SinkError> { + self.a.set_missing_endpoint_policy(policy)?; + self.b.set_missing_endpoint_policy(policy)?; + Ok(()) + } + + /// Both inner sinks are asked, so a deferring one records the endpoint it will have to + /// settle at `finalize`; the answer relayed to the extractor is the stronger of the two, + /// since a definite `Exists`/`Missing` is what the extractor can act on now and a `Deferred` + /// sink accepts either outcome. + fn resolve_event(&mut self, id: &str, event_type: Option<&str>) -> Resolution { + let a = self.a.resolve_event(id, event_type); + let b = self.b.resolve_event(id, event_type); + if let (Some(ra), Some(rb)) = (resolution_ref(&a), resolution_ref(&b)) { + self.event_handles.insert(id.to_string(), (ra, rb)); + } + stronger(a, b).map_ref(|_| EventRef::Id(id.to_string())) + } + + /// See [`resolve_event`](ExtractionSink::resolve_event). + fn resolve_object(&mut self, id: &str, object_type: Option<&str>) -> Resolution { + let a = self.a.resolve_object(id, object_type); + let b = self.b.resolve_object(id, object_type); + if let (Some(ra), Some(rb)) = (resolution_ref(&a), resolution_ref(&b)) { + self.object_handles.insert(id.to_string(), (ra, rb)); + } + stronger(a, b).map_ref(|_| ObjectRef::Id(id.to_string())) + } + + fn finalize(&mut self) -> Result { + self.a.finalize()?; + self.b.finalize() + } + + fn add_e2o( + &mut self, + event: &EventRef, + object: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let eid = Self::event_id(event)?; + let oid = Self::object_id(object)?; + let (ea, eb) = self.event_handles(eid)?; + let (oa, ob) = self.object_handles(oid)?; + self.a.add_e2o(&ea, &oa, qualifier)?; + self.b.add_e2o(&eb, &ob, qualifier)?; + Ok(()) + } + + fn add_o2o( + &mut self, + source: &ObjectRef, + target: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let sid = Self::object_id(source)?; + let tid = Self::object_id(target)?; + let (sa, sb) = self.object_handles(sid)?; + let (ta, tb) = self.object_handles(tid)?; + self.a.add_o2o(&sa, &ta, qualifier)?; + self.b.add_o2o(&sb, &tb, qualifier)?; + Ok(()) + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/duckdb_sink.rs b/process_mining/src/core/event_data/object_centric/extraction/duckdb_sink.rs new file mode 100644 index 00000000..651ceb32 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/duckdb_sink.rs @@ -0,0 +1,1086 @@ +//! Streaming [`ExtractionSink`] writing directly to a `DuckDB` file, reusing the schema and value +//! conversions from `ocel_sql::duckdb::schema`. +//! +//! Relation endpoints are not resolved eagerly: streaming to disk leaves no id index to look them +//! up in. Every [`resolve_event`](ExtractionSink::resolve_event) / +//! [`resolve_object`](ExtractionSink::resolve_object) answers [`Resolution::Deferred`], relations +//! are written unresolved, and [`finalize`](ExtractionSink::finalize) settles them with set-based +//! joins. This is safe because `e2o`/`o2o` carry no foreign key onto `events`/`objects`. +//! Under [`MissingEndpointPolicy::Create`] the object endpoints are additionally staged, since +//! that policy needs to know the type an id was named under. +//! +//! `on_missing_endpoint` behaves like in the eager sink, but is applied at finalize instead of at +//! the call site, and unresolved counts are reported in [`FinalizeReport::unresolved_endpoints`] +//! instead of per-mapping [`DropReason::UnresolvedEndpoint`](super::report::DropReason). +//! +//! Duplicate ids are not deferred: `events.id` and `objects.id` are `PRIMARY KEY`, so `DuckDB` +//! rejects repeats and they are reported as [`SinkError::DuplicateEvent`]/ +//! [`SinkError::DuplicateObject`], matching the in-memory sink. +//! +//! Two divergences from the eager sink remain: +//! +//! * Attribute columns of the wide `events` table are typed from the +//! [`declare_event_type`](ExtractionSink::declare_event_type) calls seen before the first +//! [`add_event`](ExtractionSink::add_event). Later declarations add columns on demand, but +//! cannot re-type a frozen one. +//! * Relation endpoints are not type-checked at add time, so under +//! [`IdRendering::Raw`](super::blueprint::IdRendering::Raw) an id shared by two object types +//! resolves to whichever type already holds it, where the eager sink reports a collision. +//! [`IdRendering::TypePrefixed`](super::blueprint::IdRendering::TypePrefixed) avoids this. + +use std::collections::HashMap; +use std::path::Path; + +use chrono::{DateTime, FixedOffset}; +use duckdb::{types::Value as DuckValue, Connection, ToSql}; + +use crate::core::event_data::object_centric::ocel_sql::duckdb::schema::tables::{ + build_events_table, create_indexes, create_schema, event_attr_sql_type, quote_ident, T_E2O, + T_EVENTS, T_EVENT_ATTR_META, T_O2O, T_OBJECTS, T_OBJECT_ATTR_CHANGES, T_OBJECT_ATTR_META, +}; +use crate::core::event_data::object_centric::ocel_sql::duckdb::schema::value::{ + datetime_to_duck_timestamp, ocel_value_to_duck, to_sql_value, +}; +use crate::core::event_data::object_centric::ocel_struct::OCELAttributeType; +use crate::core::event_data::object_centric::{OCELAttributeValue, OCELTypeAttribute}; + +use super::blueprint::MissingEndpointPolicy; +use super::sink::{EventRef, ExtractionSink, FinalizeReport, ObjectRef, Resolution, SinkError}; + +/// Staging table for every endpoint this sink was asked to resolve and deferred: +/// `(id, ocel_type, seq)`, one row per ask. `seq` is the order the asks arrived in, so +/// [`DuckDbSink::resolve_deferred`] can pick the first type an id was named under, as the eager +/// sink does, but only among the asks [`T_ENDPOINT_GATE`] does not rule out. Dropped again by +/// [`DuckDbSink::finalize`], so it never reaches a reader. +const T_DEFERRED: &str = "_extraction_deferred_objects"; + +/// Which [`T_DEFERRED`] asks were made for a relation row whose other endpoint might not +/// exist: `(seq, gate_id, gate_kind)`, where `gate_kind` is `'event'` for an +/// [`add_e2o`](DuckDbSink::add_e2o) and `'object'` for an [`add_o2o`](DuckDbSink::add_o2o). +/// +/// This sink cannot fail [`resolve_event`](ExtractionSink::resolve_event) or +/// [`resolve_object`](ExtractionSink::resolve_object) the way an eager sink does, so it stages a +/// relation's object endpoint even on a row the eager path abandons earlier: +/// +/// - `run_e2o` gives up before looking at the object endpoint when the event endpoint does not +/// resolve, so an `E2O` object ask on a row with no such event is one the eager path never +/// makes. +/// - `run_o2o` never reaches a row's targets when its source cannot be resolved or created, which +/// happens under [`MissingEndpointPolicy::Create`] whenever the source endpoint's `object_type` +/// expression yields nothing on that row. +/// +/// Left in [`T_DEFERRED`]'s `arg_min` unfiltered, such an ask could win the type for an id that is +/// reachable through a different, real row, picking whichever ask merely arrived first rather than +/// the one the eager path would have made. [`DuckDbSink::resolve_deferred`] excludes an ask whose +/// gate row names an entity that neither exists nor is about to be created. See the two strata +/// there for why an `'object'` gate cannot simply test `objects`. +/// +/// The gate's predicate is satisfied by any id present in `objects`, whatever type that row +/// carries, so it lets through the target ask of an `O2O` whose source id is taken by another +/// type. That is the same blind spot the module docs describe for relation endpoints, and +/// `IdRendering::TypePrefixed` removes it. +/// +/// Populated by `add_e2o`/`add_o2o` reading [`DuckDbSink::last_object_ask_seq`], which +/// `resolve_object` sets to the seq it just staged. This relies on the calling contracts on +/// [`ExtractionSink::add_e2o`]/[`add_o2o`](ExtractionSink::add_o2o). An `O2O`'s source is not +/// gated: the eager path resolves it before looking at any target, so a source ask is always one +/// the eager path also makes. +const T_ENDPOINT_GATE: &str = "_extraction_deferred_endpoint_gate"; + +/// Unique index on `object_attribute_changes (id, name, "time")`, which is how this sink keeps the +/// first value written at an instant and silently discards a repeat, as +/// [`add_object_attribute`](ExtractionSink::add_object_attribute) requires. +/// +/// An index rather than a check: reading the table back per row scans a table that grows with the +/// run, and deleting the duplicates at finalize would write a row per repeat to disk first. +/// `DuckDB` rejects the insert against its ART index in log time instead. Left in the finished +/// database, since one value per `(id, attribute, time)` is a property of a well-formed OCEL. +const I_OBJECT_ATTR_ONCE: &str = "_extraction_object_attribute_once"; + +/// Streaming [`ExtractionSink`] that writes directly to a `DuckDB` file in the consolidated +/// schema. See the module docs for how it resolves relation endpoints without an in-memory OCEL. +/// +/// Handles it hands out are always [`EventRef::Id`]/[`ObjectRef::Id`], echoing the id it was +/// given, since it has no index to hand back instead. +/// +/// Construct with [`DuckDbSink::new`], run an [`extract`](super::extract::extract) against it, +/// then call [`DuckDbSink::finalize`] before reading the file back (with +/// [`read_ocel_from_duckdb`](crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb), +/// for instance). +#[derive(Debug)] +pub struct DuckDbSink { + con: Connection, + declared_event_types: std::collections::HashSet, + declared_object_types: std::collections::HashSet, + /// What [`ExtractionSink::finalize`] applies to the endpoints this sink deferred. + on_missing_endpoint: MissingEndpointPolicy, + /// `finalize` is idempotent. This is how. + finalized: bool, + /// Event-attribute schema accumulated over every `declare_event_type` call, widened via + /// [`OCELAttributeType::coalesce`] when two declarations disagree on a name's type. + ev_attr_types: HashMap, + /// `ev_attr_types` frozen into ordered, typed wide columns on the first `add_event`, then + /// extended (typed from `ev_attr_types` when known) by any later new name. + ev_columns: Vec<(String, OCELAttributeType)>, + /// name -> index into `ev_columns`. + ev_col_index: HashMap, + events_created: bool, + /// Next `seq` for [`T_DEFERRED`]: one counter, not one entry per id. + deferred_seq: i64, + /// The `seq` [`resolve_object`](ExtractionSink::resolve_object) most recently staged into + /// [`T_DEFERRED`], consumed by [`add_e2o`](ExtractionSink::add_e2o) to populate + /// [`T_ENDPOINT_GATE`]. See that constant's docs for the calling-order invariant this relies on. + last_object_ask_seq: Option, + /// `e2o` rows not yet appended. See [`RELATION_BUFFER`]. + e2o_buffer: Vec<(String, String, String)>, + /// `o2o` rows not yet appended. See [`RELATION_BUFFER`]. + o2o_buffer: Vec<(String, String, String)>, +} + +/// How many relation rows are held before an appender is opened for them. +/// +/// Neither `e2o` nor `o2o` carries a constraint, so there is nothing for an immediate flush to +/// surface, unlike `events`/`objects`, whose primary key is how this sink reports a repeated id at +/// the call site. +const RELATION_BUFFER: usize = 4096; + +fn backend(e: duckdb::Error) -> SinkError { + SinkError::Backend(e.to_string()) +} + +/// Append one row through a short-lived appender and flush it, so a constraint violation surfaces +/// here rather than at some later, unrelated flush. +fn append_one( + con: &Connection, + table: &str, + write: impl FnOnce(&mut duckdb::Appender<'_>) -> Result<(), duckdb::Error>, +) -> Result<(), duckdb::Error> { + let mut ap = con.appender(table)?; + write(&mut ap)?; + ap.flush() +} + +/// The write-ahead log `DuckDB` keeps beside a database file: the database's own path with +/// `.wal` appended, not with its extension replaced. +fn wal_sidecar(path: &Path) -> std::path::PathBuf { + let mut wal = path.as_os_str().to_owned(); + wal.push(".wal"); + std::path::PathBuf::from(wal) +} + +/// Whether a `DuckDB` error is a primary-key violation. Matched on the message: the driver +/// exposes no error code for it. +fn is_duplicate_key(e: &duckdb::Error) -> bool { + let message = e.to_string().to_ascii_lowercase(); + message.contains("duplicate key") || message.contains("primary key") +} + +impl DuckDbSink { + /// Open a fresh `DuckDB` database at `path` (an existing file at that path is replaced) and + /// create the consolidated schema's static tables. + /// + /// # Errors + /// Returns [`SinkError::Backend`] if the existing file cannot be removed, the database cannot + /// be opened, or schema creation fails. + pub fn new(path: impl AsRef) -> Result { + let path = path.as_ref(); + if path.exists() { + std::fs::remove_file(path).map_err(|e| SinkError::Backend(e.to_string()))?; + } + // A crashed earlier run leaves a write-ahead log beside the file it belonged to, which + // `DuckDB` would replay into this fresh database. + let wal = wal_sidecar(path); + if wal.exists() { + std::fs::remove_file(&wal).map_err(|e| SinkError::Backend(e.to_string()))?; + } + let con = Connection::open(path).map_err(backend)?; + create_schema(&con).map_err(backend)?; + con.execute_batch(&format!( + "CREATE TABLE {T_DEFERRED} (id TEXT, ocel_type TEXT, seq BIGINT)" + )) + .map_err(backend)?; + con.execute_batch(&format!( + "CREATE TABLE {T_ENDPOINT_GATE} (seq BIGINT, gate_id TEXT, gate_kind TEXT)" + )) + .map_err(backend)?; + con.execute_batch(&format!( + "CREATE UNIQUE INDEX {I_OBJECT_ATTR_ONCE} \ + ON {T_OBJECT_ATTR_CHANGES} (id, name, \"time\")" + )) + .map_err(backend)?; + // Deliberately not wrapped in a transaction, however much a failed run would benefit from + // one: `DuckDB` aborts a transaction on a constraint violation ("Current transaction is + // aborted"), and this sink swallows two of those by design: the duplicate key that + // reports a repeated id, and the [`I_OBJECT_ATTR_ONCE`] rejection enforcing first-wins on + // `(id, name, time)`. Under one transaction the first static object attribute written + // twice would fail every statement after it. + Ok(Self { + con, + declared_event_types: std::collections::HashSet::new(), + declared_object_types: std::collections::HashSet::new(), + on_missing_endpoint: MissingEndpointPolicy::default(), + finalized: false, + ev_attr_types: HashMap::new(), + ev_columns: Vec::new(), + ev_col_index: HashMap::new(), + events_created: false, + deferred_seq: 0, + last_object_ask_seq: None, + e2o_buffer: Vec::new(), + o2o_buffer: Vec::new(), + }) + } + + /// Whether this run needs [`T_DEFERRED`] and [`T_ENDPOINT_GATE`] at all. + /// + /// [`Self::resolve_deferred`] reads them only to synthesise missing objects, so under any + /// other policy every staged row is written and then dropped unread, costing two appender + /// create-and-flush cycles per relation endpoint. + fn stages_endpoints(&self) -> bool { + self.on_missing_endpoint == MissingEndpointPolicy::Create + } + + /// Count the rows one aggregate query returns. + fn count(&self, sql: &str) -> Result { + let n: i64 = self.con.query_row(sql, [], |r| r.get(0))?; + Ok(u64::try_from(n).unwrap_or(0)) + } + + /// The `WHERE` fragment selecting `e2o`/`o2o` rows with an endpoint that does not exist. + const UNRESOLVED_E2O: &'static str = "SELECT count(*) FROM e2o \ + WHERE event_id NOT IN (SELECT id FROM events) \ + OR object_id NOT IN (SELECT id FROM objects)"; + /// See [`Self::UNRESOLVED_E2O`]. + const UNRESOLVED_O2O: &'static str = "SELECT count(*) FROM o2o \ + WHERE source_id NOT IN (SELECT id FROM objects) \ + OR target_id NOT IN (SELECT id FROM objects)"; + + /// Settle every endpoint this sink deferred, applying `on_missing_endpoint`. + fn resolve_deferred(&mut self) -> Result { + let mut report = FinalizeReport::default(); + + if self.on_missing_endpoint == MissingEndpointPolicy::Create { + // One object per still-missing id, typed from the first reachable ask, matching the + // eager path where the first ask creates the object and a later ask under another type + // is an `IdTypeCollision` rather than a retype. + // + // "Reachable" matters here: this sink was asked about every endpoint a row named, + // including ones the eager path never touches, since it defers everything and cannot + // tell in advance. An unreachable ask must not win `arg_min` merely for having a + // smaller `seq`, so `T_ENDPOINT_GATE` rules those asks out before a type is picked. + // The final `WHERE` still confines creation to ids a surviving relation actually + // names. + // + // The two gate kinds are tested against different sets, and in two strata, because an + // `O2O` source is one of the objects this very statement creates: + // + // - `'event'` tests `events`, which this statement does not write, so it is final. + // - `'object'` tests the post-creation object set, not `objects`: under `Create` a + // relation-only source is created by this `INSERT`, so testing the pre-creation + // `objects` would rule out the target ask of every synthesised source, and the + // target would then never be created and the relation deleted for want of it. + // + // The post-creation set is `available` below (`objects UNION creatable`), but the gate + // cannot reference it: `available` derives from `creatable`, which derives from + // `reachable`, which is what the gate filters, and SQL will not evaluate that cycle. + // It is stratified instead. An `O2O`'s source ask is never gated (see + // `T_ENDPOINT_GATE`), so a source's own creatability is settled by the event stratum + // alone: `gate_available` (`objects` plus every id with an event-reachable typed ask) + // answers the object gate without needing `creatable`, and the strata terminate. + // + // This over-approximates in one place: an id whose only typed ask belongs to another + // relation counts as available, even though that ask might itself be gated out. The + // eager path is order-dependent on exactly that shape, so no set-based rule matches it + // row for row. Erring towards available keeps `Create` working when a source is itself + // relation-only, which is what this stratification exists for. + let created = self.con.execute( + &format!( + "INSERT INTO objects (id, ocel_type) \ + WITH event_reachable AS ( \ + SELECT d.id AS id, d.ocel_type AS ocel_type, d.seq AS seq \ + FROM {T_DEFERRED} d \ + WHERE d.ocel_type IS NOT NULL \ + AND d.seq NOT IN ( \ + SELECT g.seq FROM {T_ENDPOINT_GATE} g \ + WHERE g.gate_kind = 'event' \ + AND g.gate_id NOT IN (SELECT id FROM events) \ + ) \ + ), gate_available AS ( \ + SELECT id FROM objects UNION ALL SELECT id FROM event_reachable \ + ), reachable AS ( \ + SELECT r.id AS id, r.ocel_type AS ocel_type, r.seq AS seq \ + FROM event_reachable r \ + WHERE r.seq NOT IN ( \ + SELECT g.seq FROM {T_ENDPOINT_GATE} g \ + WHERE g.gate_kind = 'object' \ + AND g.gate_id NOT IN (SELECT id FROM gate_available) \ + ) \ + ), creatable AS ( \ + SELECT id, arg_min(ocel_type, seq) AS ocel_type \ + FROM reachable \ + WHERE id NOT IN (SELECT id FROM objects) \ + GROUP BY id \ + ), available AS ( \ + SELECT id FROM objects UNION ALL SELECT id FROM creatable \ + ) \ + SELECT c.id, c.ocel_type FROM creatable c \ + WHERE c.id IN (SELECT object_id FROM e2o \ + WHERE event_id IN (SELECT id FROM events)) \ + OR c.id IN (SELECT source_id FROM o2o) \ + OR c.id IN (SELECT target_id FROM o2o \ + WHERE source_id IN (SELECT id FROM available))" + ), + [], + )?; + report.objects_created = created as u64; + } + + report.unresolved_endpoints = + self.count(Self::UNRESOLVED_E2O)? + self.count(Self::UNRESOLVED_O2O)?; + self.con.execute_batch( + "DELETE FROM e2o WHERE event_id NOT IN (SELECT id FROM events) \ + OR object_id NOT IN (SELECT id FROM objects); \ + DELETE FROM o2o WHERE source_id NOT IN (SELECT id FROM objects) \ + OR target_id NOT IN (SELECT id FROM objects);", + )?; + report.resolved_relations = + self.count("SELECT count(*) FROM e2o")? + self.count("SELECT count(*) FROM o2o")?; + + self.con.execute_batch(&format!( + "DROP TABLE {T_DEFERRED}; DROP TABLE {T_ENDPOINT_GATE}" + ))?; + Ok(report) + } + + fn event_id(r: &EventRef) -> Result<&str, SinkError> { + match r { + EventRef::Id(s) => Ok(s.as_str()), + EventRef::Index(_) => Err(SinkError::InvalidRef), + } + } + + fn object_id(r: &ObjectRef) -> Result<&str, SinkError> { + match r { + ObjectRef::Id(s) => Ok(s.as_str()), + ObjectRef::Index(_) => Err(SinkError::InvalidRef), + } + } + + /// Freeze `ev_attr_types` into `ev_columns`/`ev_col_index` and create the wide `events` + /// table. Idempotent, and called on the first `add_event` and, as a safety net for an + /// event-less run, from `finalize`. + fn ensure_events_created(&mut self) -> Result<(), duckdb::Error> { + if self.events_created { + return Ok(()); + } + let mut names: Vec = self.ev_attr_types.keys().cloned().collect(); + names.sort(); + self.ev_columns = names + .into_iter() + .map(|n| { + let ty = self.ev_attr_types[&n]; + (n, ty) + }) + .collect(); + self.ev_col_index = self + .ev_columns + .iter() + .enumerate() + .map(|(i, (n, _))| (n.clone(), i)) + .collect(); + self.con + .execute_batch(&build_events_table(&self.ev_columns))?; + self.events_created = true; + Ok(()) + } + + /// Add a column for an attribute name the wide `events` table does not yet have. Typed from + /// `ev_attr_types` when a declaration already named it (the normal case: see the module docs + /// on why a dynamically-named event type's later-arriving column still lands here typed, not + /// as text). Falls back to `VARCHAR` only if reached with no declaration at all, which does + /// not happen through [`extract`](super::extract::extract), where every name `add_event` is + /// called with came from the same attribute list its `declare_event_type` call used. + fn add_event_column(&mut self, name: &str) -> Result<(), duckdb::Error> { + let ty = self + .ev_attr_types + .get(name) + .copied() + .unwrap_or(OCELAttributeType::String); + self.con.execute_batch(&format!( + "ALTER TABLE {T_EVENTS} ADD COLUMN {} {}", + quote_ident(name), + event_attr_sql_type(ty) + ))?; + self.ev_col_index + .insert(name.to_string(), self.ev_columns.len()); + self.ev_columns.push((name.to_string(), ty)); + Ok(()) + } + + /// Turn an `objects.id` primary-key violation into the right [`SinkError`]. `DuckDB`'s error + /// says only that `id` repeats, not under what type, so this looks the existing row up. The + /// same type is an ordinary [`SinkError::DuplicateObject`]. A different type (or, defensively, + /// a lookup that somehow finds no row at all) is [`SinkError::IdTypeCollision`], which + /// [`mapping_exec`](super::mapping_exec) must not treat as "the id already exists, append". + /// A lookup that fails is neither, and is reported as the backend error it is. + /// + /// This is the one place `resolve_object`'s inability to check a declared type against the + /// object's actual one (see the module docs) is recovered from: one query, only on an actual + /// conflict, rather than a per-id structure held for the whole run. + fn classify_duplicate_object(&self, id: &str, object_type: &str) -> SinkError { + let existing_type: Option = match self.con.query_row( + &format!("SELECT ocel_type FROM {T_OBJECTS} WHERE id = ?"), + [id], + |r| r.get(0), + ) { + Ok(t) => Some(t), + Err(duckdb::Error::QueryReturnedNoRows) => None, + // A query that failed says nothing about the existing row's type. Reporting a + // collision here would have the extractor drop the row as two entities colliding, + // losing data to what may be a transient backend problem. + Err(e) => return backend(e), + }; + if existing_type.as_deref() == Some(object_type) { + SinkError::DuplicateObject { id: id.to_string() } + } else { + SinkError::IdTypeCollision { id: id.to_string() } + } + } + + /// The `(value, value_type)` pair one object attribute is stored as. + /// + /// [`to_sql_value`] alone is not enough: it renders [`OCELAttributeValue::Null`] as + /// `("", "string")`, which `from_sql_value` reads back as `String("")`, where + /// [`SlimOcelSink`](super::slim_sink::SlimOcelSink) still holds `Null`. That is an accepted + /// caveat of `write_ocel_to_duckdb`'s own round trip, but a divergence between the two + /// extraction sinks, which must agree. `"null"` is + /// [`OCELAttributeType::Null`]'s own type string, and `from_sql_value` maps it straight back + /// to `OCELAttributeValue::Null` regardless of the stored text. + fn object_attr_sql_value(v: &OCELAttributeValue) -> (std::borrow::Cow<'_, str>, &'static str) { + match v { + OCELAttributeValue::Null => (std::borrow::Cow::Borrowed(""), "null"), + other => to_sql_value(other), + } + } + + /// Write one `object_attribute_changes` row, keeping whatever was already recorded at this + /// `(id, name, time)`. A rejection from [`I_OBJECT_ATTR_ONCE`] means a value is there already, + /// which is the rule rather than a failure, see + /// [`add_object_attribute`](ExtractionSink::add_object_attribute). Any other backend error is + /// still reported. + fn write_object_attribute( + &self, + id: &str, + name: &str, + time: DateTime, + value: &OCELAttributeValue, + ) -> Result<(), SinkError> { + let (value_str, value_type) = Self::object_attr_sql_value(value); + let t = datetime_to_duck_timestamp(time); + match append_one(&self.con, T_OBJECT_ATTR_CHANGES, |ap| { + ap.append_row((id, name, &t, value_str.as_ref(), value_type)) + }) { + Ok(()) => Ok(()), + Err(e) if is_duplicate_key(&e) => Ok(()), + Err(e) => Err(backend(e)), + } + } + + /// Record that the object ask [`resolve_object`](ExtractionSink::resolve_object) most + /// recently staged was made on behalf of a relation row gated by `gate_id`: an event id for + /// an `E2O`, the source object id for an `O2O`. Consumes the pending seq, so a second relation + /// written without an intervening ask gates nothing. + /// + /// A failure to record the gate row is swallowed, but not harmless: an ungated ask can win + /// `arg_min` for an id a real row also names, so `Create` synthesises that object under a type + /// the eager path would never have used, and both sinks then hold the same id under different + /// types. The relation itself survives either way, only its type moves; the opposite mistake, + /// an over-eager gate, is what costs a relation (see `resolve_deferred`). It is swallowed + /// because failing the whole extraction over an appender error here would be worse, and + /// nothing this sink can do at this point would repair the ask. + fn gate_last_ask(&mut self, gate_kind: &'static str, gate_id: &str) { + let Some(seq) = self.last_object_ask_seq.take() else { + return; + }; + if let Ok(mut gate) = self.con.appender(T_ENDPOINT_GATE) { + let _ = gate.append_row(duckdb::params![seq, gate_id, gate_kind]); + let _ = gate.flush(); + } + } + + /// Append every buffered relation row of one kind and empty the buffer. + fn drain_relations( + con: &Connection, + table: &str, + buffer: &mut Vec<(String, String, String)>, + ) -> Result<(), SinkError> { + if buffer.is_empty() { + return Ok(()); + } + append_one(con, table, |ap| { + for (a, b, qualifier) in buffer.iter() { + ap.append_row([a.as_str(), b.as_str(), qualifier.as_str()])?; + } + Ok(()) + }) + .map_err(backend)?; + buffer.clear(); + Ok(()) + } + + /// Append both relation buffers, so a reader of `e2o`/`o2o` sees every row written so far. + fn drain_all_relations(&mut self) -> Result<(), SinkError> { + Self::drain_relations(&self.con, T_E2O, &mut self.e2o_buffer)?; + Self::drain_relations(&self.con, T_O2O, &mut self.o2o_buffer) + } +} + +impl ExtractionSink for DuckDbSink { + fn declare_event_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + self.declared_event_types.insert(name.to_string()); + if attrs.is_empty() { + return Ok(()); + } + // one row per `(event_type, attr_name)`, last declaration winning. `SlimOcelSink` + // overwrites a redeclared type wholesale (`locel.add_event_type`), while the reader's + // `collect_types` keeps the first row per pair, from a `SELECT` with no `ORDER BY`, so + // "first" was not even stable. Deleting the pair before appending collapses both rules + // onto the same answer. + let mut stmt = self + .con + .prepare(&format!( + "DELETE FROM {T_EVENT_ATTR_META} WHERE event_type = ? AND attr_name = ?" + )) + .map_err(backend)?; + for a in attrs { + stmt.execute(duckdb::params![name, a.name.as_str()]) + .map_err(backend)?; + } + drop(stmt); + + let mut ap = self.con.appender(T_EVENT_ATTR_META).map_err(backend)?; + for a in attrs { + let ty = OCELAttributeType::from_type_str(&a.value_type); + self.ev_attr_types + .entry(a.name.clone()) + .and_modify(|existing| *existing = existing.coalesce(ty)) + .or_insert(ty); + ap.append_row([name, a.name.as_str(), a.value_type.as_str()]) + .map_err(backend)?; + } + ap.flush().map_err(backend)?; + Ok(()) + } + + /// Records the declaration in `T_OBJECT_ATTR_META`, the object-side mirror of + /// `T_EVENT_ATTR_META`. + /// + /// Object attribute values are EAV (`object_attribute_changes`) rather than typed wide + /// columns, so unlike events there is nothing to declare structurally. The declaration is + /// still persisted, because it is the only record of an attribute no row ever wrote, and of a + /// type whose mapping matched nothing, which [`extract`](super::extract::extract) declares up + /// front on purpose. Rebuilding either from the observed change rows loses both, and + /// splits one attribute observed once as `Null` and once as an integer into two entries of + /// the same name. + /// + /// A type that declares no attributes at all still leaves no trace, exactly as an event type + /// with no attributes and no events does: the type list itself lives in `objects`/`events`. + fn declare_object_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + self.declared_object_types.insert(name.to_string()); + if attrs.is_empty() { + return Ok(()); + } + // One row per `(object_type, attr_name)`, last declaration winning. See + // `declare_event_type` for why the pair is deleted before it is appended. + let mut stmt = self + .con + .prepare(&format!( + "DELETE FROM {T_OBJECT_ATTR_META} WHERE object_type = ? AND attr_name = ?" + )) + .map_err(backend)?; + for a in attrs { + stmt.execute(duckdb::params![name, a.name.as_str()]) + .map_err(backend)?; + } + drop(stmt); + + let mut ap = self.con.appender(T_OBJECT_ATTR_META).map_err(backend)?; + for a in attrs { + ap.append_row([name, a.name.as_str(), a.value_type.as_str()]) + .map_err(backend)?; + } + ap.flush().map_err(backend)?; + Ok(()) + } + + fn add_event( + &mut self, + event_type: &str, + time: DateTime, + id: &str, + attributes: &[(String, OCELAttributeValue)], + ) -> Result { + if !self.declared_event_types.contains(event_type) { + return Err(SinkError::UnknownType { + kind: "event", + name: event_type.to_string(), + }); + } + self.ensure_events_created().map_err(backend)?; + for (name, _) in attributes { + if !self.ev_col_index.contains_key(name) { + self.add_event_column(name).map_err(backend)?; + } + } + + let mut row: Vec = Vec::with_capacity(3 + self.ev_columns.len()); + row.push(DuckValue::Text(id.to_string())); + row.push(DuckValue::Text(event_type.to_string())); + row.push(datetime_to_duck_timestamp(time)); + row.extend(std::iter::repeat_n(DuckValue::Null, self.ev_columns.len())); + for (name, value) in attributes { + let idx = self.ev_col_index[name]; + row[3 + idx] = ocel_value_to_duck(value, self.ev_columns[idx].1); + } + let params: Vec<&dyn ToSql> = row.iter().map(|v| v as &dyn ToSql).collect(); + // `events.id` is a PRIMARY KEY, so a repeat is rejected here rather than by a set this + // sink would otherwise have to hold in memory. See the module docs. + append_one(&self.con, T_EVENTS, |ap| ap.append_row(params.as_slice())).map_err(|e| { + if is_duplicate_key(&e) { + SinkError::DuplicateEvent { id: id.to_string() } + } else { + backend(e) + } + })?; + Ok(EventRef::Id(id.to_string())) + } + + fn add_object( + &mut self, + object_type: &str, + id: &str, + attributes: &[(String, DateTime, OCELAttributeValue)], + ) -> Result { + if !self.declared_object_types.contains(object_type) { + return Err(SinkError::UnknownType { + kind: "object", + name: object_type.to_string(), + }); + } + // See `add_event`: `objects.id` is a PRIMARY KEY. + append_one(&self.con, T_OBJECTS, |ap| ap.append_row([id, object_type])).map_err(|e| { + if is_duplicate_key(&e) { + self.classify_duplicate_object(id, object_type) + } else { + backend(e) + } + })?; + + for (name, time, value) in attributes { + self.write_object_attribute(id, name, *time, value)?; + } + + Ok(ObjectRef::Id(id.to_string())) + } + + /// First-wins on `(id, name, time)`, enforced by `I_OBJECT_ATTR_ONCE`. + fn add_object_attribute( + &mut self, + object: &ObjectRef, + name: &str, + time: DateTime, + value: OCELAttributeValue, + ) -> Result<(), SinkError> { + let id = Self::object_id(object)?; + self.write_object_attribute(id, name, time, &value) + } + + fn set_missing_endpoint_policy( + &mut self, + policy: MissingEndpointPolicy, + ) -> Result<(), SinkError> { + self.on_missing_endpoint = policy; + Ok(()) + } + + /// Always [`Resolution::Deferred`]: answering would need an id index this sink deliberately + /// does not keep. See the module docs. + fn resolve_event(&mut self, id: &str, _event_type: Option<&str>) -> Resolution { + Resolution::Deferred(EventRef::Id(id.to_string())) + } + + /// Always [`Resolution::Deferred`], recording `object_type` so + /// [`finalize`](ExtractionSink::finalize) can honour `on_missing_endpoint: Create` for an + /// endpoint that turns out not to exist. + fn resolve_object(&mut self, id: &str, object_type: Option<&str>) -> Resolution { + // Nothing reads what would be staged unless the policy is `Create`. + if !self.stages_endpoints() { + return Resolution::Deferred(ObjectRef::Id(id.to_string())); + } + // A failure here would only cost `Create` an object it could have synthesised. The + // relation itself is still written and still resolved (or dropped) at finalize, so this + // is deliberately not turned into a resolution failure. + let seq = self.deferred_seq; + self.deferred_seq += 1; + if let Ok(mut ap) = self.con.appender(T_DEFERRED) { + let _ = ap.append_row(duckdb::params![id, object_type, seq]); + let _ = ap.flush(); + } + self.last_object_ask_seq = Some(seq); + Resolution::Deferred(ObjectRef::Id(id.to_string())) + } + + fn finalize(&mut self) -> Result { + if self.finalized { + return Ok(FinalizeReport::default()); + } + self.drain_all_relations()?; + self.ensure_events_created().map_err(backend)?; + let report = self.resolve_deferred().map_err(backend)?; + create_indexes(&self.con).map_err(backend)?; + self.con.execute_batch("CHECKPOINT").map_err(backend)?; + self.finalized = true; + Ok(report) + } + + fn add_e2o( + &mut self, + event: &EventRef, + object: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let e = Self::event_id(event)?.to_string(); + let o = Self::object_id(object)?.to_string(); + self.e2o_buffer.push((e.clone(), o, qualifier.to_string())); + if self.e2o_buffer.len() >= RELATION_BUFFER { + Self::drain_relations(&self.con, T_E2O, &mut self.e2o_buffer)?; + } + // See `T_ENDPOINT_GATE`'s docs: this links the object ask just staged (if any) to the + // event this row names, so `resolve_deferred` can tell a real ask from one made on behalf + // of a row whose event never resolves. + self.gate_last_ask("event", &e); + Ok(()) + } + + fn add_o2o( + &mut self, + source: &ObjectRef, + target: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let s = Self::object_id(source)?.to_string(); + let t = Self::object_id(target)?.to_string(); + self.o2o_buffer.push((s.clone(), t, qualifier.to_string())); + if self.o2o_buffer.len() >= RELATION_BUFFER { + Self::drain_relations(&self.con, T_O2O, &mut self.o2o_buffer)?; + } + // The target ask is the immediately preceding `resolve_object` (see `add_o2o`'s contract + // on `ExtractionSink`), so the same gate that saves `E2O` covers `O2O` too, keyed on the + // source, which is what the eager path gives up on before it reaches any target. + self.gate_last_ask("object", &s); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use tempfile::tempdir; + + use super::*; + use crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb; + + fn open_sink() -> (tempfile::TempDir, DuckDbSink) { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("sink.duckdb"); + let sink = DuckDbSink::new(&path).expect("open sink"); + (dir, sink) + } + + #[test] + fn unknown_type_is_rejected_for_events_and_objects() { + let (_dir, mut sink) = open_sink(); + let t = chrono::Utc::now().fixed_offset(); + assert!(matches!( + sink.add_event("Nope", t, "e1", &[]), + Err(SinkError::UnknownType { kind: "event", .. }) + )); + assert!(matches!( + sink.add_object("Nope", "o1", &[]), + Err(SinkError::UnknownType { kind: "object", .. }) + )); + } + + #[test] + fn duplicate_event_and_object_ids_are_rejected() { + let (_dir, mut sink) = open_sink(); + sink.declare_event_type("Pay", &[]).unwrap(); + sink.declare_object_type("Order", &[]).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.add_event("Pay", t, "e1", &[]).unwrap(); + sink.add_object("Order", "o1", &[]).unwrap(); + + assert_eq!( + sink.add_event("Pay", t, "e1", &[]), + Err(SinkError::DuplicateEvent { id: "e1".into() }) + ); + assert_eq!( + sink.add_object("Order", "o1", &[]), + Err(SinkError::DuplicateObject { id: "o1".into() }) + ); + } + + /// This sink never answers `Exists`/`Missing`: it keeps no id index, by design, and settles + /// every endpoint at `finalize` instead. + #[test] + fn every_resolution_is_deferred() { + let (_dir, mut sink) = open_sink(); + assert!(matches!( + sink.resolve_event("e1", None), + Resolution::Deferred(EventRef::Id(_)) + )); + assert!(matches!( + sink.resolve_object("o1", Some("Order")), + Resolution::Deferred(ObjectRef::Id(_)) + )); + } + + /// A relation written against a deferred endpoint that never materialises is deleted at + /// finalize under `Drop`, and counted as the same loss an eager sink reports per mapping via + /// `DropReason::UnresolvedEndpoint`. + #[test] + fn finalize_drops_relations_whose_deferred_endpoint_never_appeared() { + let (dir, mut sink) = open_sink(); + sink.set_missing_endpoint_policy(MissingEndpointPolicy::Drop) + .unwrap(); + sink.declare_event_type("Pay", &[]).unwrap(); + sink.declare_object_type("Order", &[]).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.add_event("Pay", t, "e1", &[]).unwrap(); + sink.add_object("Order", "o1", &[]).unwrap(); + + let ev = sink.resolve_event("e1", None).into_ref().unwrap(); + let good = sink.resolve_object("o1", Some("Order")).into_ref().unwrap(); + let ghost = sink + .resolve_object("nope", Some("Order")) + .into_ref() + .unwrap(); + sink.add_e2o(&ev, &good, "q").unwrap(); + sink.add_e2o(&ev, &ghost, "q").unwrap(); + + let report = ExtractionSink::finalize(&mut sink).unwrap(); + assert_eq!(report.unresolved_endpoints, 1); + assert_eq!(report.resolved_relations, 1); + assert_eq!(report.objects_created, 0); + + let con = duckdb::Connection::open(dir.path().join("sink.duckdb")).unwrap(); + let ocel = read_ocel_from_duckdb(&con).unwrap(); + let e1 = ocel.events.iter().find(|e| e.id == "e1").unwrap(); + assert_eq!(e1.relationships.len(), 1, "the dangling relation is gone"); + } + + /// `on_missing_endpoint: Create` is honoured at finalize, from the type the endpoint + /// declared, which is why `resolve_object` is given it. + #[test] + fn finalize_creates_missing_objects_under_the_create_policy() { + let (dir, mut sink) = open_sink(); + sink.set_missing_endpoint_policy(MissingEndpointPolicy::Create) + .unwrap(); + sink.declare_event_type("Pay", &[]).unwrap(); + sink.declare_object_type("Order", &[]).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.add_event("Pay", t, "e1", &[]).unwrap(); + let ev = sink.resolve_event("e1", None).into_ref().unwrap(); + let ghost = sink + .resolve_object("o-new", Some("Order")) + .into_ref() + .unwrap(); + sink.add_e2o(&ev, &ghost, "q").unwrap(); + + let report = ExtractionSink::finalize(&mut sink).unwrap(); + assert_eq!(report.objects_created, 1); + assert_eq!(report.unresolved_endpoints, 0); + assert_eq!(report.resolved_relations, 1); + + let con = duckdb::Connection::open(dir.path().join("sink.duckdb")).unwrap(); + let ocel = read_ocel_from_duckdb(&con).unwrap(); + let created = ocel.objects.iter().find(|o| o.id == "o-new").unwrap(); + assert_eq!(created.object_type, "Order"); + } + + /// A second event type declaring an attribute name under a genuinely different type than an + /// earlier declaration widens the accumulated declaration ([`OCELAttributeType::coalesce`]), + /// regardless of whether the wide `events` table already exists. Exercises the "declared + /// after the freeze, but still declared" path documented on [`DuckDbSink::add_event_column`]. + #[test] + fn a_column_declared_after_the_freeze_still_gets_its_declared_type() { + let (dir, mut sink) = open_sink(); + let t = chrono::Utc::now().fixed_offset(); + sink.declare_event_type( + "A", + &[OCELTypeAttribute { + name: "n".into(), + value_type: "integer".into(), + }], + ) + .unwrap(); + // Freezes the wide table with "n" as BIGINT. + sink.add_event( + "A", + t, + "e1", + &[("n".to_string(), OCELAttributeValue::Integer(1))], + ) + .unwrap(); + // A second, later-declared type gives "n" an unrelated but compatible attribute name + // "m", exercising the on-demand ALTER path rather than "n" itself (widening an existing + // column's type is a separate, inherited limitation documented on the sink). + sink.declare_event_type( + "B", + &[OCELTypeAttribute { + name: "m".into(), + value_type: "float".into(), + }], + ) + .unwrap(); + sink.add_event( + "B", + t, + "e2", + &[("m".to_string(), OCELAttributeValue::Float(2.5))], + ) + .unwrap(); + + let path = dir.path().join("sink.duckdb"); + ExtractionSink::finalize(&mut sink).unwrap(); + let con = duckdb::Connection::open(&path).unwrap(); + let ty: String = con + .query_row( + "SELECT data_type FROM information_schema.columns \ + WHERE table_name = 'events' AND column_name = 'm'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + ty, "DOUBLE", + "late column keeps its declared type, not VARCHAR" + ); + + let ocel = read_ocel_from_duckdb(&con).unwrap(); + let e2 = ocel.events.iter().find(|e| e.id == "e2").unwrap(); + assert_eq!( + e2.attributes.first().map(|a| &a.value), + Some(&OCELAttributeValue::Float(2.5)) + ); + } + + /// `SlimOcelSink` keeps the last declaration for a `(type, attribute)`, since + /// `locel.add_event_type` overwrites, while the `DuckDB` reader's `collect_types` keeps the + /// first row per `(type, name)`, from a `SELECT` with no `ORDER BY`. Appending a row per + /// `declare_event_type` call therefore made the two sinks disagree, and made "first" not even + /// stable across reads. One row per `(type, name)` makes first-wins and last-wins the same + /// answer. + #[test] + fn a_redeclared_event_attribute_type_reads_back_as_the_last_declaration() { + let (dir, mut sink) = open_sink(); + let declare = |sink: &mut DuckDbSink, value_type: &str| { + sink.declare_event_type( + "A", + &[OCELTypeAttribute { + name: "n".into(), + value_type: value_type.into(), + }], + ) + .unwrap(); + }; + declare(&mut sink, "integer"); + declare(&mut sink, "string"); + ExtractionSink::finalize(&mut sink).unwrap(); + + let con = duckdb::Connection::open(dir.path().join("sink.duckdb")).unwrap(); + let rows: i64 = con + .query_row( + &format!("SELECT count(*) FROM {T_EVENT_ATTR_META} WHERE attr_name = 'n'"), + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + rows, 1, + "one row per (type, attribute), so first-wins and last-wins coincide" + ); + let stored: String = con + .query_row( + &format!("SELECT attr_type FROM {T_EVENT_ATTR_META} WHERE attr_name = 'n'"), + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + stored, "string", + "the last declaration wins, as in SlimOcelSink" + ); + } + + /// `to_sql_value` renders `OCELAttributeValue::Null` as `("", "string")`, so the reader + /// hands back `String("")` where `SlimOcelSink` holds `Null`. The `value_type` column is the + /// sink's to choose, and `"null"` round-trips through `from_sql_value` exactly. + #[test] + fn a_null_object_attribute_round_trips_as_null_not_an_empty_string() { + let (dir, mut sink) = open_sink(); + sink.declare_object_type("Order", &[]).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.add_object( + "Order", + "o1", + &[("note".to_string(), t, OCELAttributeValue::Null)], + ) + .unwrap(); + // The same value through the other write path. + let o = ObjectRef::Id("o1".to_string()); + sink.add_object_attribute(&o, "memo", t, OCELAttributeValue::Null) + .unwrap(); + ExtractionSink::finalize(&mut sink).unwrap(); + + let con = duckdb::Connection::open(dir.path().join("sink.duckdb")).unwrap(); + let ocel = read_ocel_from_duckdb(&con).unwrap(); + let o1 = ocel.objects.iter().find(|o| o.id == "o1").unwrap(); + for name in ["note", "memo"] { + let a = o1 + .attributes + .iter() + .find(|a| a.name == name) + .unwrap_or_else(|| panic!("attribute {name} present")); + assert_eq!( + a.value, + OCELAttributeValue::Null, + "a Null object attribute must not read back as String(\"\")" + ); + } + } + + // The `O2O` half, that a target ask staged for a row whose source never resolves must not win + // `arg_min`, is covered by `case_11_an_o2o_target_ask_for_an_unresolvable_source_does_not_win_the_type` + // in this module's `tests` sibling, which compares against the eager sink rather than pinning + // this sink's own output. + + #[test] + fn finalize_creates_events_table_even_with_zero_events() { + let (dir, mut sink) = open_sink(); + let path = dir.path().join("sink.duckdb"); + ExtractionSink::finalize(&mut sink).unwrap(); + let con = duckdb::Connection::open(&path).unwrap(); + let n: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 0); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/expr.rs b/process_mining/src/core/event_data/object_centric/extraction/expr.rs new file mode 100644 index 00000000..068bb8fe --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/expr.rs @@ -0,0 +1,1067 @@ +//! Expressions producing a value from a row, plus splitting and timestamp parsing. + +use std::collections::HashSet; + +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::row::Row; +use super::value::Value; +use crate::core::event_data::object_centric::OCELAttributeType; + +/// Produces one text value from a row. +/// +/// Used by every position that becomes an identity: entity ids, relation endpoints, type names +/// and qualifiers. Every variant propagates absence, so if any input has no +/// [`Value::canonical_string`] the whole expression is `None` and the caller drops the row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum ValueExpression { + /// The value in a column. + Column { + /// Column name. + column: String, + }, + /// A fixed string. + Constant { + /// The value. + value: String, + }, + /// Text with `{column}` placeholders, for example `ORD-{order_id}-{region}`. + Template { + /// The template. + template: String, + }, + /// The first part that produces a value. + Coalesce { + /// Parts, tried in order. + parts: Vec, + }, +} + +impl ValueExpression { + /// Evaluate against one row. + pub(crate) fn evaluate(&self, row: &Row<'_>) -> Option { + match self { + ValueExpression::Constant { value } => Some(value.clone()), + ValueExpression::Column { column } => row.get(column).and_then(Value::canonical_string), + ValueExpression::Template { template } => render_template(template, row), + ValueExpression::Coalesce { parts } => parts.iter().find_map(|p| p.evaluate(row)), + } + } + + /// Collect every column name this expression reads into `out`. + pub fn referenced_columns<'a>(&'a self, out: &mut HashSet<&'a str>) { + match self { + ValueExpression::Column { column } => { + out.insert(column); + } + ValueExpression::Template { template } => { + for name in template_placeholders(template) { + out.insert(name); + } + } + ValueExpression::Coalesce { parts } => { + for p in parts { + p.referenced_columns(out); + } + } + ValueExpression::Constant { .. } => {} + } + } +} + +/// Substitute `{column}` placeholders, scanning the template rather than the result. +/// +/// Scanning the output instead misreads a substituted value that itself contains braces, and +/// accepts an unterminated placeholder. +fn render_template(template: &str, row: &Row<'_>) -> Option { + let mut out = String::with_capacity(template.len()); + let mut rest = template; + while let Some(open) = rest.find('{') { + out.push_str(&rest[..open]); + let after = &rest[open + 1..]; + let close = after.find('}')?; + let name = &after[..close]; + out.push_str(&row.get(name)?.canonical_string()?); + rest = &after[close + 1..]; + } + out.push_str(rest); + Some(out) +} + +/// The placeholder names in a template, in order. An unterminated placeholder ends the scan. An +/// empty placeholder (`{}`) contributes no name: it is a template defect, reported once by +/// `validate`'s `InvalidTemplate` check rather than again as `UnknownColumn { column: "" }`. +fn template_placeholders(template: &str) -> Vec<&str> { + let mut names = Vec::new(); + let mut rest = template; + while let Some(open) = rest.find('{') { + let after = &rest[open + 1..]; + let Some(close) = after.find('}') else { break }; + let name = &after[..close]; + if !name.is_empty() { + names.push(name); + } + rest = &after[close + 1..]; + } + names +} + +/// How to split one cell into several values. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SplitSpec { + /// The splitting rule. + pub kind: SplitKind, + /// Trim surrounding whitespace from each part. + pub trim: bool, +} + +/// The splitting rule of a [`SplitSpec`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum SplitKind { + /// Split on a literal separator. + Delimiter { + /// The separator. + delimiter: String, + }, + /// Extract values with a regular expression. + /// + /// With capture groups, every group of every match yields a value. Without them, each whole + /// match does. + Regex { + /// The pattern. + pattern: String, + }, +} + +impl SplitSpec { + /// Compile this split's regular expression once, ahead of repeated row evaluation. + /// + /// # Errors + /// Returns the underlying [`regex::Error`] if a `Regex` pattern does not compile. + pub(crate) fn prepare(&self) -> Result { + let kind = match &self.kind { + SplitKind::Delimiter { delimiter } => PreparedSplitKind::Delimiter(delimiter.clone()), + SplitKind::Regex { pattern } => PreparedSplitKind::Regex(regex::Regex::new(pattern)?), + }; + Ok(PreparedSplit { + kind, + trim: self.trim, + }) + } +} + +/// A [`SplitSpec`] with its regular expression compiled, ready for repeated evaluation. +#[derive(Debug)] +pub(crate) struct PreparedSplit { + kind: PreparedSplitKind, + trim: bool, +} + +#[derive(Debug)] +enum PreparedSplitKind { + Delimiter(String), + Regex(regex::Regex), +} + +impl PreparedSplit { + /// Split `raw` into values. Empty parts are dropped. + pub(crate) fn split(&self, raw: &str) -> Vec { + let keep = |s: &str| -> Option { + let v = if self.trim { s.trim() } else { s }; + (!v.is_empty()).then(|| v.to_string()) + }; + match &self.kind { + PreparedSplitKind::Delimiter(delimiter) => { + if delimiter.is_empty() { + return keep(raw).into_iter().collect(); + } + raw.split(delimiter.as_str()).filter_map(keep).collect() + } + PreparedSplitKind::Regex(re) => { + let mut out = Vec::new(); + for caps in re.captures_iter(raw) { + if caps.len() > 1 { + for i in 1..caps.len() { + if let Some(m) = caps.get(i) { + out.extend(keep(m.as_str())); + } + } + } else if let Some(m) = caps.get(0) { + out.extend(keep(m.as_str())); + } + } + out + } + } + } +} + +/// Maps a source column to a named OCEL attribute. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AttributeMapping { + /// Column to read. + pub source_column: String, + /// Attribute name in the resulting log. + pub name: String, + /// Declared attribute type, or `None` to take the catalog's type for `source_column`. + pub value_type: Option, +} + +/// How to interpret a timestamp value. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum TimestampFormat { + /// Try a cascade of common formats, most specific first. + Auto, + /// A `chrono` format string. + FormatString { + /// The format. + format: String, + }, + /// Seconds since the Unix epoch. + UnixSeconds, + /// Milliseconds since the Unix epoch. + UnixMillis, +} + +/// One value read as a timestamp: where the text comes from, and how to read it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TimestampPart { + /// Where the text comes from. + pub source: ValueExpression, + /// How to read it. `None` means auto-detection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, +} + +impl TimestampPart { + /// A part reading `column` with auto-detection. + pub fn column(column: impl Into) -> Self { + Self { + source: ValueExpression::Column { + column: column.into(), + }, + format: None, + } + } + + /// Parse this part as a whole timestamp. + pub(crate) fn parse(&self, row: &Row<'_>) -> Option> { + // A driver-decoded timestamp is already parsed; rendering it back to text to re-parse + // it would be both slower and lossier. + if let ValueExpression::Column { column } = &self.source { + if let Some(Value::Timestamp(ts)) = row.get(column) { + return Some(*ts); + } + } + let format = self.format.as_ref().unwrap_or(&TimestampFormat::Auto); + parse_timestamp(&self.resolve(row)?, format) + } + + /// The part's text, or `None` if the row carries nothing for it. Blank counts as nothing. + fn resolve(&self, row: &Row<'_>) -> Option { + let text = match &self.source { + // Not `evaluate`: its `canonical_string` is `None` for `Float`, which would drop + // every row of a Unix-epoch column a driver reports as a float. + ValueExpression::Column { column } => row.get(column).and_then(Value::display_string), + other => other.evaluate(row), + }?; + (!text.trim().is_empty()).then_some(text) + } +} + +/// Where an entity's timestamp comes from. +/// +/// `deny_unknown_fields`: a misspelled key would otherwise be ignored, leaving a timestamp that +/// silently drops every row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] +pub enum TimestampSource { + /// One value: a column, a constant, a template or a coalesce, with a format. + Value(TimestampPart), + /// Separate date and time parts, combined. + /// + /// Required by schemas that split them: `ERPNext`'s `posting_date` plus `posting_time`, + /// and SAP `CDHDR`'s `UDATE` plus `UTIME`. A date column paired with a constant + /// `"00:00:00"` states that the source has no time of day. + Components { + /// Where the date comes from, if anywhere. + #[serde(default)] + date: Option, + /// Where the time comes from, if anywhere. + #[serde(default)] + time: Option, + }, +} + +impl TimestampSource { + /// Read `column` with auto-detection. + #[must_use] + pub fn column(column: impl Into) -> Self { + Self::Value(TimestampPart::column(column)) + } + + /// The same fixed instant for every row. + #[must_use] + pub fn constant(value: impl Into) -> Self { + Self::Value(TimestampPart { + source: ValueExpression::Constant { + value: value.into(), + }, + format: None, + }) + } + + /// Resolve against one row. + pub(crate) fn parse(&self, row: &Row<'_>) -> Option> { + match self { + TimestampSource::Value(part) => part.parse(row), + TimestampSource::Components { date, time } => { + let d = date.as_ref().and_then(|p| p.resolve(row)); + let t = time.as_ref().and_then(|p| p.resolve(row)); + parse_timestamp_components( + d.as_deref(), + date.as_ref().and_then(|p| p.format.as_ref()), + t.as_deref(), + time.as_ref().and_then(|p| p.format.as_ref()), + ) + } + } + } + + /// Whether the row carried any timestamp text at all, to tell a wrong format apart from an + /// absent value when [`TimestampSource::parse`] returns `None`. Only called on that failure + /// path. + pub(crate) fn has_input(&self, row: &Row<'_>) -> bool { + match self { + TimestampSource::Value(part) => part.resolve(row).is_some(), + // A time of day with no date does not name an instant, so only the date side counts. + TimestampSource::Components { date, .. } => { + date.as_ref().is_some_and(|d| d.resolve(row).is_some()) + } + } + } + + /// Collect every column name this source reads into `out`. + pub fn referenced_columns<'a>(&'a self, out: &mut HashSet<&'a str>) { + match self { + TimestampSource::Value(part) => part.source.referenced_columns(out), + TimestampSource::Components { date, time } => { + for part in [date, time].into_iter().flatten() { + part.source.referenced_columns(out); + } + } + } + } +} + +/// Read a date, using `format` if given. The separator-less `%Y%m%d` that SAP's `UDATE` uses is +/// accepted only here, not in the general `Auto` cascade, so eight digits elsewhere stay a +/// number. +fn parse_date_part(value: &str, format: Option<&TimestampFormat>) -> Option { + const DATE_COMPONENT_FORMATS: &[&str] = &[ + "%Y-%m-%d", "%Y%m%d", "%d/%m/%Y", "%d.%m.%Y", "%m/%d/%Y", "%Y/%m/%d", + ]; + match format { + Some(TimestampFormat::FormatString { format }) => { + return NaiveDate::parse_from_str(value, format).ok(); + } + Some(f @ (TimestampFormat::UnixSeconds | TimestampFormat::UnixMillis)) => { + return parse_timestamp(value, f).map(|ts| ts.date_naive()); + } + Some(TimestampFormat::Auto) | None => {} + } + let head = value + .split_once('T') + .or_else(|| value.split_once(' ')) + .map_or(value, |(head, _)| head); + for candidate in [value, head] { + for f in DATE_COMPONENT_FORMATS { + if let Ok(d) = NaiveDate::parse_from_str(candidate, f) { + return Some(d); + } + } + } + parse_timestamp(value, &TimestampFormat::Auto).map(|ts| ts.date_naive()) +} + +/// Read a time of day, using `format` if given. [`parse_timestamp`]'s `Auto` has no bare-time +/// spelling, so the cascade here is the only thing that reads one. +fn parse_time_part(value: &str, format: Option<&TimestampFormat>) -> Option { + const TIME_COMPONENT_FORMATS: &[&str] = &["%H:%M:%S%.f", "%H:%M:%S", "%H:%M", "%H%M%S", "%H%M"]; + match format { + Some(TimestampFormat::FormatString { format }) => { + return NaiveTime::parse_from_str(value, format).ok(); + } + Some(f @ (TimestampFormat::UnixSeconds | TimestampFormat::UnixMillis)) => { + return parse_timestamp(value, f).map(|ts| ts.time()); + } + Some(TimestampFormat::Auto) | None => {} + } + let tail = value + .rsplit_once('T') + .or_else(|| value.rsplit_once(' ')) + .map_or(value, |(_, tail)| tail); + for candidate in [value, tail] { + for f in TIME_COMPONENT_FORMATS { + if let Ok(t) = NaiveTime::parse_from_str(candidate, f) { + return Some(t); + } + } + } + parse_timestamp(value, &TimestampFormat::Auto).map(|ts| ts.time()) +} + +/// Combine a date string and a time string into one instant, with either side's format pinned or +/// auto-detected. +fn parse_timestamp_components( + date_str: Option<&str>, + date_format: Option<&TimestampFormat>, + time_str: Option<&str>, + time_format: Option<&TimestampFormat>, +) -> Option> { + let auto = &TimestampFormat::Auto; + + match (date_str, time_str) { + (Some(d), Some(t)) => { + // Read each side as what it claims to be first, because it is the only strategy that + // cannot lose the time: the ones below fall back to parsing the date alone, turning + // an unread time into a silent midnight. + if let (Some(date), Some(time)) = ( + parse_date_part(d, date_format), + parse_time_part(t, time_format), + ) { + return Some(DateTime::from_naive_utc_and_offset( + date.and_time(time), + FixedOffset::east_opt(0)?, + )); + } + // Concatenating can only be read back with `Auto`, so it is off the table once either + // side pinned a format: `Auto` tries `%d/%m/%Y` first, reading `%m/%d/%Y`'s + // "01/02/2024" as February 1st rather than January 2nd. + if !is_pinned(date_format) && !is_pinned(time_format) { + if let Some(ts) = parse_timestamp(&format!("{d} {t}"), auto) { + return Some(ts); + } + // Each side may be a whole datetime, as in "2015-01-06T00:00:00" plus + // "1970-01-01T15:02:03". + let date_part = d + .split_once('T') + .or_else(|| d.split_once(' ')) + .map_or(d, |(p, _)| p); + let time_part = t + .rsplit_once('T') + .or_else(|| t.rsplit_once(' ')) + .map_or(t, |(_, p)| p); + if let Some(ts) = parse_timestamp(&format!("{date_part} {time_part}"), auto) { + return Some(ts); + } + } + // Last, each value as a standalone whole timestamp, still under its own side's format. + parse_timestamp(d, date_format.unwrap_or(auto)) + .or_else(|| parse_timestamp(t, time_format.unwrap_or(auto))) + } + // A date with no time means midnight. + (Some(d), None) => parse_timestamp(d, date_format.unwrap_or(auto)) + .or_else(|| parse_date_part(d, date_format).map(midnight_utc)), + // A time with no date names no instant unless the value is really a whole timestamp. + (None, Some(t)) => parse_timestamp(t, time_format.unwrap_or(auto)), + (None, None) => None, + } +} + +/// Whether the author fixed this side's format, as opposed to leaving it to be detected. +fn is_pinned(format: Option<&TimestampFormat>) -> bool { + !matches!(format, None | Some(TimestampFormat::Auto)) +} + +fn midnight_utc(date: NaiveDate) -> DateTime { + DateTime::from_naive_utc_and_offset( + date.and_time(NaiveTime::MIN), + FixedOffset::east_opt(0).expect("UTC is a valid offset"), + ) +} + +/// Parse a timestamp, trying every format `Auto` covers when no format is pinned. +/// +/// The cascade runs most specific first: RFC 3339 / ISO 8601, then RFC 2822, then naive +/// datetimes assumed UTC (most fractional digits to none), then date-only values (midnight +/// UTC), then `GMT`-style and UTC-suffix spellings, ending in `chrono`'s generic parser. +fn parse_timestamp(value: &str, format: &TimestampFormat) -> Option> { + let utc = FixedOffset::east_opt(0)?; + match format { + TimestampFormat::Auto => { + if let Ok(dt) = DateTime::parse_from_rfc3339(value) { + return Some(dt); + } + // ISO 8601 with non-colon offset (e.g., +0000) + if let Ok(dt) = DateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%z") { + return Some(dt); + } + if let Ok(dt) = DateTime::parse_from_rfc2822(value) { + return Some(dt); + } + + // Naive formats, assumed UTC, ordered by specificity. + const NAIVE_FORMATS: &[&str] = &[ + "%Y-%m-%d %H:%M:%S%.f", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y-%m-%dT%H:%M:%S%.f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M", + "%d/%m/%Y %H:%M:%S", + "%d/%m/%Y %H:%M", + "%d.%m.%Y %H:%M:%S", + "%d.%m.%Y %H:%M", + "%m/%d/%Y %H:%M:%S", + "%m/%d/%Y %H:%M", + // UTC suffix + "%Y-%m-%d %H:%M:%S UTC", + ]; + for fmt in NAIVE_FORMATS { + if let Ok(dt) = NaiveDateTime::parse_from_str(value, fmt) { + return Some(DateTime::from_naive_utc_and_offset(dt, utc)); + } + } + + // Date-only formats, set to midnight UTC. + const DATE_FORMATS: &[&str] = &["%Y-%m-%d", "%d/%m/%Y", "%d.%m.%Y", "%m/%d/%Y"]; + for fmt in DATE_FORMATS { + if let Ok(d) = chrono::NaiveDate::parse_from_str(value, fmt) { + return Some(DateTime::from_naive_utc_and_offset( + d.and_hms_opt(0, 0, 0)?, + utc, + )); + } + } + + // GMT format: "Mon Apr 03 2023 12:08:18 GMT+0200 (...)" + if let Ok((dt, _)) = DateTime::parse_and_remainder(value, "%Z %b %d %Y %T GMT%z") { + return Some(dt); + } + + // Last resort: chrono's generic DateTime parse. + value.parse::>().ok() + } + TimestampFormat::FormatString { format: fmt } => { + // Try as NaiveDateTime first (format includes time components) + if let Ok(dt) = NaiveDateTime::parse_from_str(value, fmt) { + return Some(DateTime::from_naive_utc_and_offset(dt, utc)); + } + // Fallback: date-only format strings (NaiveDateTime fails without hour) + if let Ok(d) = chrono::NaiveDate::parse_from_str(value, fmt) { + return Some(DateTime::from_naive_utc_and_offset( + d.and_hms_opt(0, 0, 0)?, + utc, + )); + } + None + } + TimestampFormat::UnixSeconds => value + .parse::() + .ok() + .and_then(|s| DateTime::from_timestamp(s, 0)) + .map(|dt| dt.with_timezone(&utc)), + TimestampFormat::UnixMillis => value + .parse::() + .ok() + .and_then(DateTime::from_timestamp_millis) + .map(|dt| dt.with_timezone(&utc)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::row::with_row; + use crate::core::event_data::object_centric::extraction::value::Value; + + #[test] + fn template_substitutes_and_propagates_missing_values() { + let e = ValueExpression::Template { + template: "ORD-{id}-{region}".into(), + }; + with_row( + &[ + ("id", Value::Integer(7)), + ("region", Value::Text("EU".into())), + ], + |row| { + assert_eq!(e.evaluate(row).as_deref(), Some("ORD-7-EU")); + }, + ); + with_row( + &[("id", Value::Integer(7)), ("region", Value::Null)], + |row| { + assert_eq!(e.evaluate(row), None); + }, + ); + } + + #[test] + fn a_substituted_value_containing_braces_is_not_treated_as_a_placeholder() { + // Regression: the old implementation inspected the output for braces, so a JSON + // value (ERPNext tabVersion.data) made the whole expression None and dropped the row. + let e = ValueExpression::Template { + template: "v-{payload}".into(), + }; + with_row(&[("payload", Value::Text("{\"a\":1}".into()))], |row| { + assert_eq!(e.evaluate(row).as_deref(), Some("v-{\"a\":1}")); + }); + } + + #[test] + fn an_unterminated_placeholder_yields_no_value() { + // Regression in the other direction: "a{b" used to pass the old brace check and + // become the literal identity "a{b". + let e = ValueExpression::Template { + template: "a{b".into(), + }; + with_row(&[("b", Value::Text("x".into()))], |row| { + assert_eq!(e.evaluate(row), None) + }); + } + + #[test] + fn coalesce_takes_the_first_value_that_renders() { + // Odoo mail_tracking_value: five typed columns, one populated. + let e = ValueExpression::Coalesce { + parts: vec![ + ValueExpression::Column { + column: "old_value_integer".into(), + }, + ValueExpression::Column { + column: "old_value_char".into(), + }, + ], + }; + with_row( + &[ + ("old_value_integer", Value::Null), + ("old_value_char", Value::Text("draft".into())), + ], + |row| assert_eq!(e.evaluate(row).as_deref(), Some("draft")), + ); + with_row( + &[ + ("old_value_integer", Value::Integer(3)), + ("old_value_char", Value::Null), + ], + |row| { + assert_eq!(e.evaluate(row).as_deref(), Some("3")); + }, + ); + with_row( + &[ + ("old_value_integer", Value::Null), + ("old_value_char", Value::Null), + ], + |row| { + assert_eq!(e.evaluate(row), None); + }, + ); + } + + #[test] + fn an_empty_column_name_is_not_swallowed() { + // A dropped guard here used to hide `{"type":"column","column":""}`, which drops every + // row at evaluation time, from validation. + let e = ValueExpression::Column { + column: String::new(), + }; + let mut cols = HashSet::new(); + e.referenced_columns(&mut cols); + assert_eq!(cols, HashSet::from([""])); + } + + #[test] + fn delimiter_split_trims_and_drops_empties() { + let s = SplitSpec { + kind: SplitKind::Delimiter { + delimiter: ",".into(), + }, + trim: true, + }; + assert_eq!(s.prepare().unwrap().split("a, b ,,c"), vec!["a", "b", "c"]); + } + + #[test] + fn prepare_reports_an_invalid_split_regex_instead_of_panicking() { + let s = SplitSpec { + kind: SplitKind::Regex { + pattern: "(".into(), + }, + trim: true, + }; + assert!(s.prepare().is_err()); + } + + #[test] + fn regex_split_uses_capture_groups_when_present() { + let s = SplitSpec { + kind: SplitKind::Regex { + pattern: "([a-z])=([0-9]+)".into(), + }, + trim: true, + }; + assert_eq!( + s.prepare().unwrap().split("x=1;y=22"), + vec!["x", "1", "y", "22"] + ); + let whole = SplitSpec { + kind: SplitKind::Regex { + pattern: "[a-z][0-9]+".into(), + }, + trim: true, + }; + assert_eq!(whole.prepare().unwrap().split("a1,b22"), vec!["a1", "b22"]); + } + + fn components(date: TimestampPart, time: TimestampPart) -> TimestampSource { + TimestampSource::Components { + date: Some(date), + time: Some(time), + } + } + + fn constant_part(value: &str) -> TimestampPart { + TimestampPart { + source: ValueExpression::Constant { + value: value.into(), + }, + format: None, + } + } + + fn formatted_column(column: &str, format: &str) -> TimestampPart { + TimestampPart { + source: ValueExpression::Column { + column: column.into(), + }, + format: Some(TimestampFormat::FormatString { + format: format.into(), + }), + } + } + + /// A constant time cannot invent an instant for a row whose date is `NULL`, and reporting + /// those rows as unparseable would point at a format string that is already right. + #[test] + fn a_null_date_is_missing_not_unparseable() { + let ts = components( + formatted_column("start_date", "%Y-%m-%d"), + constant_part("00:00:00"), + ); + with_row(&[("start_date", Value::Text("2018-02-06".into()))], |row| { + assert_eq!( + ts.parse(row).expect("a real date parses").to_rfc3339(), + "2018-02-06T00:00:00+00:00" + ); + assert!(ts.has_input(row)); + }); + with_row(&[("start_date", Value::Null)], |row| { + assert!(ts.parse(row).is_none()); + assert!( + !ts.has_input(row), + "a NULL date must report as missing, not as a format failure" + ); + }); + } + + #[test] + fn a_blank_part_is_absent_so_a_date_alone_is_midnight() { + let ts = components(TimestampPart::column("d"), constant_part(" ")); + with_row(&[("d", Value::Text("2024-03-04".into()))], |row| { + assert_eq!( + ts.parse(row).expect("date alone is midnight").to_rfc3339(), + "2024-03-04T00:00:00+00:00" + ); + }); + } + + #[test] + fn a_present_but_malformed_value_is_unparseable_not_missing() { + let ts = TimestampSource::column("ts"); + with_row(&[("ts", Value::Text("not a date".into()))], |row| { + assert!(ts.parse(row).is_none()); + assert!( + ts.has_input(row), + "text was there; the format is the problem" + ); + }); + } + + #[test] + fn components_timestamp_combines_separate_date_and_time_columns() { + // ERPNext posting_date + posting_time. + let ts = components(TimestampPart::column("d"), TimestampPart::column("t")); + with_row( + &[ + ("d", Value::Text("2015-01-06".into())), + ("t", Value::Text("15:02:03".into())), + ], + |row| { + let got = ts.parse(row).expect("should parse"); + assert_eq!(got.to_rfc3339(), "2015-01-06T15:02:03+00:00"); + }, + ); + } + + /// Compact spellings (SAP `CDHDR`'s `UDATE` + `UTIME`) that the concatenation strategies + /// cannot read must not degrade to a valid-looking midnight with the time dropped. + #[test] + fn separatorless_date_and_time_parts_are_read_not_dropped() { + let ts = components(TimestampPart::column("d"), TimestampPart::column("t")); + for (d, t) in [("2024-01-02", "150203"), ("20240102", "150203")] { + with_row( + &[("d", Value::Text(d.into())), ("t", Value::Text(t.into()))], + |row| { + assert_eq!( + ts.parse(row).expect("should parse").to_rfc3339(), + "2024-01-02T15:02:03+00:00" + ); + }, + ); + } + } + + #[test] + fn each_side_can_be_a_constant_independently_of_the_other() { + let constant_time = components(TimestampPart::column("d"), constant_part("00:00:00")); + with_row(&[("d", Value::Text("2024-03-04".into()))], |row| { + assert_eq!( + constant_time.parse(row).expect("should parse").to_rfc3339(), + "2024-03-04T00:00:00+00:00" + ); + }); + + let constant_date = components(constant_part("2024-03-04"), TimestampPart::column("t")); + with_row(&[("t", Value::Text("07:08:09".into()))], |row| { + assert_eq!( + constant_date.parse(row).expect("should parse").to_rfc3339(), + "2024-03-04T07:08:09+00:00" + ); + }); + } + + #[test] + fn a_per_side_format_pins_an_ambiguous_spelling() { + let american = components( + formatted_column("d", "%m/%d/%Y"), + TimestampPart::column("t"), + ); + with_row( + &[ + ("d", Value::Text("01/02/2024".into())), + ("t", Value::Text("00:00:00".into())), + ], + |row| { + // January 2nd; Auto reads this spelling as February 1st. + assert_eq!( + american.parse(row).expect("should parse").to_rfc3339(), + "2024-01-02T00:00:00+00:00" + ); + }, + ); + } + + /// A time cell the pinned time format cannot read must not send the date side back to + /// `Auto`, whose cascade reads "01/02/2024" as February 1st where `%m/%d/%Y` says January 2nd. + #[test] + fn an_unreadable_time_never_re_reads_a_pinned_date_with_auto() { + let ts = components( + formatted_column("d", "%m/%d/%Y"), + formatted_column("t", "%H:%M:%S"), + ); + with_row( + &[ + ("d", Value::Text("01/02/2024".into())), + ("t", Value::Text("not a time".into())), + ], + |row| { + assert_eq!( + ts.parse(row).expect("the date still parses").to_rfc3339(), + "2024-01-02T00:00:00+00:00" + ); + }, + ); + } + + #[test] + fn the_json_shape_is_a_source_and_a_format_per_side() { + let parsed: TimestampSource = serde_json::from_str( + r#"{"type":"components", + "date":{"source":{"type":"column","column":"UDATE"}}, + "time":{"source":{"type":"column","column":"UTIME"}}}"#, + ) + .expect("current shape parses"); + assert_eq!( + parsed, + components( + TimestampPart::column("UDATE"), + TimestampPart::column("UTIME") + ) + ); + + let ts = components(formatted_column("d", "%Y%m%d"), constant_part("00:00:00")); + let json = serde_json::to_string(&ts).expect("serialises"); + assert_eq!( + serde_json::from_str::(&json).expect("round trips"), + ts + ); + } + + #[test] + fn the_retired_spellings_no_longer_deserialise() { + // Silently ignoring `date_column` would leave both sides unset and drop every row. + let legacy = serde_json::from_str::( + r#"{"type":"components","date_column":"UDATE","time_column":"UTIME"}"#, + ); + assert!(legacy.is_err(), "the old key names must not be accepted"); + + let old_column = + serde_json::from_str::(r#"{"type":"column","column":"at"}"#); + assert!(old_column.is_err(), "`column` folded into `value`"); + } + + #[test] + fn a_timestamp_can_be_a_template_or_a_coalesce() { + let templated = TimestampSource::Value(TimestampPart { + source: ValueExpression::Template { + template: "{d}T{t}Z".into(), + }, + format: None, + }); + with_row( + &[ + ("d", Value::Text("2024-01-02".into())), + ("t", Value::Text("15:02:03".into())), + ], + |row| { + assert_eq!( + templated.parse(row).expect("template parses").to_rfc3339(), + "2024-01-02T15:02:03+00:00" + ); + }, + ); + + let coalesced = TimestampSource::Value(TimestampPart { + source: ValueExpression::Coalesce { + parts: vec![ + ValueExpression::Column { + column: "start_date".into(), + }, + ValueExpression::Column { + column: "retrieved_at".into(), + }, + ], + }, + format: None, + }); + with_row( + &[ + ("start_date", Value::Null), + ("retrieved_at", Value::Text("2020-05-06".into())), + ], + |row| { + assert_eq!( + coalesced.parse(row).expect("falls back").to_rfc3339(), + "2020-05-06T00:00:00+00:00" + ); + }, + ); + } + + /// `referenced_columns` feeds the column list each scan requests, so an expression-valued + /// part has to report through to its own columns or the scan will not fetch them. + #[test] + fn referenced_columns_sees_through_both_parts() { + let ts = components( + TimestampPart { + source: ValueExpression::Coalesce { + parts: vec![ + ValueExpression::Column { + column: "posting_date".into(), + }, + ValueExpression::Column { + column: "creation_date".into(), + }, + ], + }, + format: None, + }, + constant_part("00:00:00"), + ); + let mut cols = HashSet::new(); + ts.referenced_columns(&mut cols); + let mut got: Vec<&str> = cols.into_iter().collect(); + got.sort_unstable(); + assert_eq!(got, vec!["creation_date", "posting_date"]); + } + + #[test] + fn a_unix_epoch_column_reported_as_float_still_parses() { + // The non-Timestamp fallback must not read through `canonical_string`, which is `None` + // for `Float` and would silently drop every row. + let ts = TimestampSource::Value(TimestampPart { + source: ValueExpression::Column { column: "t".into() }, + format: Some(TimestampFormat::UnixSeconds), + }); + with_row(&[("t", Value::Float(1_580_698_806.0))], |row| { + assert!(ts.parse(row).is_some()); + }); + } + + #[test] + fn a_typed_timestamp_column_is_used_without_a_string_roundtrip() { + let parsed = chrono::DateTime::parse_from_rfc3339("2020-02-03T04:05:06+02:00").unwrap(); + let ts = TimestampSource::column("t"); + with_row(&[("t", Value::Timestamp(parsed))], |row| { + assert_eq!(ts.parse(row), Some(parsed)); + }); + } + /// Not a correctness test: a measurement of what `Auto` costs per row against a pinned format, + /// for the common `SQLite` spelling. Ignored by default. Run with + /// `cargo test -p process_mining --features extraction-blueprint auto_timestamp_cost -- --ignored --nocapture`. + #[test] + #[ignore] + fn auto_timestamp_cost() { + const N: usize = 1_000_000; + let sqlite_style = "2023-04-03 12:08:18"; + let rfc3339 = "2023-04-03T12:08:18+00:00"; + let pinned = TimestampFormat::FormatString { + format: "%Y-%m-%d %H:%M:%S".to_string(), + }; + + for (label, value, format) in [ + ( + "auto / sqlite 'Y-m-d H:M:S'", + sqlite_style, + &TimestampFormat::Auto, + ), + ("auto / rfc3339", rfc3339, &TimestampFormat::Auto), + ("pinned / sqlite", sqlite_style, &pinned), + ] { + let start = std::time::Instant::now(); + let mut ok = 0usize; + for _ in 0..N { + if parse_timestamp(std::hint::black_box(value), format).is_some() { + ok += 1; + } + } + let elapsed = start.elapsed(); + assert_eq!(ok, N, "{label} failed to parse"); + println!( + "{label:32} {:>8.0} ns/row {:>7.2} s per 10M rows", + elapsed.as_nanos() as f64 / N as f64, + elapsed.as_secs_f64() * 10.0, + ); + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/extract.rs b/process_mining/src/core/event_data/object_centric/extraction/extract.rs new file mode 100644 index 00000000..cabf1136 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/extract.rs @@ -0,0 +1,364 @@ +//! Executing a [`Blueprint`] against real data: the reference semantics everything else (a SQL +//! compiler, in particular) is checked against. + +use std::collections::{HashMap, HashSet}; + +use super::blueprint::{Blueprint, MissingEndpointPolicy, Target}; +use super::catalog::{Catalog, TableSchema}; +use super::desugar::desugar_with_paths; +use super::expr::ValueExpression; +use super::graph::GraphExecutor; +use super::mapping_exec::{self, MappingPasses, Phase, RunCtx}; +use super::provider::RowProvider; +use super::report::{ + DropReason, ErrorLog, ExtractionError, ExtractionReport, MappingRef, MappingStats, + PushdownDeclined, +}; +use super::row::{build_column_index, Row}; +use super::sink::ExtractionSink; +use super::validate::validate; + +/// Execute `blueprint` against `providers`, sending every event, object and relation to `sink`. +/// +/// Refuses to run a blueprint that does not [`validate`] against `catalog`. `providers` is keyed +/// by `source_id`, as a [`NodeOp::Source`](super::blueprint::NodeOp::Source) names it. +/// +/// # Three passes, not one +/// +/// Every `Object` target runs first, across the whole blueprint, then every `Event` target, then +/// everything that relates two of them. A relation therefore resolves against every entity the +/// blueprint produces, so relation resolution does not depend on mapping order, and matches what +/// a compiled SQL view produces: a join against all objects. See `Phase`. +/// +/// Objects and events are separate passes because an inline object reference on a `Target::Event` +/// with no `id` must be emitted alongside the event: a run-minted `UUID` cannot be re-derived +/// later. The cost is that a node read in more than one pass is scanned once per pass. +/// +/// # What execution order still decides +/// +/// Resolution is order-independent, the rest is not. Order decides which of two `Target::Object` +/// mappings naming one id creates it and which appends, under +/// [`DuplicateObjectPolicy::FirstWins`](super::blueprint::DuplicateObjectPolicy::FirstWins), which +/// of two same-name, same-time attribute writes the sink keeps, and under +/// [`MissingEndpointPolicy::Create`], which of two relation mappings declaring different types for +/// one missing id wins. Nothing here issues an `ORDER BY`, so that order is scan order. +/// +/// "Whichever runs first" is not mapping order. Within a phase, mappings are grouped by the node +/// they read so each node is scanned once, and the groups run in first-seen node order across the +/// whole desugared list, making the real order `(phase, node position, mapping index)`. Adding an +/// unrelated mapping on a different node can move that node earlier and flip which of two +/// colliding `Target::Object` mappings wins. Two mappings on the same node still run in mapping +/// order. A blueprint that wants a stable answer should not collide at all (distinct ids, or +/// [`IdRendering::TypePrefixed`](super::blueprint::IdRendering::TypePrefixed)) or put the colliding +/// mappings on one node. +/// +/// # Errors +/// Returns [`ExtractionError`] when the blueprint does not validate, a `Source` node names a +/// source absent from `providers`, a [`RowProvider`] call fails, or any [`ExtractionSink`] call +/// fails. A sink failure aborts the run however small the write was, since carrying on would leave +/// a half-written OCEL reporting success. A policy configured to error is collected into the +/// returned [`ExtractionReport::errors`] instead, so one bad row does not abort the whole run. +pub fn extract( + blueprint: &Blueprint, + catalog: &dyn Catalog, + providers: &HashMap, + sink: &mut dyn ExtractionSink, +) -> Result { + let validation_errors = validate(blueprint, catalog); + if !validation_errors.is_empty() { + return Err(ExtractionError::Invalid(validation_errors)); + } + + let desugared = desugar_with_paths(blueprint); + let mapping_refs: Vec = desugared + .iter() + .enumerate() + .map(|(index, (path, m))| MappingRef::new(index, path.clone(), m)) + .collect(); + + let exec = GraphExecutor::new(blueprint, catalog, providers, &desugared)?; + + // Only a sink that defers endpoint resolution needs this; it is the one that applies the + // policy, at `finalize`, where the mapping that named the endpoint is long gone. + sink.set_missing_endpoint_policy(blueprint.on_missing_endpoint) + .map_err(|e| ExtractionError::Sink { + context: "announcing the missing-endpoint policy".to_string(), + source: e, + })?; + + let mut errors = ErrorLog::new(); + let mut stats: Vec = mapping_refs + .iter() + .cloned() + .map(MappingStats::new) + .collect(); + let mut declared_events: Vec> = vec![HashSet::new(); desugared.len()]; + let mut declared_objects: Vec> = vec![HashSet::new(); desugared.len()]; + let mut attr_types = mapping_exec::DeclaredAttrTypes::new(); + // Outlive the per-row `RunCtx`, so every row writes into the same allocation. + let mut event_attrs = Vec::new(); + let mut object_attrs = Vec::new(); + + // Statically-named types are declared up front, so the declared type set is a function of the + // blueprint alone, not of which rows happen to match. + for (i, (_, m)) in desugared.iter().enumerate() { + let node_schema = exec.schema_of(&m.node); + declare_static_types( + sink, + node_schema, + &m.target, + &mut declared_events[i], + &mut declared_objects[i], + &mut attr_types, + &mut errors, + )?; + } + + let mut prepared_when = Vec::with_capacity(desugared.len()); + let mut prepared_splits = Vec::with_capacity(desugared.len()); + for (path, m) in &desugared { + let node_schema = exec.schema_of(&m.node); + let when = match &m.when { + Some(p) => Some( + p.prepare(node_schema) + .map_err(|e| ExtractionError::InvalidRegex { + pattern: format!("mapping '{path}' when"), + message: e.to_string(), + })?, + ), + None => None, + }; + prepared_when.push(when); + let splits = + mapping_exec::prepare_splits(&m.target).map_err(|e| ExtractionError::InvalidRegex { + pattern: format!("mapping '{path}' split"), + message: e.to_string(), + })?; + prepared_splits.push(splits); + } + + // Group mapping indices by the node they read, preserving first-seen node order, so all + // mappings sharing one node share one scan. + let mut node_order: Vec = Vec::new(); + let mut groups: HashMap> = HashMap::new(); + for (i, (_, m)) in desugared.iter().enumerate() { + if !groups.contains_key(&m.node) { + node_order.push(m.node.clone()); + } + groups.entry(m.node.clone()).or_default().push(i); + } + + let passes: Vec = desugared + .iter() + .map(|(_, m)| MappingPasses::of(&m.target)) + .collect(); + + for phase in [Phase::Objects, Phase::Events, Phase::Relations] { + for node_id in &node_order { + let indices: Vec = groups[node_id] + .iter() + .copied() + .filter(|&i| passes[i].runs_in(phase)) + .collect(); + if indices.is_empty() { + continue; + } + let node_schema = exec.schema_of(node_id); + let names: Vec = node_schema + .map(|s| s.columns.keys().cloned().collect()) + .unwrap_or_default(); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let index = build_column_index(&refs); + + exec.stream(node_id, &mut |vals| { + let row = Row { + values: vals, + index: &index, + }; + for &mi in &indices { + // A mapping read in more than one pass must not report its rows twice. + let counts_here = passes[mi].counting_phase() == phase; + if counts_here { + stats[mi].rows_read += 1; + } + let (_, m) = &desugared[mi]; + let guard_ok = match &prepared_when[mi] { + Some(p) => p.evaluate(&row), + None => true, + }; + if !guard_ok { + if counts_here { + stats[mi].drop(DropReason::PredicateExcluded); + } + continue; + } + let mut ctx = RunCtx { + blueprint, + sink, + declared_events: &mut declared_events[mi], + declared_objects: &mut declared_objects[mi], + errors: &mut errors, + attr_types: &mut attr_types, + event_attrs: &mut event_attrs, + object_attrs: &mut object_attrs, + }; + mapping_exec::run_target( + &mut ctx, + &mapping_refs[mi], + exec.schema_of(&m.node), + &m.target, + &prepared_splits[mi], + &row, + &mut stats[mi], + phase, + )?; + } + Ok(()) + })?; + } + } + + // Fatal on failure: a sink that could not finish resolving what it deferred has produced an + // incomplete log, and reporting that as a success would be worse than failing. + let finalize = sink.finalize().map_err(|e| ExtractionError::Sink { + context: "finalizing the sink".to_string(), + source: e, + })?; + + // A deferring sink applies `on_missing_endpoint` itself, at finalize, and can only count what + // it dropped, so the `Error` policy's errors have to be raised here, from that count. + // Without this the policy silently degraded to `Drop` for every such sink: the per-endpoint + // `MissingEndpoint` errors are pushed where an endpoint is resolved, and a deferring sink + // resolves none. + if blueprint.on_missing_endpoint == MissingEndpointPolicy::Error + && finalize.unresolved_endpoints > 0 + { + errors.push(ExtractionError::MissingEndpointsAtFinalize { + count: finalize.unresolved_endpoints, + }); + } + + let (errors, errors_suppressed) = errors.into_parts(); + Ok(ExtractionReport { + per_mapping: stats, + errors, + errors_suppressed, + rows_materialized: exec.rows_materialized(), + pushdown_declined: exec + .take_pushdown_rejections() + .into_iter() + .map(|(node, reason)| PushdownDeclined { node, reason }) + .collect(), + finalize, + timing: None, + }) +} + +fn constant_name(e: &ValueExpression) -> Option { + match e { + ValueExpression::Constant { value } => Some(value.clone()), + _ => None, + } +} + +/// Declare every statically-named (`Constant`) event/object type one mapping's target names: its +/// own type if it has one, and every relation endpoint's type. This keeps `Create`-policy +/// synthesis from racing an undeclared type, and lets a zero-match mapping still declare its +/// type. +#[allow(clippy::too_many_arguments)] +fn declare_static_types( + sink: &mut dyn ExtractionSink, + node_schema: Option<&TableSchema>, + target: &Target, + declared_events: &mut HashSet, + declared_objects: &mut HashSet, + attr_types: &mut mapping_exec::DeclaredAttrTypes, + errors: &mut ErrorLog, +) -> Result<(), ExtractionError> { + match target { + Target::Event { + event_type, + attributes, + objects, + .. + } => { + if let Some(name) = constant_name(event_type) { + let attrs = mapping_exec::reconcile_attr_types( + "event", + &name, + &mapping_exec::build_type_attrs(attributes, node_schema), + attr_types, + errors, + ); + sink.declare_event_type(&name, &attrs) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring event type '{name}'"), + source: e, + })?; + declared_events.insert(name); + } + for o in objects { + if let Some(name) = o.object.object_type.as_ref().and_then(constant_name) { + sink.declare_object_type(&name, &[]) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{name}'"), + source: e, + })?; + declared_objects.insert(name); + } + } + } + Target::Object { + object_type, + attributes, + .. + } => { + if let Some(name) = constant_name(object_type) { + let attrs = mapping_exec::reconcile_attr_types( + "object", + &name, + &mapping_exec::build_type_attrs(attributes, node_schema), + attr_types, + errors, + ); + sink.declare_object_type(&name, &attrs) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{name}'"), + source: e, + })?; + declared_objects.insert(name); + } + } + Target::E2O { event, object, .. } => { + if let Some(name) = event.event_type.as_ref().and_then(constant_name) { + sink.declare_event_type(&name, &[]) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring event type '{name}'"), + source: e, + })?; + declared_events.insert(name); + } + if let Some(name) = object.object_type.as_ref().and_then(constant_name) { + sink.declare_object_type(&name, &[]) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{name}'"), + source: e, + })?; + declared_objects.insert(name); + } + } + Target::O2O { source, target, .. } => { + for endpoint in [source, target] { + if let Some(name) = endpoint.object_type.as_ref().and_then(constant_name) { + sink.declare_object_type(&name, &[]) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{name}'"), + source: e, + })?; + declared_objects.insert(name); + } + } + } + } + Ok(()) +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/graph.rs b/process_mining/src/core/event_data/object_centric/extraction/graph.rs new file mode 100644 index 00000000..49f4558a --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/graph.rs @@ -0,0 +1,696 @@ +//! Executes the flat node graph, holding as little of it in memory as each node allows. +//! +//! `Source`, `Filter` and `Union` nodes hold a single row at a time. A `Join` has to hold one of +//! its inputs in full while the other is scanned past it; it holds the right input and streams +//! the left, so put the smaller table on the right. If both join inputs read the same source, +//! [`super::pushdown`] lets that source perform the join and neither side is held. +//! +//! Nothing is cached between nodes: a node read by two consumers is executed twice. See +//! [`GraphExecutor::stream`]. +//! +//! A join resolves both key column sets and all output columns before reading a row; anything that +//! does not resolve is an error rather than a skipped key or a null fill. +//! +//! A [`Union`](super::blueprint::NodeOp::Union) concatenates its inputs' rows keeping duplicates +//! (`UNION ALL`). Its output columns are the union of its inputs' column names, and an input +//! lacking one contributes `Null` for it. + +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::fmt::Write; +use std::ops::ControlFlow; + +use super::blueprint::{Blueprint, NodeOp}; +use super::catalog::{Catalog, TableSchema}; +use super::compile::RejectReason; +use super::predicate::PreparedPredicate; +use super::provider::{ProviderError, RowProvider}; +use super::pushdown; +use super::report::ExtractionError; +use super::row::{build_column_index, Row}; +use super::schema::{ + demanded_columns, full_node_schemas, join_column_source, projected_schema, JoinSide, +}; +use super::validate::ValidationError; +use super::value::Value; +use super::Mapping; + +/// A hash join's build side: its right input, held in full while the left streams past it. +/// +/// Two things are cut before a row lands here: its columns are narrowed to what the join's output +/// reads from the right (key columns are already in the key), and a row whose key is `Null` is +/// dropped, since no left row can match it. +#[derive(Debug, Default)] +struct BuildSide { + /// Right rows grouped by join key (see [`JoinKeyBuf::render`]). A key with several rows is a + /// many-to-many join, and each is paired with every matching left row. + by_key: HashMap>>, + /// How many rows this table holds, for [`GraphExecutor::rows_materialized`]. + rows: u64, +} + +/// Where one output column of a `Join` comes from. +enum ColSource { + /// A position in the left input's row, which is read directly as it streams past. + Left(usize), + /// A position in a [`BuildSide`] row, not in the right input's row, which is wider. + Right(usize), +} + +/// Runs a blueprint's node graph against a set of providers. +/// +/// Owns each node's execution-time schema (see [`super::schema`]) and every `Filter`'s prepared +/// predicate, resolved once up front so row-by-row evaluation re-parses nothing. No node's rows +/// are cached, so this struct's size is a function of the blueprint, not of the data. +pub(crate) struct GraphExecutor<'a> { + blueprint: &'a Blueprint, + providers: &'a HashMap, + schemas: HashMap<&'a str, TableSchema>, + /// Every node's full column shape, before projection. A `Join` needs both its inputs' full + /// schemas to route an output column back to the side it came from, not their projected ones. + /// See [`join_column_source`]. + full: HashMap<&'a str, TableSchema>, + prepared_filters: HashMap<&'a str, PreparedPredicate>, + /// See [`Self::rows_materialized`]. + rows_materialized: Cell, + /// See [`Self::take_pushdown_rejections`]. + pushdown_rejections: RefCell>, +} + +impl<'a> GraphExecutor<'a> { + /// Resolve every node's execution schema and prepare every `Filter`'s predicate. + pub(crate) fn new( + blueprint: &'a Blueprint, + catalog: &dyn Catalog, + providers: &'a HashMap, + mappings: &[(String, Mapping)], + ) -> Result { + let full = full_node_schemas(blueprint, catalog); + let demand = demanded_columns(blueprint, &full, mappings); + let mut schemas: HashMap<&str, TableSchema> = HashMap::new(); + for node in &blueprint.nodes { + let full_schema = full + .get(node.id.as_str()) + .cloned() + .unwrap_or_else(|| TableSchema { + name: node.id.clone(), + columns: std::collections::BTreeMap::new(), + }); + let want = demand.get(node.id.as_str()).cloned().unwrap_or_default(); + schemas.insert( + node.id.as_str(), + projected_schema(&full_schema, &want, &node.id), + ); + } + + let mut prepared_filters = HashMap::new(); + for node in &blueprint.nodes { + if let NodeOp::Filter { input, condition } = &node.op { + let input_schema = schemas.get(input.as_str()); + let prepared = + condition + .prepare(input_schema) + .map_err(|e| ExtractionError::InvalidRegex { + pattern: format!("filter '{}'", node.id), + message: e.to_string(), + })?; + prepared_filters.insert(node.id.as_str(), prepared); + } + } + + Ok(Self { + blueprint, + providers, + schemas, + full, + prepared_filters, + rows_materialized: Cell::new(0), + pushdown_rejections: RefCell::new(Vec::new()), + }) + } + + /// Total rows this executor has put in memory, summed over every hash join's [`BuildSide`]. + /// + /// Zero for a graph of `Source`, `Filter` and `Union` nodes, and for a `Join` pushed down to + /// its source. A running total, not a peak. See + /// [`ExtractionReport::rows_materialized`](super::report::ExtractionReport::rows_materialized). + pub(crate) fn rows_materialized(&self) -> u64 { + self.rows_materialized.get() + } + + /// Every node the emitter refused to push down, paired with the reason it gave, leaving the + /// executor's own list empty. + pub(crate) fn take_pushdown_rejections(&self) -> Vec<(String, RejectReason)> { + std::mem::take(&mut self.pushdown_rejections.borrow_mut()) + } + + /// The execution-time (projected) schema of `node_id`'s output rows. + pub(crate) fn schema_of(&self, node_id: &str) -> Option<&TableSchema> { + self.schemas.get(node_id) + } + + fn column_names(&self, node_id: &str) -> Vec { + self.schemas + .get(node_id) + .map(|s| s.columns.keys().cloned().collect()) + .unwrap_or_default() + } + + /// Call `on_row` once per row of `node_id`'s output. + /// + /// Every node but a locally executed `Join` passes its rows straight through, one at a time, + /// with no intermediate buffer. A `Join` holds its right input and streams its left past it + /// (see [`Self::hash_join`]), unless [`Self::push_down`] can hand the whole thing to the + /// source. + /// + /// Nothing here is cached, so a `Union` feeding two mappings scans everything beneath it once + /// per consumer. Caching would make memory a function of the data, where rescanning makes time + /// a function of the consumer count, which is a property of the blueprint. + pub(crate) fn stream( + &self, + node_id: &str, + on_row: &mut dyn FnMut(&[Value]) -> Result<(), ExtractionError>, + ) -> Result<(), ExtractionError> { + let node = self + .blueprint + .node(node_id) + .expect("node id resolved from a validated blueprint"); + match &node.op { + NodeOp::Source { source_id, table } => { + let provider = self.providers.get(source_id).copied().ok_or_else(|| { + ExtractionError::MissingProvider { + source_id: source_id.clone(), + } + })?; + let names = self.column_names(node_id); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let mut first_err: Option = None; + provider + .scan(table, &refs, &mut |vals| match on_row(vals) { + Ok(()) => ControlFlow::Continue(()), + Err(e) => { + // Abandon the scan: the error is fatal, so reading the rest of the + // table would be pure waste. + first_err = Some(e); + ControlFlow::Break(()) + } + }) + .map_err(|e| ExtractionError::Provider { + node: node_id.to_string(), + source: e, + })?; + match first_err { + Some(e) => Err(e), + None => Ok(()), + } + } + NodeOp::Filter { input, .. } => { + // Safe only because a `Filter`'s demand equals its input's, so the row forwarded + // verbatim below is as wide as the schema its consumers index it with. See + // `schema::demanded_columns`. + let names = self.column_names(input); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let index = build_column_index(&refs); + let prepared = self + .prepared_filters + .get(node_id) + .expect("every Filter node has a prepared predicate"); + self.stream(input, &mut |vals| { + let row = Row { + values: vals, + index: &index, + }; + if prepared.evaluate(&row) { + on_row(vals) + } else { + Ok(()) + } + }) + } + NodeOp::Union { inputs } => { + let out_cols = self.column_names(node_id); + for input in inputs { + let in_cols = self.column_names(input); + // Resolved once per input, not per row. `None` is a column this input does + // not have, and gets the null fill. + let positions: Vec> = out_cols + .iter() + .map(|c| in_cols.iter().position(|ic| ic == c)) + .collect(); + let mut buf = vec![Value::Null; out_cols.len()]; + self.stream(input, &mut |vals| { + for (slot, position) in buf.iter_mut().zip(&positions) { + *slot = match position { + Some(i) => vals[*i].clone(), + None => Value::Null, + }; + } + on_row(&buf) + })?; + } + Ok(()) + } + NodeOp::Join { left, right, on } => { + if self.push_down(node_id, on_row)? { + return Ok(()); + } + self.hash_join(node_id, left, right, on, on_row) + } + } + } + + /// Ask `node_id`'s source to execute the whole node, join and filters included, and stream + /// the result back, holding nothing here. + /// + /// Returns `false` without emitting a row when that is not possible, and then the caller + /// executes the node itself. Every step can decline: the leaves may not share one source + /// ([`pushdown::single_source`]), the provider may not run SQL + /// ([`RowProvider::query_dialect`]), the emitter may refuse the node + /// ([`pushdown::node_query_or_reason`], whose reason is kept for + /// [`Self::take_pushdown_rejections`]), or the provider may reject the query. + /// + /// Only the last can happen after rows have already been emitted, so falling back is allowed + /// only while nothing has been emitted yet. + fn push_down( + &self, + node_id: &str, + on_row: &mut dyn FnMut(&[Value]) -> Result<(), ExtractionError>, + ) -> Result { + let Some(source_id) = pushdown::single_source(self.blueprint, node_id) else { + return Ok(false); + }; + let Some(provider) = self.providers.get(source_id).copied() else { + return Ok(false); + }; + let Some(dialect) = provider.query_dialect() else { + return Ok(false); + }; + let names = self.column_names(node_id); + let sql = match pushdown::node_query_or_reason( + self.blueprint, + &self.full, + node_id, + &names, + dialect, + ) { + Ok(sql) => sql, + Err(reason) => { + // A node read by two consumers gets here twice, with the same reason. + let mut declined = self.pushdown_rejections.borrow_mut(); + if !declined.iter().any(|(n, _)| n == node_id) { + declined.push((node_id.to_string(), reason)); + } + return Ok(false); + } + }; + + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let mut first_err: Option = None; + let mut emitted = false; + let result = provider.scan_query(&sql, &refs, &mut |vals| { + emitted = true; + match on_row(vals) { + Ok(()) => ControlFlow::Continue(()), + Err(e) => { + first_err = Some(e); + ControlFlow::Break(()) + } + } + }); + match result { + Ok(()) => match first_err { + Some(e) => Err(e), + None => Ok(true), + }, + Err(ProviderError::QueryUnsupported) if !emitted => Ok(false), + Err(e) => Err(ExtractionError::Provider { + node: node_id.to_string(), + source: e, + }), + } + } + + /// A hash join that holds one side, not two: the right input becomes a [`BuildSide`] and + /// the left is streamed past it. + /// + /// Which side is held, and why every column is resolved before a row is read, are this + /// module's docs. + fn hash_join( + &self, + node_id: &str, + left: &str, + right: &str, + on: &[(String, String)], + on_row: &mut dyn FnMut(&[Value]) -> Result<(), ExtractionError>, + ) -> Result<(), ExtractionError> { + let out_cols = self.column_names(node_id); + let l_cols = self.column_names(left); + let r_cols = self.column_names(right); + + let key_position = |columns: &[String], column: &str, side: &'static str| { + columns.iter().position(|c| c == column).ok_or_else(|| { + ExtractionError::JoinKeyColumnMissing { + node: node_id.to_string(), + side, + column: column.to_string(), + } + }) + }; + let left_pos: Vec = on + .iter() + .map(|(l, _)| key_position(&l_cols, l, "left")) + .collect::>()?; + let right_pos: Vec = on + .iter() + .map(|(_, r)| key_position(&r_cols, r, "right")) + .collect::>()?; + + let empty = TableSchema { + name: String::new(), + columns: std::collections::BTreeMap::new(), + }; + let l_full = self.full.get(left).unwrap_or(&empty); + let r_full = self.full.get(right).unwrap_or(&empty); + // `kept` lists positions in a right input row, and a `ColSource::Right` indexes into that + // narrowed tuple, so a right column the output never mentions is never stored. + let mut kept: Vec = Vec::new(); + let sources: Vec = out_cols + .iter() + .map(|name| { + let unresolved = || { + ExtractionError::Invalid(vec![ValidationError::UnknownColumn { + node: node_id.to_string(), + column: name.clone(), + }]) + }; + match join_column_source(name, l_full, r_full) { + Some((JoinSide::Left, source_column)) => l_cols + .iter() + .position(|c| c == source_column) + .map(ColSource::Left) + .ok_or_else(unresolved), + Some((JoinSide::Right, source_column)) => { + let p = r_cols + .iter() + .position(|c| c == source_column) + .ok_or_else(unresolved)?; + let slot = kept.iter().position(|k| *k == p).unwrap_or_else(|| { + kept.push(p); + kept.len() - 1 + }); + Ok(ColSource::Right(slot)) + } + None => Err(unresolved()), + } + }) + .collect::>()?; + + let mut keys = JoinKeyBuf::default(); + let mut build = BuildSide::default(); + self.stream(right, &mut |vals| { + // A `Null` key matches nothing, so the row is dropped here rather than stored and + // skipped later. + if let Some(key) = keys.render(vals, &right_pos) { + let row: Vec = kept.iter().map(|&i| vals[i].clone()).collect(); + match build.by_key.get_mut(key) { + Some(rows) => rows.push(row), + None => { + build.by_key.insert(key.to_string(), vec![row]); + } + } + build.rows += 1; + } + Ok(()) + })?; + self.rows_materialized + .set(self.rows_materialized.get() + build.rows); + + let mut out = vec![Value::Null; out_cols.len()]; + self.stream(left, &mut |lrow| { + let Some(key) = keys.render(lrow, &left_pos) else { + return Ok(()); + }; + let Some(matches) = build.by_key.get(key) else { + return Ok(()); + }; + for rrow in matches { + for (slot, source) in out.iter_mut().zip(&sources) { + *slot = match source { + ColSource::Left(p) => lrow[*p].clone(), + ColSource::Right(p) => rrow[*p].clone(), + }; + } + on_row(&out)?; + } + Ok(()) + }) + } +} + +/// Scratch buffers for rendering join keys, reused across every row of both a join's inputs. +/// +/// One key is one string, not a `Vec` of them: the streamed side discards its key right after the +/// probe, so a per-column allocation there would be pure waste. +#[derive(Debug, Default)] +struct JoinKeyBuf { + key: String, + part: String, +} + +impl JoinKeyBuf { + /// One row's join key, or `None` if any key column is missing or is `Null`, which excludes the + /// row from the join (SQL inner-join semantics: `NULL` never joins). + /// + /// Each part carries its length, so two columns cannot run together into a key another pair of + /// values also renders. Rendered through [`Value::write_join_key_part`] rather than + /// [`Value::canonical_string`], which is `None` for `Float` and `Timestamp`. + fn render(&mut self, row: &[Value], positions: &[usize]) -> Option<&str> { + self.key.clear(); + for &i in positions { + self.part.clear(); + if !row.get(i)?.write_join_key_part(&mut self.part) { + return None; + } + let _ = write!(self.key, "{}:{}", self.part.len(), self.part); + } + Some(&self.key) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::core::event_data::object_centric::extraction::blueprint::Node; + use crate::core::event_data::object_centric::extraction::catalog::{ + ExtractionCatalog, TableSchema, + }; + use crate::core::event_data::object_centric::extraction::provider::ProviderError; + + /// A [`RowProvider`] over literal rows, so these tests can drive the executor without a + /// database, and without `validate` first rejecting the deliberately malformed graphs below. + #[derive(Debug)] + struct VecProvider(BTreeMap, Vec>)>); + + impl RowProvider for VecProvider { + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError> { + let (names, rows) = self + .0 + .get(table) + .ok_or_else(|| ProviderError::UnknownTable { + table: table.to_string(), + })?; + let positions: Vec = columns + .iter() + .map(|c| { + names + .iter() + .position(|n| n == c) + .ok_or_else(|| ProviderError::UnknownColumn { + table: table.to_string(), + column: (*c).to_string(), + }) + }) + .collect::>()?; + for row in rows { + let projected: Vec = positions.iter().map(|&i| row[i].clone()).collect(); + if f(&projected).is_break() { + break; + } + } + Ok(()) + } + } + + fn source(id: &str, table: &str) -> Node { + Node { + id: id.to_string(), + label: None, + op: NodeOp::Source { + source_id: "db".into(), + table: table.to_string(), + }, + } + } + + /// A `Join` whose `on` names a key column its input's rows do not carry must be an error. + /// Silently dropping that pair from the key shortens the key, so rows agreeing on only the + /// remaining columns get paired: a partial cross product reported as a successful run. + #[test] + fn a_join_key_column_missing_from_an_input_is_an_error_not_a_shorter_key() { + let provider = VecProvider( + [ + ( + "l".to_string(), + ( + vec!["id".to_string()], + vec![vec![Value::Integer(1)], vec![Value::Integer(2)]], + ), + ), + ( + "r".to_string(), + ( + vec!["id".to_string()], + vec![vec![Value::Integer(1)], vec![Value::Integer(2)]], + ), + ), + ] + .into_iter() + .collect(), + ); + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + + let blueprint = Blueprint { + version: crate::core::event_data::object_centric::extraction::MODEL_VERSION, + id_rendering: crate::core::event_data::object_centric::extraction::IdRendering::Raw, + nodes: vec![ + source("l", "l"), + source("r", "r"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + // "missing" is on neither side; `validate` would reject this blueprint, + // which is why this test drives the executor directly. + on: vec![("missing".into(), "id".into())], + }, + }, + ], + mappings: vec![], + on_missing_endpoint: Default::default(), + on_duplicate_object: Default::default(), + }; + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("l", [("id", "INTEGER", false)])) + .with_table("db", TableSchema::new("r", [("id", "INTEGER", false)])); + + let exec = GraphExecutor::new(&blueprint, &catalog, &providers, &[]).expect("executor"); + let err = exec + .stream("j", &mut |_| Ok(())) + .expect_err("must not silently cross-join"); + assert!( + matches!( + &err, + ExtractionError::JoinKeyColumnMissing { node, side, column } + if node == "j" && *side == "left" && column == "missing" + ), + "got {err:?}" + ); + } + + /// A left column literally named `right_` and the rename of the right's `` want + /// the same output name. + #[test] + fn a_join_output_column_claimed_by_both_sides_is_an_error_not_a_silent_choice() { + let provider = VecProvider( + [ + ( + "l".to_string(), + ( + vec!["id".to_string(), "right_id".to_string()], + vec![vec![Value::Integer(1), Value::Text("left".into())]], + ), + ), + ( + "r".to_string(), + (vec!["id".to_string()], vec![vec![Value::Integer(1)]]), + ), + ] + .into_iter() + .collect(), + ); + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + + let blueprint = Blueprint { + version: crate::core::event_data::object_centric::extraction::MODEL_VERSION, + id_rendering: crate::core::event_data::object_centric::extraction::IdRendering::Raw, + nodes: vec![ + source("l", "l"), + source("r", "r"), + Node { + id: "j".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("id".into(), "id".into())], + }, + }, + ], + mappings: vec![], + on_missing_endpoint: Default::default(), + on_duplicate_object: Default::default(), + }; + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("l", [("id", "INTEGER", false), ("right_id", "TEXT", false)]), + ) + .with_table("db", TableSchema::new("r", [("id", "INTEGER", false)])); + + let mapping = Mapping { + node: "j".into(), + label: None, + when: None, + target: crate::core::event_data::object_centric::extraction::Target::Object { + object_type: crate::core::event_data::object_centric::extraction::expr::ValueExpression::Constant { + value: "t".into(), + }, + id: crate::core::event_data::object_centric::extraction::expr::ValueExpression::Column { + column: "right_id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }; + let mappings = vec![("mappings[0]".to_string(), mapping)]; + + let exec = + GraphExecutor::new(&blueprint, &catalog, &providers, &mappings).expect("executor"); + let err = exec + .stream("j", &mut |_| Ok(())) + .expect_err("must not pick a side"); + assert!( + matches!( + &err, + ExtractionError::Invalid(errors) + if matches!( + errors.as_slice(), + [ValidationError::UnknownColumn { node, column }] + if node == "j" && column == "right_id" + ) + ), + "got {err:?}" + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/mapping_exec.rs b/process_mining/src/core/event_data/object_centric/extraction/mapping_exec.rs new file mode 100644 index 00000000..e3c4be0e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/mapping_exec.rs @@ -0,0 +1,1116 @@ +//! Turning one row into events, objects and relations, per [`Target`] kind. +//! +//! Every entity-and-relation path funnels through [`resolve_object_endpoint`] / +//! [`resolve_event_endpoint`], so `on_missing_endpoint` is honoured identically at an inline +//! object reference, an `E2O`'s object side and an `O2O`'s source and target. + +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, FixedOffset}; + +use super::blueprint::{ + Blueprint, DuplicateObjectPolicy, EventEndpoint, IdRendering, InlineObjectRef, + MissingEndpointPolicy, ObjectEndpoint, Target, +}; +use super::catalog::{ColumnSchema, TableSchema}; +use super::expr::{AttributeMapping, PreparedSplit, SplitSpec, TimestampSource, ValueExpression}; +use super::report::{DropReason, ErrorLog, ExtractionError, MappingRef, MappingStats}; +use super::row::Row; +use super::sink::{EventRef, ExtractionSink, ObjectRef, Resolution, SinkError}; +use super::validate::target_object_endpoints; +use super::value::{Value, ValueKind}; +use crate::core::event_data::object_centric::{ + OCELAttributeType, OCELAttributeValue, OCELTypeAttribute, +}; + +/// Per-extraction-run state a mapping's row processing needs, threaded through instead of +/// passed as a long, ever-growing parameter list. +/// +/// Everything here is sized by the blueprint, not by the data. Nothing tracks ids: the sink +/// answers both questions that would need to, deduplication (see [`MappingStats::deduplicated`]) +/// and the first-wins rule on a repeated `(id, attribute, time)` (see +/// [`ExtractionSink::add_object_attribute`]). +pub(crate) struct RunCtx<'a> { + /// The blueprint being executed, for its id-rendering and endpoint policies. + pub(crate) blueprint: &'a Blueprint, + /// Where entities and relations go. + pub(crate) sink: &'a mut dyn ExtractionSink, + /// Event type names this mapping has already declared. Reset per mapping. See + /// [`ensure_event_declared`]. + pub(crate) declared_events: &'a mut HashSet, + /// Object type names this mapping has already declared. See [`declared_events`](Self::declared_events). + pub(crate) declared_objects: &'a mut HashSet, + /// Non-fatal problems collected so far. + pub(crate) errors: &'a mut ErrorLog, + /// Every `(kind, type name, attribute name)` declared so far, and the type it was declared + /// with. Run-level rather than per mapping, see [`reconcile_attr_types`]. + pub(crate) attr_types: &'a mut DeclaredAttrTypes, + /// Scratch buffer for one event row's attribute values, owned by the run rather than by this + /// context, which is rebuilt per row. See [`fill_event_attrs`]. + pub(crate) event_attrs: &'a mut Vec<(String, OCELAttributeValue)>, + /// Scratch buffer for one object row's timed attribute values. + pub(crate) object_attrs: &'a mut Vec<(String, DateTime, OCELAttributeValue)>, +} + +/// `kind -> type name -> attribute name -> declared value type`, accumulated across every mapping +/// in a run. `kind` is `"event"` or `"object"`. +/// +/// Nested rather than keyed by one tuple so [`effective_attr_type`], which runs once per attribute +/// per row, probes it with `&str` lookups instead of building an owning key. +pub(crate) type DeclaredAttrTypes = + HashMap<&'static str, HashMap>>; + +/// The type `(kind, type_name, attribute)` was declared with, if it has been declared. +fn declared_attr_type( + declared: &DeclaredAttrTypes, + kind: &'static str, + type_name: &str, + attribute: &str, +) -> Option { + declared.get(kind)?.get(type_name)?.get(attribute).copied() +} + +/// Record `ty` as what `(kind, type_name, attribute)` is declared with. +fn record_attr_type( + declared: &mut DeclaredAttrTypes, + kind: &'static str, + type_name: &str, + attribute: &str, + ty: OCELAttributeType, +) { + declared + .entry(kind) + .or_default() + .entry(type_name.to_string()) + .or_default() + .insert(attribute.to_string(), ty); +} + +/// Record what `attrs` declares for `type_name`, reporting any attribute two mappings declare +/// under genuinely different types. +/// +/// Reported as [`ExtractionError::ConflictingAttributeType`], with the declaration widened via +/// [`OCELAttributeType::coalesce`]. Rows are converted to the widened type too, not only the +/// declaration: see [`effective_attr_type`]. +pub(crate) fn reconcile_attr_types( + kind: &'static str, + type_name: &str, + attrs: &[OCELTypeAttribute], + seen: &mut DeclaredAttrTypes, + errors: &mut ErrorLog, +) -> Vec { + attrs + .iter() + .map(|a| { + let declared = OCELAttributeType::from_type_str(&a.value_type); + match declared_attr_type(seen, kind, type_name, &a.name) { + Some(previous) if previous != declared => { + let widened = previous.coalesce(declared); + errors.push(ExtractionError::ConflictingAttributeType { + kind, + type_name: type_name.to_string(), + attribute: a.name.clone(), + declared: previous, + conflicting: declared, + }); + record_attr_type(seen, kind, type_name, &a.name, widened); + OCELTypeAttribute::new(&a.name, &widened) + } + Some(previous) => OCELTypeAttribute::new(&a.name, &previous), + None => { + record_attr_type(seen, kind, type_name, &a.name, declared); + a.clone() + } + } + }) + .collect() +} + +/// Prepare every [`ObjectEndpoint`] a target names, in the order [`target_object_endpoints`] +/// walks them, which is what this module's `run_*` functions index into. +pub(crate) fn prepare_splits(target: &Target) -> Result>, regex::Error> { + target_object_endpoints(target) + .into_iter() + .map(|e| e.split.as_ref().map(SplitSpec::prepare).transpose()) + .collect() +} + +/// One of [`extract`](super::extract::extract)'s three passes, run in the order they are +/// declared here. +/// +/// Endpoint resolution is staged, not incremental: every `Object` target runs first, then every +/// `Event` target, then everything that relates two of them. Resolving incrementally would make +/// a relation mapping's output depend on mapping order, and is unreproducible in SQL, where a +/// relation view joins against all objects rather than those emitted so far. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Phase { + /// Every `Object` target. + Objects, + /// Every `Event` target, plus the inline object references of an event whose id this run + /// mints itself (see [`MappingPasses`]). Runs after every object exists. + Events, + /// Every `E2O` and `O2O` target, plus the inline object references of an event with an + /// author-given id. + Relations, +} + +/// Which of the three [`Phase`]s one mapping does work in. +/// +/// Almost everything belongs to exactly one. The exception is a +/// [`Target::Event`] with inline object references: the event itself is an entity +/// ([`Phase::Events`]) and its references are relations ([`Phase::Relations`]), so such a +/// mapping's node is read once per pass. +/// +/// Unless the event's `id` is `None`: the id is then a freshly minted `UUID`, which the relations +/// pass could not re-derive, so the references go in the events pass instead. That is sound +/// because the objects pass has already run. +#[derive(Debug, Clone, Copy)] +pub(crate) struct MappingPasses { + /// Runs in [`Phase::Objects`]. + pub(crate) objects: bool, + /// Runs in [`Phase::Events`]. + pub(crate) events: bool, + /// Runs in [`Phase::Relations`]. + pub(crate) relations: bool, +} + +impl MappingPasses { + /// Which passes `target` needs. + pub(crate) fn of(target: &Target) -> Self { + match target { + Target::Event { id, objects, .. } => Self { + objects: false, + events: true, + relations: id.is_some() && !objects.is_empty(), + }, + Target::Object { .. } => Self { + objects: true, + events: false, + relations: false, + }, + Target::E2O { .. } | Target::O2O { .. } => Self { + objects: false, + events: false, + relations: true, + }, + } + } + + /// Whether this mapping does anything in `phase`. + pub(crate) fn runs_in(self, phase: Phase) -> bool { + match phase { + Phase::Objects => self.objects, + Phase::Events => self.events, + Phase::Relations => self.relations, + } + } + + /// The pass whose rows count toward this mapping's `rows_read` and `PredicateExcluded` + /// tallies, so a mapping read in more than one pass does not report its rows twice. + pub(crate) fn counting_phase(self) -> Phase { + if self.objects { + Phase::Objects + } else if self.events { + Phase::Events + } else { + Phase::Relations + } + } +} + +/// Execute one mapping's target against one row. +/// +/// # Errors +/// Returns [`ExtractionError`] for any sink failure, which aborts the run. Policy violations are +/// pushed to `ctx.errors` instead, so one bad row does not. +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_target( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + node_schema: Option<&TableSchema>, + target: &Target, + splits: &[Option], + row: &Row<'_>, + stats: &mut MappingStats, + phase: Phase, +) -> Result<(), ExtractionError> { + match (target, phase) { + ( + Target::Event { + event_type, + id, + timestamp, + attributes, + objects, + }, + Phase::Events, + ) => run_event( + ctx, + mapping_ref, + node_schema, + splits, + row, + stats, + event_type, + id, + timestamp, + attributes, + objects, + ), + ( + Target::Event { + event_type, + id, + objects, + .. + }, + Phase::Relations, + ) => run_event_inline_objects( + ctx, + mapping_ref, + splits, + row, + stats, + event_type, + id.as_ref(), + objects, + ), + ( + Target::Object { + object_type, + id, + timestamp, + attributes, + }, + Phase::Objects, + ) => run_object( + ctx, + mapping_ref, + node_schema, + row, + stats, + object_type, + id, + timestamp, + attributes, + ), + ( + Target::E2O { + event, + object, + qualifier, + }, + Phase::Relations, + ) => run_e2o( + ctx, + mapping_ref, + splits, + row, + stats, + event, + object, + qualifier, + ), + ( + Target::O2O { + source, + target, + qualifier, + }, + Phase::Relations, + ) => run_o2o( + ctx, + mapping_ref, + splits, + row, + stats, + source, + target, + qualifier, + ), + // The combinations `MappingPasses::of` never schedules. + (Target::Object { .. }, Phase::Events | Phase::Relations) + | (Target::Event { .. }, Phase::Objects) + | (Target::E2O { .. } | Target::O2O { .. }, Phase::Objects | Phase::Events) => Ok(()), + } +} + +#[allow(clippy::too_many_arguments)] +fn run_event( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + node_schema: Option<&TableSchema>, + splits: &[Option], + row: &Row<'_>, + stats: &mut MappingStats, + event_type: &ValueExpression, + id: &Option, + timestamp: &TimestampSource, + attributes: &[AttributeMapping], + objects: &[InlineObjectRef], +) -> Result<(), ExtractionError> { + let Some(type_name) = event_type.evaluate(row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + ensure_event_declared(ctx, &type_name, attributes, node_schema)?; + + let raw_id = match id { + Some(expr) => match identity(expr, row) { + Some(s) => s, + None => { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + } + }, + None => uuid::Uuid::new_v4().to_string(), + }; + let rendered_id = render_id(ctx.blueprint.id_rendering, &raw_id, &type_name); + + let Some(ts) = timestamp.parse(row) else { + stats.drop(drop_reason_for(timestamp, row)); + return Ok(()); + }; + + let mut attrs = std::mem::take(ctx.event_attrs); + fill_event_attrs( + &mut attrs, + attributes, + ctx.attr_types, + &type_name, + node_schema, + row, + stats, + ); + let added = ctx.sink.add_event(&type_name, ts, &rendered_id, &attrs); + *ctx.event_attrs = attrs; + + let ev_ref = match added { + Ok(r) => r, + Err(SinkError::DuplicateEvent { .. }) => { + // The dropped event's inline object references are not lost with it: the relations + // pass emits them against the event this id already names. + stats.deduplicated += 1; + return Ok(()); + } + Err(e) => { + return Err(ExtractionError::Sink { + context: format!("adding event '{rendered_id}'"), + source: e, + }) + } + }; + stats.entities_emitted += 1; + + // Only when the id was minted here: see `MappingPasses`. + if id.is_none() { + for (i, o) in objects.iter().enumerate() { + run_inline_object( + ctx, + mapping_ref, + &ev_ref, + o, + splits.get(i).and_then(Option::as_ref), + row, + stats, + )?; + } + } + Ok(()) +} + +/// The relations-pass half of a [`Target::Event`] with an author-given id: re-derive which event +/// this row named, then emit its inline object references against it. +#[allow(clippy::too_many_arguments)] +fn run_event_inline_objects( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + splits: &[Option], + row: &Row<'_>, + stats: &mut MappingStats, + event_type: &ValueExpression, + id: Option<&ValueExpression>, + objects: &[InlineObjectRef], +) -> Result<(), ExtractionError> { + let (Some(type_name), Some(id_expr)) = (event_type.evaluate(row), id) else { + // Already counted in the events pass, where this row produced no event either. + return Ok(()); + }; + let Some(raw_id) = identity(id_expr, row) else { + return Ok(()); + }; + let rendered_id = render_id(ctx.blueprint.id_rendering, &raw_id, &type_name); + + let Some(ev_ref) = ctx + .sink + .resolve_event(&rendered_id, Some(&type_name)) + .into_ref() + else { + // The event this row names does not exist, because its own events-pass run dropped it. + for _ in objects { + stats.drop(DropReason::UnresolvedEndpoint); + } + return Ok(()); + }; + + for (i, o) in objects.iter().enumerate() { + run_inline_object( + ctx, + mapping_ref, + &ev_ref, + o, + splits.get(i).and_then(Option::as_ref), + row, + stats, + )?; + } + Ok(()) +} + +fn run_inline_object( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + ev_ref: &EventRef, + o: &InlineObjectRef, + split: Option<&PreparedSplit>, + row: &Row<'_>, + stats: &mut MappingStats, +) -> Result<(), ExtractionError> { + let Some(raw) = identity(&o.object.id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let raw_ids = split_or_single(split, raw); + if raw_ids.is_empty() { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + } + let qualifier = o + .qualifier + .as_ref() + .and_then(|q| q.evaluate(row)) + .unwrap_or_default(); + for raw_id in raw_ids { + match resolve_object_endpoint(ctx, mapping_ref, &o.object, &raw_id, "inline object", row)? { + Some(obj_ref) => { + ctx.sink + .add_e2o(ev_ref, &obj_ref, &qualifier) + .map_err(|e| ExtractionError::Sink { + context: "adding inline object relation".to_string(), + source: e, + })?; + stats.entities_emitted += 1; + } + None => stats.drop(DropReason::UnresolvedEndpoint), + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn run_object( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + node_schema: Option<&TableSchema>, + row: &Row<'_>, + stats: &mut MappingStats, + object_type: &ValueExpression, + id: &ValueExpression, + timestamp: &Option, + attributes: &[AttributeMapping], +) -> Result<(), ExtractionError> { + let Some(type_name) = object_type.evaluate(row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + ensure_object_declared(ctx, &type_name, attributes, node_schema)?; + + let Some(raw_id) = identity(id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let rendered_id = render_id(ctx.blueprint.id_rendering, &raw_id, &type_name); + + let ts = match timestamp { + Some(source) => match source.parse(row) { + Some(t) => t, + None => { + stats.drop(drop_reason_for(source, row)); + return Ok(()); + } + }, + // A static object attribute has no instant of its own, so every row writes at the epoch + // and the sink's first-wins rule keeps one value. + None => DateTime::UNIX_EPOCH.into(), + }; + let mut attrs = std::mem::take(ctx.object_attrs); + fill_object_attrs( + &mut attrs, + attributes, + ctx.attr_types, + &type_name, + node_schema, + row, + ts, + stats, + ); + + let resolution = ctx.sink.resolve_object(&rendered_id, Some(&type_name)); + // A deferring sink's handle. It cannot say whether the object exists, so `add_object` below is + // what finds out, and this handle is what the attributes are written against if it does. + let mut deferred = None; + let existing = match resolution { + Resolution::Exists(r) => Some(r), + Resolution::Deferred(r) => { + deferred = Some(r); + None + } + Resolution::Missing => None, + }; + + let outcome = if let Some(existing) = existing { + write_to_existing_object(ctx, mapping_ref, stats, &existing, &rendered_id, &attrs) + } else { + match ctx.sink.add_object(&type_name, &rendered_id, &attrs) { + Ok(_) => { + stats.entities_emitted += 1; + Ok(()) + } + // The id already exists, which is what an eager sink answers `Exists` to, so this + // takes the same path. + Err(SinkError::DuplicateObject { .. }) if deferred.is_some() => { + let existing = deferred.expect("guarded above"); + write_to_existing_object(ctx, mapping_ref, stats, &existing, &rendered_id, &attrs) + } + // The id is taken by an object of a different type, which only `IdRendering::Raw` + // allows. Two distinct entities collided, so the row is dropped rather than written + // onto the other type's object. + Err(SinkError::DuplicateObject { .. } | SinkError::IdTypeCollision { .. }) => { + stats.drop(DropReason::IdTypeCollision); + ctx.errors.push(ExtractionError::IdTypeCollision { + mapping: mapping_ref.clone(), + id: rendered_id.clone(), + requested_type: type_name, + }); + Ok(()) + } + Err(e) => Err(ExtractionError::Sink { + context: format!("adding object '{rendered_id}'"), + source: e, + }), + } + }; + *ctx.object_attrs = attrs; + outcome +} + +/// What a [`Target::Object`] row does when the object it names already exists, whether the sink +/// reported that as an eager `Exists` or as an `add_object` rejection. +/// +/// Every attribute the row carries is offered to the sink unconditionally. A change-tracked +/// mapping's rows carry distinct timestamps and are all recorded. A static mapping's all carry +/// the epoch, so the sink's first-wins rule on `(id, name, time)` keeps exactly one. See +/// [`ExtractionSink::add_object_attribute`]. +fn write_to_existing_object( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + stats: &mut MappingStats, + existing: &ObjectRef, + rendered_id: &str, + attrs: &[(String, DateTime, OCELAttributeValue)], +) -> Result<(), ExtractionError> { + match ctx.blueprint.on_duplicate_object { + DuplicateObjectPolicy::Error => { + // A loss, not a deduplication: the row's attributes go nowhere. + stats.drop(DropReason::DuplicateObjectRejected); + ctx.errors.push(ExtractionError::DuplicateObject { + mapping: mapping_ref.clone(), + id: rendered_id.to_string(), + }); + } + DuplicateObjectPolicy::FirstWins => { + stats.deduplicated += 1; + for (name, t, v) in attrs { + ctx.sink + .add_object_attribute(existing, name, *t, v.clone()) + .map_err(|e| ExtractionError::Sink { + context: format!("appending attribute to object '{rendered_id}'"), + source: e, + })?; + } + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn run_e2o( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + splits: &[Option], + row: &Row<'_>, + stats: &mut MappingStats, + event: &EventEndpoint, + object: &ObjectEndpoint, + qualifier: &Option, +) -> Result<(), ExtractionError> { + let Some(ev_raw) = identity(&event.id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let Some(ev_ref) = resolve_event_endpoint(ctx, mapping_ref, event, &ev_raw, row)? else { + stats.drop(DropReason::UnresolvedEndpoint); + return Ok(()); + }; + + let Some(obj_raw) = identity(&object.id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let raw_ids = split_or_single(splits.first().and_then(Option::as_ref), obj_raw); + if raw_ids.is_empty() { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + } + let q = qualifier + .as_ref() + .and_then(|e| e.evaluate(row)) + .unwrap_or_default(); + for raw_id in raw_ids { + match resolve_object_endpoint(ctx, mapping_ref, object, &raw_id, "object", row)? { + Some(obj_ref) => { + ctx.sink + .add_e2o(&ev_ref, &obj_ref, &q) + .map_err(|e| ExtractionError::Sink { + context: "adding e2o relation".to_string(), + source: e, + })?; + stats.entities_emitted += 1; + } + None => stats.drop(DropReason::UnresolvedEndpoint), + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn run_o2o( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + splits: &[Option], + row: &Row<'_>, + stats: &mut MappingStats, + source: &ObjectEndpoint, + target: &ObjectEndpoint, + qualifier: &Option, +) -> Result<(), ExtractionError> { + let Some(src_raw) = identity(&source.id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let Some(tgt_raw) = identity(&target.id, row) else { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + }; + let src_ids = split_or_single(splits.first().and_then(Option::as_ref), src_raw); + let tgt_ids = split_or_single(splits.get(1).and_then(Option::as_ref), tgt_raw); + if src_ids.is_empty() || tgt_ids.is_empty() { + stats.drop(DropReason::NullOrUnrenderableId); + return Ok(()); + } + let q = qualifier + .as_ref() + .and_then(|e| e.evaluate(row)) + .unwrap_or_default(); + for s_raw in &src_ids { + let Some(s_ref) = resolve_object_endpoint(ctx, mapping_ref, source, s_raw, "source", row)? + else { + stats.drop(DropReason::UnresolvedEndpoint); + continue; + }; + for t_raw in &tgt_ids { + match resolve_object_endpoint(ctx, mapping_ref, target, t_raw, "target", row)? { + Some(t_ref) => { + ctx.sink + .add_o2o(&s_ref, &t_ref, &q) + .map_err(|e| ExtractionError::Sink { + context: "adding o2o relation".to_string(), + source: e, + })?; + stats.entities_emitted += 1; + } + None => stats.drop(DropReason::UnresolvedEndpoint), + } + } + } + Ok(()) +} + +/// Resolve an object endpoint to a handle, applying `on_missing_endpoint`. Every object-relating +/// position (inline references, `E2O`'s object, `O2O`'s source/target) goes through here, so the +/// policy is honoured identically at each. +/// +/// Touches no counter, which is why it takes no [`MappingStats`]: resolving an endpoint that +/// already exists is the normal successful case, not a deduplication. +fn resolve_object_endpoint( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + endpoint: &ObjectEndpoint, + raw_id: &str, + label: &'static str, + row: &Row<'_>, +) -> Result, ExtractionError> { + let type_name = endpoint.object_type.as_ref().and_then(|e| e.evaluate(row)); + let rendered_id = match ctx.blueprint.id_rendering { + IdRendering::Raw => raw_id.to_string(), + IdRendering::TypePrefixed => match &type_name { + Some(t) => format!("{t}-{raw_id}"), + None => return Ok(None), + }, + }; + // `Deferred` is a handle to write a relation against, not a promise the object exists: the + // sink resolves it at `finalize` and applies `on_missing_endpoint` there. + match ctx.sink.resolve_object(&rendered_id, type_name.as_deref()) { + Resolution::Exists(r) | Resolution::Deferred(r) => return Ok(Some(r)), + Resolution::Missing => {} + } + match ctx.blueprint.on_missing_endpoint { + MissingEndpointPolicy::Drop => Ok(None), + MissingEndpointPolicy::Error => { + ctx.errors.push(ExtractionError::MissingEndpoint { + mapping: mapping_ref.clone(), + endpoint: label, + id: rendered_id, + }); + Ok(None) + } + MissingEndpointPolicy::Create => { + let Some(t) = &type_name else { + return Ok(None); + }; + if ctx.declared_objects.insert(t.clone()) { + ctx.sink + .declare_object_type(t, &[]) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{t}'"), + source: e, + })?; + } + match ctx.sink.add_object(t, &rendered_id, &[]) { + Ok(r) => Ok(Some(r)), + // The id is taken by an object of a different type, which is why + // `resolve_object` answered `Missing`. Creating is impossible and merging the two + // types would be worse, so the relation is dropped. + Err(SinkError::DuplicateObject { .. }) => { + ctx.errors.push(ExtractionError::IdTypeCollision { + mapping: mapping_ref.clone(), + id: rendered_id, + requested_type: t.clone(), + }); + Ok(None) + } + Err(e) => Err(ExtractionError::Sink { + context: format!("creating missing object '{rendered_id}'"), + source: e, + }), + } + } + } +} + +/// Resolve an event endpoint (`E2O`'s event side) to a handle. +/// +/// `Create` cannot synthesise a missing event, since there is no timestamp to give it, so it +/// behaves like `Drop` here. Only object endpoints can be created (see +/// [`resolve_object_endpoint`]). +fn resolve_event_endpoint( + ctx: &mut RunCtx<'_>, + mapping_ref: &MappingRef, + endpoint: &EventEndpoint, + raw_id: &str, + row: &Row<'_>, +) -> Result, ExtractionError> { + let type_name = endpoint.event_type.as_ref().and_then(|e| e.evaluate(row)); + let rendered_id = match ctx.blueprint.id_rendering { + IdRendering::Raw => raw_id.to_string(), + IdRendering::TypePrefixed => match &type_name { + Some(t) => format!("{t}-{raw_id}"), + None => return Ok(None), + }, + }; + match ctx.sink.resolve_event(&rendered_id, type_name.as_deref()) { + Resolution::Exists(r) | Resolution::Deferred(r) => return Ok(Some(r)), + Resolution::Missing => {} + } + match ctx.blueprint.on_missing_endpoint { + MissingEndpointPolicy::Drop | MissingEndpointPolicy::Create => Ok(None), + MissingEndpointPolicy::Error => { + ctx.errors.push(ExtractionError::MissingEndpoint { + mapping: mapping_ref.clone(), + endpoint: "event", + id: rendered_id, + }); + Ok(None) + } + } +} + +fn ensure_event_declared( + ctx: &mut RunCtx<'_>, + name: &str, + attributes: &[AttributeMapping], + node_schema: Option<&TableSchema>, +) -> Result<(), ExtractionError> { + if ctx.declared_events.insert(name.to_string()) { + let attrs = reconcile_attr_types( + "event", + name, + &build_type_attrs(attributes, node_schema), + ctx.attr_types, + ctx.errors, + ); + ctx.sink + .declare_event_type(name, &attrs) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring event type '{name}'"), + source: e, + })?; + } + Ok(()) +} + +fn ensure_object_declared( + ctx: &mut RunCtx<'_>, + name: &str, + attributes: &[AttributeMapping], + node_schema: Option<&TableSchema>, +) -> Result<(), ExtractionError> { + if ctx.declared_objects.insert(name.to_string()) { + let attrs = reconcile_attr_types( + "object", + name, + &build_type_attrs(attributes, node_schema), + ctx.attr_types, + ctx.errors, + ); + ctx.sink + .declare_object_type(name, &attrs) + .map_err(|e| ExtractionError::Sink { + context: format!("declaring object type '{name}'"), + source: e, + })?; + } + Ok(()) +} + +/// The declared [`OCELTypeAttribute`] list for a target's attribute mappings, used both to +/// declare a type and (by [`resolve_attribute_type`]) to convert each row's values. +pub(crate) fn build_type_attrs( + attributes: &[AttributeMapping], + node_schema: Option<&TableSchema>, +) -> Vec { + attributes + .iter() + .map(|a| { + let t = resolve_attribute_type(a, node_schema); + OCELTypeAttribute::new(&a.name, &t) + }) + .collect() +} + +/// An attribute's declared type: `value_type` if given, else the source column's declared kind +/// (from `node_schema`, which carries [`Predicate::prepare`](super::predicate::Predicate::prepare)'s +/// same catalog-derived types) mapped to an [`OCELAttributeType`], else `String`. +fn resolve_attribute_type( + a: &AttributeMapping, + node_schema: Option<&TableSchema>, +) -> OCELAttributeType { + if let Some(t) = a.value_type { + return t; + } + node_schema + .and_then(|s| s.columns.get(&a.source_column)) + .and_then(ColumnSchema::declared_kind) + .map(kind_to_attr_type) + .unwrap_or(OCELAttributeType::String) +} + +/// The type one row's value for `a` is converted to: the reconciled declaration for +/// `(kind, type_name, a.name)` when there is one, not this mapping's own `value_type`. +/// +/// Two mappings declaring one attribute under different types have the declaration widened by +/// [`reconcile_attr_types`], and each mapping's rows convert to the widened type. +/// +/// Only as good as what has been declared so far: a type named from a column declares lazily, +/// so rows written before a later mapping widens the declaration keep the narrower type. +fn effective_attr_type( + declared: &DeclaredAttrTypes, + kind: &'static str, + type_name: &str, + a: &AttributeMapping, + node_schema: Option<&TableSchema>, +) -> OCELAttributeType { + declared_attr_type(declared, kind, type_name, &a.name) + .unwrap_or_else(|| resolve_attribute_type(a, node_schema)) +} + +/// Overwrite `buf` with this row's values for `attributes`, reusing the `Vec` and each name +/// `String`, since one mapping's attribute list is the same on every row. +fn fill_event_attrs( + buf: &mut Vec<(String, OCELAttributeValue)>, + attributes: &[AttributeMapping], + declared: &DeclaredAttrTypes, + type_name: &str, + node_schema: Option<&TableSchema>, + row: &Row<'_>, + stats: &mut MappingStats, +) { + buf.truncate(attributes.len()); + for (i, a) in attributes.iter().enumerate() { + let ty = effective_attr_type(declared, "event", type_name, a, node_schema); + let value = attribute_value(row.get(&a.source_column), ty, stats); + match buf.get_mut(i) { + Some(slot) => { + set_name(&mut slot.0, &a.name); + slot.1 = value; + } + None => buf.push((a.name.clone(), value)), + } + } +} + +/// [`fill_event_attrs`] for an object's timed values, all at `ts`. +#[allow(clippy::too_many_arguments)] +fn fill_object_attrs( + buf: &mut Vec<(String, DateTime, OCELAttributeValue)>, + attributes: &[AttributeMapping], + declared: &DeclaredAttrTypes, + type_name: &str, + node_schema: Option<&TableSchema>, + row: &Row<'_>, + ts: DateTime, + stats: &mut MappingStats, +) { + buf.truncate(attributes.len()); + for (i, a) in attributes.iter().enumerate() { + let ty = effective_attr_type(declared, "object", type_name, a, node_schema); + let value = attribute_value(row.get(&a.source_column), ty, stats); + match buf.get_mut(i) { + Some(slot) => { + set_name(&mut slot.0, &a.name); + slot.1 = ts; + slot.2 = value; + } + None => buf.push((a.name.clone(), ts, value)), + } + } +} + +fn set_name(slot: &mut String, name: &str) { + slot.clear(); + slot.push_str(name); +} + +fn kind_to_attr_type(k: ValueKind) -> OCELAttributeType { + match k { + ValueKind::Text => OCELAttributeType::String, + ValueKind::Integer => OCELAttributeType::Integer, + ValueKind::Float => OCELAttributeType::Float, + ValueKind::Boolean => OCELAttributeType::Boolean, + ValueKind::Timestamp => OCELAttributeType::Time, + } +} + +fn kind_from_attr_type(t: OCELAttributeType) -> Option { + match t { + OCELAttributeType::String => Some(ValueKind::Text), + OCELAttributeType::Integer => Some(ValueKind::Integer), + OCELAttributeType::Float => Some(ValueKind::Float), + OCELAttributeType::Boolean => Some(ValueKind::Boolean), + OCELAttributeType::Time => Some(ValueKind::Timestamp), + OCELAttributeType::Null => None, + } +} + +/// Render `v` as `declared`, coercing when it does not already match. +/// +/// A value that will not coerce becomes `Null`, counted in +/// [`MappingStats::uncoercible_attributes`]. Keeping `v`'s own rendering instead would let the two +/// sinks disagree on the same input, since a typed column in `DuckDB` stores `NULL` regardless. +/// +/// An attribute declared with no type at all ([`OCELAttributeType::Null`]) has nothing to convert +/// to, so its value is stored as it comes. +fn attribute_value( + v: Option<&Value>, + declared: OCELAttributeType, + stats: &mut MappingStats, +) -> OCELAttributeValue { + let Some(v) = v else { + return OCELAttributeValue::Null; + }; + let Some(kind) = kind_from_attr_type(declared) else { + return natural(v); + }; + match v.coerce_to(kind) { + Some(coerced) => natural(&coerced), + None => { + // A `Null` cell is an absent value, not one the declaration could not hold. + if !matches!(v, Value::Null) { + stats.uncoercible_attributes += 1; + } + OCELAttributeValue::Null + } + } +} + +fn natural(v: &Value) -> OCELAttributeValue { + match v { + Value::Null => OCELAttributeValue::Null, + Value::Text(s) => OCELAttributeValue::String(s.clone()), + Value::Integer(i) => OCELAttributeValue::Integer(*i), + Value::Float(f) => OCELAttributeValue::Float(*f), + Value::Boolean(b) => OCELAttributeValue::Boolean(*b), + Value::Timestamp(t) => OCELAttributeValue::Time(*t), + } +} + +fn render_id(id_rendering: IdRendering, raw: &str, type_name: &str) -> String { + match id_rendering { + IdRendering::Raw => raw.to_string(), + IdRendering::TypePrefixed => format!("{type_name}-{raw}"), + } +} + +/// The ids one endpoint cell yields: the split parts, or the cell itself when there is no split. +/// +/// An empty cell yields nothing, so the caller counts it as +/// [`DropReason::NullOrUnrenderableId`] and drops the row: `''` is how an ERP export writes "no +/// id", and accepting it collapses every such row into one entity. +fn split_or_single(split: Option<&PreparedSplit>, raw: String) -> Vec { + match split { + Some(s) => s.split(&raw), + None => { + if raw.is_empty() { + Vec::new() + } else { + vec![raw] + } + } + } +} + +/// An id expression's value, or `None` when it renders to nothing usable as an identity -- +/// `Null`, an unrenderable value, or the empty string (see [`split_or_single`]). +fn identity(expr: &ValueExpression, row: &Row<'_>) -> Option { + expr.evaluate(row).filter(|s| !s.is_empty()) +} + +/// Why a timestamp yielded nothing: a value that would not parse, or no value to parse. The two +/// call for opposite fixes, so the report keeps them apart. +fn drop_reason_for(timestamp: &TimestampSource, row: &Row<'_>) -> DropReason { + if timestamp.has_input(row) { + DropReason::UnparseableTimestamp + } else { + DropReason::MissingTimestamp + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/mod.rs b/process_mining/src/core/event_data/object_centric/extraction/mod.rs new file mode 100644 index 00000000..c5ee6dcd --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/mod.rs @@ -0,0 +1,107 @@ +//! Build an OCEL from relational data using a declarative blueprint. +//! +//! A [`Blueprint`](crate::core::event_data::object_centric::extraction::Blueprint) declares a flat +//! graph of nodes producing rows, and mappings turning those rows into OCEL events, objects and +//! relations. It contains no connection details and no schema snapshot: both are supplied by the +//! caller at execution time, which keeps a blueprint portable and free of secrets. + +/// The blueprint schema version this build reads and writes. +/// +/// A Blueprint's `version` field is checked against this during validation. +pub const MODEL_VERSION: u32 = 1; + +pub mod blueprint; +pub mod case_centric; +pub mod catalog; +pub mod compile; +#[cfg(feature = "extraction-dbcon")] +pub mod dbcon_provider; +pub mod desugar; +// The `DuckDbSink`-driving half is gated item by item inside; `snapshot`/`OcelSnapshot` are +// pure and are what the ordering tests compare with, so gating the whole module on +// `ocel-duckdb` made `--features extraction-blueprint` alone fail to compile its own tests. +#[cfg(test)] +mod differential; +#[cfg(feature = "ocel-duckdb")] +pub mod duckdb_sink; +pub mod expr; +mod extract; +mod graph; +mod mapping_exec; +pub mod predicate; +pub mod provider; +mod pushdown; +pub mod report; +pub(crate) mod row; +mod schema; +pub mod sink; +pub mod slim_sink; + +#[cfg(feature = "ocel-sqlite")] +pub mod sqlite_provider; +#[cfg(test)] +mod tests; +pub mod validate; +pub mod value; + +pub use blueprint::{ + Blueprint, BlueprintParseError, DuplicateObjectPolicy, EventEndpoint, IdRendering, + InlineObjectRef, Mapping, MappingEntry, MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, + Target, +}; +pub use case_centric::{ + event_log_to_ocel, event_log_to_slim_ocel, write_event_log_to_sink, EventLogWriteReport, + FlatEventTable, CASE_OBJECT_TYPE, CASE_QUALIFIER, +}; +pub use catalog::{Catalog, ColumnSchema, ExtractionCatalog, TablePreview, TableSchema}; +pub use compile::{ + compile, CompileError, CompiledOcel, EmissionShape, Probe, ProbeKind, RejectReason, SqlDialect, + ViewDef, +}; +#[cfg(feature = "extraction-dbcon")] +pub use dbcon_provider::{discover_catalog, DbconProviderError, DbconRowProvider}; +pub use desugar::desugar; +#[cfg(feature = "ocel-duckdb")] +pub use duckdb_sink::DuckDbSink; +pub use expr::{ + AttributeMapping, SplitKind, SplitSpec, TimestampFormat, TimestampSource, ValueExpression, +}; +pub use extract::extract; +pub use predicate::{CompareOp, Literal, Operand, Predicate}; +pub use provider::{ProviderError, RowProvider}; +pub use report::{ + DropReason, ExtractionError, ExtractionReport, ExtractionTiming, MappingRef, MappingStats, +}; +pub use sink::{EventRef, ExtractionSink, FinalizeReport, ObjectRef, Resolution, SinkError}; +pub use slim_sink::SlimOcelSink; +#[cfg(feature = "ocel-sqlite")] +pub use sqlite_provider::SqliteRowProvider; +pub use validate::{validate, ValidationError}; +pub use value::{Value, ValueKind}; + +#[cfg(test)] +mod schema_tests { + use super::*; + + #[test] + fn the_blueprint_schema_names_its_top_level_fields() { + let schema = schemars::schema_for!(Blueprint); + let json = serde_json::to_value(&schema).expect("serialize schema"); + let properties = json + .get("properties") + .and_then(|p| p.as_object()) + .expect("an object schema with properties"); + for field in ["version", "id_rendering", "nodes", "mappings"] { + assert!( + properties.contains_key(field), + "schema is missing '{field}'" + ); + } + } + + #[test] + fn the_catalog_schema_is_generated_too() { + let schema = schemars::schema_for!(ExtractionCatalog); + assert!(serde_json::to_value(&schema).is_ok()); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/predicate.rs b/process_mining/src/core/event_data/object_centric/extraction/predicate.rs new file mode 100644 index 00000000..ddbe5c4e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/predicate.rs @@ -0,0 +1,754 @@ +//! Row predicates, used both to filter a node's rows and as a mapping's `when` guard. + +use std::collections::HashSet; + +use chrono::{DateTime, FixedOffset}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::catalog::{ColumnSchema, TableSchema}; +use super::row::Row; +use super::value::{Value, ValueKind}; + +/// A literal in a comparison or membership test. +/// +/// Untagged in JSON, so `true`, `5`, `5.0` and `"x"` deserialize to the matching variant. +/// Order matters for `Boolean`, `Integer`, `Float` and `Text`, whose JSON shapes overlap only +/// with each other (a bare JSON boolean, number or string): boolean before integer before float +/// before text. +/// +/// `Timestamp` is exempt from that ordering: it is a single-field object (`{"timestamp": "..."}`) +/// rather than a bare string, so its position does not matter and no ordinary string that happens +/// to parse as RFC 3339 is silently reclassified. A plain string against a timestamp column works +/// without it, through `Predicate::prepare`'s literal coercion. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum Literal { + /// A boolean. + Boolean(bool), + /// An integer. + Integer(i64), + /// A float. + Float(f64), + /// Text. + Text(String), + /// An instant with a fixed UTC offset, given as RFC 3339 text. + Timestamp { + /// The instant. + timestamp: DateTime, + }, +} + +impl Literal { + /// The [`Value`] this literal denotes, with no coercion applied. + /// + /// See `Predicate::prepare` for the coercion `Compare` applies on top of this when a + /// column's declared type is known. + #[must_use] + pub fn as_value(&self) -> Value { + match self { + Literal::Boolean(b) => Value::Boolean(*b), + Literal::Integer(i) => Value::Integer(*i), + Literal::Float(f) => Value::Float(*f), + Literal::Text(s) => Value::Text(s.clone()), + Literal::Timestamp { timestamp } => Value::Timestamp(*timestamp), + } + } +} + +/// One side of a comparison. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum Operand { + /// The value in a column of the current row. + Column { + /// Column name. + column: String, + }, + /// A fixed value. + Literal { + /// The literal. + value: Literal, + }, +} + +/// Comparison operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum CompareOp { + /// Equal. + Eq, + /// Not equal. + Ne, + /// Less than. + Lt, + /// Less than or equal. + Le, + /// Greater than. + Gt, + /// Greater than or equal. + Ge, +} + +/// A boolean test over one row. +/// +/// Comparisons are typed: numbers compare numerically, so `amount > 0` means what it says +/// rather than comparing text. Any comparison involving `NULL` is false, including `NULL = NULL`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum Predicate { + /// All conditions hold. An empty list is true. + And { + /// Conditions. + conditions: Vec, + }, + /// Any condition holds. An empty list is false. + Or { + /// Conditions. + conditions: Vec, + }, + /// The condition does not hold. + Not { + /// The negated condition. + condition: Box, + }, + /// Compare two operands. + Compare { + /// Left side. + left: Operand, + /// Operator. + op: CompareOp, + /// Right side. + right: Operand, + }, + /// The column is `NULL` or absent. + IsNull { + /// Column name. + column: String, + }, + /// The column is `NULL`, absent, or renders as the empty string. + IsEmpty { + /// Column name. + column: String, + }, + /// The column's text (see [`Value::display_string`]) matches the regular expression. + Matches { + /// Column name. + column: String, + /// Regular expression. + regex: String, + }, + /// The column equals one of the listed literals. + In { + /// Column name. + column: String, + /// Accepted values. + values: Vec, + }, +} + +/// A [`Predicate`] with its regular expressions compiled, ready for repeated evaluation. +#[derive(Debug)] +pub(crate) struct PreparedPredicate { + kind: PreparedKind, +} + +#[derive(Debug)] +enum PreparedKind { + And(Vec), + Or(Vec), + Not(Box), + Compare { + left: PreparedOperand, + op: CompareOp, + right: PreparedOperand, + }, + IsNull { + column: String, + }, + IsEmpty { + column: String, + }, + Matches { + column: String, + regex: regex::Regex, + }, + In { + column: String, + values: Vec, + }, +} + +/// An [`Operand`] resolved once at [`Predicate::prepare`] time: a literal is coerced then, so +/// [`PreparedPredicate::evaluate`] only ever clones an already-typed [`Value`]. +#[derive(Debug)] +enum PreparedOperand { + Column(String), + Literal(Value), +} + +impl Predicate { + /// Compile this predicate's regular expressions once, ahead of row evaluation, and coerce + /// every literal (in a `Compare` or an `In`) to its column's declared type where `schema` + /// names one. + /// + /// # Literal coercion + /// + /// A blueprint's `Literal` is untagged JSON, so an editor's text input for `docstatus = 1` + /// emits `Literal::Text("1")`, which matches zero rows of an `INTEGER` column as-is. Each + /// literal position is therefore reinterpreted as the column's [`ValueKind`] via + /// [`Value::coerce_to`], once here instead of per row. A literal that does not parse as the + /// target kind is left as authored and never matches, as is every literal when `schema` is + /// `None`. + /// + /// This is part of the model's semantics: a SQL compiler must reproduce it at every literal + /// position, casting the literal to the column's SQL type or emitting an always-false + /// predicate when the cast is impossible. Otherwise the compiled view and this evaluator + /// disagree about which rows match. + /// + /// # Errors + /// Returns the underlying [`regex::Error`] if a `Matches` pattern does not compile. + pub(crate) fn prepare( + &self, + schema: Option<&TableSchema>, + ) -> Result { + let kind = match self { + Predicate::And { conditions } => PreparedKind::And( + conditions + .iter() + .map(|c| c.prepare(schema)) + .collect::>()?, + ), + Predicate::Or { conditions } => PreparedKind::Or( + conditions + .iter() + .map(|c| c.prepare(schema)) + .collect::>()?, + ), + Predicate::Not { condition } => PreparedKind::Not(Box::new(condition.prepare(schema)?)), + Predicate::Compare { left, op, right } => PreparedKind::Compare { + left: prepare_operand(left, column_kind(right, schema)), + op: *op, + right: prepare_operand(right, column_kind(left, schema)), + }, + Predicate::IsNull { column } => PreparedKind::IsNull { + column: column.clone(), + }, + Predicate::IsEmpty { column } => PreparedKind::IsEmpty { + column: column.clone(), + }, + Predicate::Matches { column, regex } => PreparedKind::Matches { + column: column.clone(), + regex: regex::Regex::new(regex)?, + }, + Predicate::In { column, values } => { + let kind = schema + .and_then(|s| s.columns.get(column)) + .and_then(ColumnSchema::declared_kind); + PreparedKind::In { + column: column.clone(), + values: values.iter().map(|v| prepare_literal(v, kind)).collect(), + } + } + }; + Ok(PreparedPredicate { kind }) + } + + /// Collect every column name this predicate reads into `out`. + pub fn referenced_columns<'a>(&'a self, out: &mut HashSet<&'a str>) { + match self { + Predicate::And { conditions } | Predicate::Or { conditions } => { + for c in conditions { + c.referenced_columns(out); + } + } + Predicate::Not { condition } => condition.referenced_columns(out), + Predicate::Compare { left, right, .. } => { + for side in [left, right] { + if let Operand::Column { column } = side { + out.insert(column); + } + } + } + Predicate::IsNull { column } + | Predicate::IsEmpty { column } + | Predicate::Matches { column, .. } + | Predicate::In { column, .. } => { + out.insert(column); + } + } + } +} + +impl PreparedPredicate { + /// Evaluate against one row. + pub(crate) fn evaluate(&self, row: &Row<'_>) -> bool { + match &self.kind { + PreparedKind::And(cs) => cs.iter().all(|c| c.evaluate(row)), + PreparedKind::Or(cs) => cs.iter().any(|c| c.evaluate(row)), + PreparedKind::Not(c) => !c.evaluate(row), + PreparedKind::Compare { left, op, right } => { + let (Some(l), Some(r)) = (resolve(left, row), resolve(right, row)) else { + return false; + }; + match l.compare(&r) { + Some(ord) => match op { + CompareOp::Eq => ord.is_eq(), + CompareOp::Ne => ord.is_ne(), + CompareOp::Lt => ord.is_lt(), + CompareOp::Le => ord.is_le(), + CompareOp::Gt => ord.is_gt(), + CompareOp::Ge => ord.is_ge(), + }, + None => false, + } + } + PreparedKind::IsNull { column } => row.get(column).is_none_or(Value::is_null), + PreparedKind::IsEmpty { column } => match row.get(column) { + None | Some(Value::Null) => true, + Some(v) => v.canonical_string().is_some_and(|s| s.is_empty()), + }, + PreparedKind::Matches { column, regex } => row + .get(column) + .and_then(Value::display_string) + .is_some_and(|s| regex.is_match(&s)), + PreparedKind::In { column, values } => match row.get(column) { + Some(v) => values + .iter() + .any(|c| v.compare(c).is_some_and(std::cmp::Ordering::is_eq)), + None => false, + }, + } + } +} + +/// Resolve a prepared operand against a row. A column absent from the row yields `None`. +fn resolve(operand: &PreparedOperand, row: &Row<'_>) -> Option { + match operand { + PreparedOperand::Column(column) => row.get(column).cloned(), + PreparedOperand::Literal(value) => Some(value.clone()), + } +} + +/// The declared [`ValueKind`] of `operand`, if it is an `Operand::Column` with an entry in +/// `schema`. `None` for a `Literal` operand, or for a `Column` whose type is not known. +fn column_kind(operand: &Operand, schema: Option<&TableSchema>) -> Option { + match operand { + Operand::Column { column } => schema + .and_then(|s| s.columns.get(column)) + .and_then(ColumnSchema::declared_kind), + Operand::Literal { .. } => None, + } +} + +/// Prepare one side of a `Compare`, coercing a `Literal` to `other_side_kind` (the other side's +/// declared column kind, if any) when it parses cleanly. See [`Predicate::prepare`] for the full +/// rule. +fn prepare_operand(operand: &Operand, other_side_kind: Option) -> PreparedOperand { + match operand { + Operand::Column { column } => PreparedOperand::Column(column.clone()), + Operand::Literal { value } => { + PreparedOperand::Literal(prepare_literal(value, other_side_kind)) + } + } +} + +/// Coerce `literal` to `kind`, when given, if it parses cleanly. Otherwise leave it exactly as +/// authored. +/// +/// The coercion rule shared by every literal position, in this evaluator and in the SQL a +/// compiled view carries. See [`Predicate::prepare`] for the full rule. +pub(crate) fn prepare_literal(literal: &Literal, kind: Option) -> Value { + let natural = literal.as_value(); + let coerced = kind.and_then(|kind| natural.coerce_to(kind)); + coerced.unwrap_or(natural) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::row::with_row; + use crate::core::event_data::object_centric::extraction::value::Value; + + fn col(name: &str) -> Operand { + Operand::Column { + column: name.to_string(), + } + } + + fn schema_of(columns: &[(&str, &str)]) -> TableSchema { + TableSchema::new( + "t", + columns + .iter() + .map(|&(name, col_type)| (name, col_type, true)), + ) + } + + #[test] + fn a_text_literal_is_coerced_to_match_an_integer_column() { + // The regression this replaces: a text-input editor emits Literal::Text("1"), which + // used to leave docstatus = 1 matching zero rows against an INTEGER column. + let schema = schema_of(&[("docstatus", "INTEGER")]); + let p = Predicate::Compare { + left: col("docstatus"), + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("1".into()), + }, + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + with_row(&[("docstatus", Value::Integer(1))], |row| { + assert!(prepared.evaluate(row)); + }); + } + + #[test] + fn a_text_iso8601_literal_is_coerced_and_orders_against_a_timestamp_column() { + let schema = schema_of(&[("created_at", "TIMESTAMPTZ")]); + let p = Predicate::Compare { + left: col("created_at"), + op: CompareOp::Gt, + right: Operand::Literal { + value: Literal::Text("2019-01-01T00:00:00Z".into()), + }, + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + let ts = chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap(); + with_row(&[("created_at", Value::Timestamp(ts))], |row| { + assert!(prepared.evaluate(row)); + }); + } + + #[test] + fn an_uncoercible_literal_matches_nothing_without_panicking() { + let schema = schema_of(&[("docstatus", "INTEGER")]); + let p = Predicate::Compare { + left: col("docstatus"), + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("abc".into()), + }, + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + with_row(&[("docstatus", Value::Integer(1))], |row| { + assert!(!prepared.evaluate(row)); + }); + } + + #[test] + fn an_unrecognised_col_type_leaves_the_literal_untouched() { + let schema = schema_of(&[("shape", "GEOMETRY")]); + let p = Predicate::Compare { + left: col("shape"), + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("1".into()), + }, + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + // Left uncoerced, the literal stays Text and never matches an Integer column value -- + // same outcome as an uncoercible literal, which is the point: an unknown col_type must + // not be guessed at. + with_row(&[("shape", Value::Integer(1))], |row| { + assert!(!prepared.evaluate(row)); + }); + with_row(&[("shape", Value::Text("1".into()))], |row| { + assert!(prepared.evaluate(row)); + }); + } + + #[test] + fn a_timestamp_literal_compares_directly_without_needing_coercion() { + let ts = chrono::DateTime::parse_from_rfc3339("2020-06-15T00:00:00Z").unwrap(); + let p = Predicate::Compare { + left: col("created_at"), + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Timestamp { timestamp: ts }, + }, + }; + let prepared = p.prepare(None).unwrap(); + with_row(&[("created_at", Value::Timestamp(ts))], |row| { + assert!(prepared.evaluate(row)); + }); + } + + #[test] + fn a_bare_json_string_is_not_swallowed_by_the_timestamp_variant() { + // Literal::Timestamp's object shape ({"timestamp": ...}) must never match a bare JSON + // string, regardless of where it sits among the untagged variants. + let text: Literal = serde_json::from_str(r#""2020-01-01T00:00:00Z""#).unwrap(); + assert_eq!(text, Literal::Text("2020-01-01T00:00:00Z".into())); + + let ts = chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap(); + let json = serde_json::to_string(&Literal::Timestamp { timestamp: ts }).unwrap(); + let back: Literal = serde_json::from_str(&json).unwrap(); + assert_eq!(back, Literal::Timestamp { timestamp: ts }); + } + + #[test] + fn compares_numbers_numerically_not_as_text() { + // "9" > "10" lexicographically; 9 < 10 numerically. The typed path must win. + let p = Predicate::Compare { + left: col("n"), + op: CompareOp::Gt, + right: Operand::Literal { + value: Literal::Integer(10), + }, + }; + with_row(&[("n", Value::Integer(9))], |row| { + assert!(!p.prepare(None).unwrap().evaluate(row)); + }); + } + + #[test] + fn compares_two_columns() { + // The change-log case: old <> new. + let p = Predicate::Compare { + left: col("old"), + op: CompareOp::Ne, + right: col("new"), + }; + let prepared = p.prepare(None).unwrap(); + with_row( + &[ + ("old", Value::Text("A".into())), + ("new", Value::Text("B".into())), + ], + |row| { + assert!(prepared.evaluate(row)); + }, + ); + with_row( + &[ + ("old", Value::Text("A".into())), + ("new", Value::Text("A".into())), + ], + |row| { + assert!(!prepared.evaluate(row)); + }, + ); + } + + #[test] + fn null_never_compares_equal_even_to_null() { + let p = Predicate::Compare { + left: col("a"), + op: CompareOp::Eq, + right: col("b"), + }; + with_row(&[("a", Value::Null), ("b", Value::Null)], |row| { + assert!(!p.prepare(None).unwrap().evaluate(row)); + }); + } + + #[test] + fn not_negates_and_empty_and_or_are_identity_and_absorbing() { + let t = Predicate::And { conditions: vec![] }; + let f = Predicate::Or { conditions: vec![] }; + with_row(&[("a", Value::Integer(1))], |row| { + assert!(t.prepare(None).unwrap().evaluate(row)); + assert!(!f.prepare(None).unwrap().evaluate(row)); + let n = Predicate::Not { + condition: Box::new(f.clone()), + }; + assert!(n.prepare(None).unwrap().evaluate(row)); + }); + } + + #[test] + fn is_null_and_is_empty_are_different_questions() { + let is_null = Predicate::IsNull { column: "a".into() }; + let is_empty = Predicate::IsEmpty { column: "a".into() }; + with_row(&[("a", Value::Null)], |row| { + assert!(is_null.prepare(None).unwrap().evaluate(row)); + assert!(is_empty.prepare(None).unwrap().evaluate(row)); + }); + with_row(&[("a", Value::Text(String::new()))], |row| { + assert!(!is_null.prepare(None).unwrap().evaluate(row)); + assert!(is_empty.prepare(None).unwrap().evaluate(row)); + }); + } + + #[test] + fn in_matches_any_listed_literal() { + let p = Predicate::In { + column: "t".into(), + values: vec![ + Literal::Text("out_invoice".into()), + Literal::Text("in_invoice".into()), + ], + }; + let prepared = p.prepare(None).unwrap(); + with_row(&[("t", Value::Text("in_invoice".into()))], |row| { + assert!(prepared.evaluate(row)) + }); + with_row(&[("t", Value::Text("entry".into()))], |row| { + assert!(!prepared.evaluate(row)) + }); + } + + #[test] + fn in_coerces_a_text_literal_to_match_an_integer_column() { + // Same regression as `a_text_literal_is_coerced_to_match_an_integer_column`, but for + // `In`: `docstatus IN ["1"]` authored the same way as `docstatus = 1` must behave the + // same way, not silently match zero rows. + let schema = schema_of(&[("docstatus", "INTEGER")]); + let p = Predicate::In { + column: "docstatus".into(), + values: vec![Literal::Text("1".into())], + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + with_row(&[("docstatus", Value::Integer(1))], |row| { + assert!(prepared.evaluate(row)); + }); + } + + #[test] + fn in_with_an_uncoercible_literal_matches_nothing_without_panicking() { + let schema = schema_of(&[("docstatus", "INTEGER")]); + let p = Predicate::In { + column: "docstatus".into(), + values: vec![Literal::Text("abc".into())], + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + with_row(&[("docstatus", Value::Integer(1))], |row| { + assert!(!prepared.evaluate(row)); + }); + } + + #[test] + fn in_coerces_each_literal_independently() { + // One uncoercible value in the list must not stop a coercible sibling from matching. + let schema = schema_of(&[("docstatus", "INTEGER")]); + let p = Predicate::In { + column: "docstatus".into(), + values: vec![Literal::Text("1".into()), Literal::Text("abc".into())], + }; + let prepared = p.prepare(Some(&schema)).unwrap(); + with_row(&[("docstatus", Value::Integer(1))], |row| { + assert!(prepared.evaluate(row), "the coercible literal must match"); + }); + with_row(&[("docstatus", Value::Integer(2))], |row| { + assert!(!prepared.evaluate(row), "neither literal matches 2"); + }); + } + + /// `"NaN".parse::()` succeeds, so a plain-JSON `Literal::Text("NaN")` against a + /// `DOUBLE` column coerces to `Value::Float(NaN)` and the emitter renders + /// `CAST('NaN' AS DOUBLE)`. `DuckDB` gives floats a total order, so `col = 'NaN'`, + /// `col > 1.0` and `col IN ('NaN')` are all true there, so `Value::compare` has to agree or + /// the extractor drops every row a compiled view keeps. + #[test] + fn compare_and_in_follow_sql_s_total_float_order_for_nan() { + let schema = schema_of(&[("amount", "DOUBLE")]); + let cmp = |op: CompareOp, lit: &str| { + Predicate::Compare { + left: Operand::Column { + column: "amount".into(), + }, + op, + right: Operand::Literal { + value: Literal::Text(lit.into()), + }, + } + .prepare(Some(&schema)) + .unwrap() + }; + with_row(&[("amount", Value::Float(f64::NAN))], |row| { + assert!(cmp(CompareOp::Eq, "NaN").evaluate(row), "NaN = NaN"); + assert!(!cmp(CompareOp::Ne, "NaN").evaluate(row), "NaN <> NaN"); + assert!(cmp(CompareOp::Gt, "1.0").evaluate(row), "NaN > 1.0"); + assert!(!cmp(CompareOp::Lt, "1.0").evaluate(row), "NaN < 1.0"); + + let in_nan = Predicate::In { + column: "amount".into(), + values: vec![Literal::Text("NaN".into())], + } + .prepare(Some(&schema)) + .unwrap(); + assert!(in_nan.evaluate(row), "NaN IN (NaN)"); + }); + with_row(&[("amount", Value::Float(1.0))], |row| { + assert!(cmp(CompareOp::Lt, "NaN").evaluate(row), "1.0 < NaN"); + assert!(!cmp(CompareOp::Gt, "NaN").evaluate(row), "1.0 > NaN"); + }); + } + + #[test] + fn matches_reads_a_timestamp_or_float_column_instead_of_matching_nothing() { + // `canonical_string` is `None` for `Float` and `Timestamp`, so reusing it here would make + // a `Matches` against those columns false for every row. + let ts = chrono::DateTime::parse_from_rfc3339("2020-02-03T04:05:06+02:00").unwrap(); + let p = Predicate::Matches { + column: "created_at".into(), + regex: "^2020".into(), + } + .prepare(None) + .unwrap(); + with_row(&[("created_at", Value::Timestamp(ts))], |row| { + assert!(p.evaluate(row)); + }); + + let p = Predicate::Matches { + column: "amount".into(), + regex: r"^1\.5$".into(), + } + .prepare(None) + .unwrap(); + with_row(&[("amount", Value::Float(1.5))], |row| { + assert!(p.evaluate(row)); + }); + } + + #[test] + fn prepare_reports_an_invalid_regex_instead_of_panicking() { + let p = Predicate::Matches { + column: "a".into(), + regex: "([".into(), + }; + assert!(p.prepare(None).is_err()); + } + + #[test] + fn referenced_columns_collects_from_every_variant() { + let p = Predicate::And { + conditions: vec![ + Predicate::Compare { + left: col("a"), + op: CompareOp::Eq, + right: col("b"), + }, + Predicate::Not { + condition: Box::new(Predicate::IsNull { column: "c".into() }), + }, + Predicate::In { + column: "d".into(), + values: vec![], + }, + Predicate::Matches { + column: "e".into(), + regex: ".".into(), + }, + ], + }; + let mut cols = HashSet::new(); + p.referenced_columns(&mut cols); + let mut got: Vec<&str> = cols.into_iter().collect(); + got.sort_unstable(); + assert_eq!(got, vec!["a", "b", "c", "d", "e"]); + } + + #[test] + fn an_empty_column_name_is_not_swallowed() { + // A dropped guard here used to hide `{"type":"column","column":""}`, which drops every + // row at evaluation time, from validation. + let p = Predicate::IsNull { + column: String::new(), + }; + let mut cols = HashSet::new(); + p.referenced_columns(&mut cols); + assert_eq!(cols, HashSet::from([""])); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/provider.rs b/process_mining/src/core/event_data/object_centric/extraction/provider.rs new file mode 100644 index 00000000..7d3e145e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/provider.rs @@ -0,0 +1,211 @@ +//! Pulling rows out of a data source, one table at a time. + +use std::fmt::Debug; +use std::ops::ControlFlow; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::compile::SqlDialect; +use super::value::Value; + +/// A source of rows for the tables a blueprint's [`Source`](super::blueprint::NodeOp::Source) +/// nodes name. +/// +/// `scan` is push-based: it calls `f` once per row rather than returning a collection, so a +/// `Source -> Filter` chain never holds more than one row. Implementations are expected to stream +/// from their backend the same way rather than collecting into a `Vec` first. +pub trait RowProvider: Debug { + /// Call `f` once for every row of `table`, restricted to `columns` and in that order. + /// + /// A column named in `columns` that the table does not have is an error, not a `Null` + /// fill-in: callers only ever ask for columns a [`Catalog`](super::catalog::Catalog) or a + /// prior successful call already confirmed exist. + /// + /// An empty `columns` is a legitimate request for zero-length rows, not a request for + /// every column: a mapping whose target names only constants still needs one call per row. + /// + /// A [`ControlFlow::Break`] from `f` must abandon the scan and return `Ok(())` rather than + /// reading the table to the end. + /// + /// # Errors + /// Returns [`ProviderError`] if `table` is unknown, a column in `columns` does not exist on + /// it, or the underlying source fails while reading. + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError>; + + /// The SQL dialect this provider can execute through [`Self::scan_query`], or `None` + /// (the default) if it cannot run SQL at all. + /// + /// A CSV- or API-backed provider says `None` here and is never handed a query. The answer is + /// a hint, not a promise: the executor falls back to row-level execution whenever a provider + /// that claimed a dialect answers [`ProviderError::QueryUnsupported`]. + /// + /// # Declaring a dialect is a statement about the catalog, not only about the engine + /// + /// The SQL handed to [`Self::scan_query`] comes from the same emitter + /// [`compile`](super::compile()) uses, which decides literal coercion, join-key comparability + /// and identity rendering from the [`Catalog`](super::catalog::Catalog)'s declared column + /// types. That reproduces this extractor's row-level semantics only where the source's runtime + /// values really have the declared kinds. A dynamically typed engine such as `SQLite`, which + /// stores a type per cell, must keep the default `None`, or a pushed-down join could return + /// different rows than the row-level path. + fn query_dialect(&self) -> Option { + None + } + + /// Run `sql` and call `f` once per result row, under the same push-based, + /// [`ControlFlow`]-honouring contract as [`Self::scan`]. + /// + /// `sql` is a single `SELECT` in this provider's [`Self::query_dialect`] whose result columns + /// are exactly `columns`, in that order. + /// + /// The default refuses with [`ProviderError::QueryUnsupported`], which is also the answer for + /// a query this provider happens not to be able to run. The caller carries on without it. + /// + /// # Errors + /// Returns [`ProviderError::QueryUnsupported`] if this provider cannot execute SQL (the + /// default), or [`ProviderError::Backend`] if the query itself failed. + fn scan_query( + &self, + sql: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError> { + let _ = (sql, columns, f); + Err(ProviderError::QueryUnsupported) + } +} + +/// Why a [`RowProvider::scan`] call failed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum ProviderError { + /// `table` has no entry in this provider. + UnknownTable { + /// The table name. + table: String, + }, + /// A requested column is not present on `table`. + UnknownColumn { + /// The table name. + table: String, + /// The column name. + column: String, + }, + /// This provider cannot execute SQL: it reports no [`RowProvider::query_dialect`], or the + /// query it was handed is one it cannot run. Not fatal: the caller falls back to executing + /// the node row by row. + QueryUnsupported, + /// The underlying source failed while reading. + Backend { + /// The table being read when the failure happened. + table: String, + /// The backend's error message. + message: String, + }, +} + +impl std::fmt::Display for ProviderError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProviderError::UnknownTable { table } => write!(f, "unknown table '{table}'"), + ProviderError::UnknownColumn { table, column } => { + write!(f, "table '{table}' has no column '{column}'") + } + ProviderError::QueryUnsupported => { + write!(f, "this provider cannot execute SQL queries") + } + ProviderError::Backend { table, message } => { + write!(f, "reading '{table}' failed: {message}") + } + } + } +} + +impl std::error::Error for ProviderError {} + +/// The first `limit` rows of `table`, restricted to `columns`, for showing a person what the +/// data looks like while they build a blueprint against it. +/// +/// Built on [`RowProvider::scan`], so it stops row transfer by breaking out of the scan rather +/// than bounding the query. A backend that can express the bound itself will do better. +/// +/// A `NULL` cell comes back as `None`, distinct from an empty string. +/// +/// # Errors +/// Returns whatever [`RowProvider::scan`] reports for an unknown table or column. +pub fn preview_rows( + provider: &dyn RowProvider, + table: &str, + columns: &[&str], + limit: usize, +) -> Result { + let mut rows: Vec>> = Vec::new(); + if limit > 0 { + provider.scan(table, columns, &mut |values| { + rows.push(values.iter().map(Value::display_string).collect()); + if rows.len() >= limit { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })?; + } + Ok(super::catalog::TablePreview { + columns: columns.iter().map(|c| (*c).to_string()).collect(), + rows, + }) +} + +/// Every distinct non-null value of `table.column`, as text, for populating +/// [`ExtractionCatalog::with_domain`](super::catalog::ExtractionCatalog::with_domain). +/// +/// This reads the whole column, because a partial answer would silently drop rows: the compiler +/// lowers a dynamic type name from the domain. A provider whose backend has `SELECT DISTINCT` +/// should prefer that. See +/// [`DbconRowProvider::distinct_values`](super::DbconRowProvider::distinct_values). +/// +/// Values come back in first-seen order. +/// +/// # Errors +/// Returns whatever [`RowProvider::scan`] reports for an unknown table or column. +pub fn distinct_column_values( + provider: &dyn RowProvider, + table: &str, + column: &str, +) -> Result, ProviderError> { + let mut seen = std::collections::HashSet::new(); + let mut values = Vec::new(); + provider.scan(table, &[column], &mut |row| { + if let Some(v) = row.first().and_then(Value::display_string) { + if seen.insert(v.clone()) { + values.push(v); + } + } + ControlFlow::Continue(()) + })?; + Ok(values) +} + +/// The [`ProviderError`] a `SQLite` error message names, if it names one. +/// +/// `SQLite` distinguishes a missing table from a missing column only in its message text, so every +/// provider reading a `SQLite` file has to parse it. Shared so they cannot disagree. +pub(crate) fn sqlite_message_error(table: &str, message: &str) -> Option { + if message.contains("no such table") { + return Some(ProviderError::UnknownTable { + table: table.to_string(), + }); + } + message + .split("no such column: ") + .nth(1) + .map(|column| ProviderError::UnknownColumn { + table: table.to_string(), + column: column.to_string(), + }) +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/pushdown.rs b/process_mining/src/core/event_data/object_centric/extraction/pushdown.rs new file mode 100644 index 00000000..b9571c35 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/pushdown.rs @@ -0,0 +1,94 @@ +//! Deciding when a node can be executed by its source instead of row by row, and asking the +//! step-3 compiler to write the SQL that does it. +//! +//! A [`Join`](super::blueprint::NodeOp::Join) is the one node +//! [`GraphExecutor`](super::graph::GraphExecutor) cannot execute in constant memory. When both +//! inputs read the same source, the engine already has both tables and can do the join itself. +//! +//! The SQL comes from [`Emitter`], the compiler's own emitter, so that a memory optimisation +//! cannot change results by drifting from it. [`Emitter::node_sql`] projects a node's full column +//! set while the executor's rows carry only the demanded subset (see [`super::schema`]), so the +//! query is wrapped in a projection naming exactly the executor's columns in its order. +//! +//! The emitter decides everything from declared column types, so its SQL matches the row-level +//! path only when the source's runtime values have the kinds the catalog declares. A provider +//! asserts that by returning a dialect from +//! [`RowProvider::query_dialect`](super::provider::RowProvider::query_dialect); a dynamically +//! typed source keeps the default `None` and is never pushed down to. + +use std::collections::HashMap; + +use super::blueprint::{Blueprint, NodeOp}; +use super::catalog::TableSchema; +use super::compile::emit::{Emitter, ROW_ALIAS}; +use super::compile::{RejectReason, SqlDialect}; + +/// The one `source_id` every [`Source`](super::blueprint::NodeOp::Source) leaf under `node_id` +/// reads, or `None` when the leaves disagree, a node is missing, or the graph cycles. +/// +/// `Filter`, `Union` and `Join` are all transparent here: a filter inherits its input's answer, +/// and a union or join inherits its inputs' only when they agree. So a join over two filtered +/// tables of one database is pushable, and the same join over a database and a CSV file is not. +pub(crate) fn single_source<'a>(blueprint: &'a Blueprint, node_id: &str) -> Option<&'a str> { + walk(blueprint, node_id, 0) +} + +fn walk<'a>(blueprint: &'a Blueprint, node_id: &str, depth: usize) -> Option<&'a str> { + // A validated blueprint is acyclic, but this runs on unvalidated ones too (the executor's + // own tests build them deliberately); no path can visit more nodes than exist. + if depth > blueprint.nodes.len() { + return None; + } + match &blueprint.node(node_id)?.op { + NodeOp::Source { source_id, .. } => Some(source_id.as_str()), + NodeOp::Filter { input, .. } => walk(blueprint, input, depth + 1), + NodeOp::Union { inputs } => { + let mut it = inputs.iter(); + let first = walk(blueprint, it.next()?, depth + 1)?; + it.all(|i| walk(blueprint, i, depth + 1) == Some(first)) + .then_some(first) + } + NodeOp::Join { left, right, .. } => { + let l = walk(blueprint, left, depth + 1)?; + (walk(blueprint, right, depth + 1) == Some(l)).then_some(l) + } + } +} + +/// A `SELECT` producing `node_id`'s rows as the executor expects them: exactly `columns`, in +/// that order, under those names. +/// +/// `full` is [`full_node_schemas`](super::schema::full_node_schemas)' result, which the executor +/// already holds. Passing it avoids resolving every node against the catalog a second time. +/// +/// `Err` when the emitter declines the node (an unsupported predicate, a join key whose declared +/// type does not decide comparability, an unresolved table) or when `columns` is empty. Declining +/// is safe, since the caller then executes the node row by row, but the fall-back is `hash_join`, +/// whose memory grows with the data, so the reason is carried out rather than discarded. +pub(crate) fn node_query_or_reason<'a>( + blueprint: &'a Blueprint, + full: &HashMap<&'a str, TableSchema>, + node_id: &str, + columns: &[String], + dialect: SqlDialect, +) -> Result { + if columns.is_empty() { + return Err(RejectReason::EmptyProjection { + node: node_id.to_string(), + }); + } + let emitter = Emitter::from_schemas(blueprint, full, dialect); + let inner = emitter.node_sql(node_id)?; + let list: Vec = columns + .iter() + .map(|c| { + let quoted = dialect.quote_ident(c); + format!("{ROW_ALIAS}.{quoted} AS {quoted}") + }) + .collect(); + Ok(format!( + "SELECT {} FROM {}", + list.join(", "), + dialect.derived_table(&inner, ROW_ALIAS) + )) +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/report.rs b/process_mining/src/core/event_data/object_centric/extraction/report.rs new file mode 100644 index 00000000..ad7900db --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/report.rs @@ -0,0 +1,461 @@ +//! What [`extract`](super::extract::extract) produces alongside the OCEL: what ran, and every +//! row it could not use. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::compile::RejectReason; +use super::provider::ProviderError; +use super::sink::{FinalizeReport, SinkError}; +use super::validate::ValidationError; +use crate::core::event_data::object_centric::OCELAttributeType; + +/// Points a diagnostic back at the mapping it came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct MappingRef { + /// Position in the desugared, flattened mapping list this run executed. + pub index: usize, + /// The mapping's own label, if it has one. + pub label: Option, + /// The JSON path of the authored entry this mapping came from (see `desugar_with_paths`), so + /// a diagnostic points at what the author wrote rather than a position in the flattened + /// list. + pub path: String, + /// What this mapping produces, derived from its target: `event "appoint officer"`, + /// `event -> object relation`, and so on. Present whether or not a `label` was typed. + pub describes: String, +} + +impl MappingRef { + /// Build a reference to `mapping`, describing it from its target. + #[must_use] + pub(crate) fn new(index: usize, path: String, mapping: &super::blueprint::Mapping) -> Self { + Self { + index, + label: mapping.label.clone(), + path, + describes: describe_target(&mapping.target), + } + } + + /// The author's label if there is one, else the derived description. Prefer this over `path` + /// when rendering a diagnostic. + #[must_use] + pub fn title(&self) -> &str { + self.label.as_deref().unwrap_or(&self.describes) + } +} + +fn describe_target(target: &super::blueprint::Target) -> String { + use super::blueprint::Target; + use super::expr::ValueExpression; + // Only a constant names a type the reader can match against the canvas. + fn named(e: &ValueExpression) -> Option<&str> { + match e { + ValueExpression::Constant { value } if !value.is_empty() => Some(value.as_str()), + _ => None, + } + } + match target { + Target::Event { event_type, .. } => { + named(event_type).map_or_else(|| "event".to_string(), |t| format!("event \"{t}\"")) + } + Target::Object { object_type, .. } => { + named(object_type).map_or_else(|| "object".to_string(), |t| format!("object \"{t}\"")) + } + Target::E2O { object, .. } => object.object_type.as_ref().and_then(named).map_or_else( + || "event -> object relation".to_string(), + |t| format!("event -> \"{t}\" relation"), + ), + Target::O2O { source, target, .. } => match ( + source.object_type.as_ref().and_then(named), + target.object_type.as_ref().and_then(named), + ) { + (Some(s), Some(t)) => format!("\"{s}\" -> \"{t}\" relation"), + _ => "object -> object relation".to_string(), + }, + } +} + +/// Why one row a mapping read produced nothing. +/// +/// Does not include a repeated object id at event grain, see [`MappingStats::deduplicated`]. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, +)] +pub enum DropReason { + /// A relation named an event or object id that could not be resolved, under + /// [`MissingEndpointPolicy::Drop`](super::blueprint::MissingEndpointPolicy::Drop). + UnresolvedEndpoint, + /// A timestamp expression produced text, and that text did not parse: the format is wrong. + UnparseableTimestamp, + /// The row had no timestamp to parse: the column was `NULL`, or the expression produced + /// nothing (or only blank text). + /// + /// For a [`TimestampSource::Components`](super::expr::TimestampSource::Components) pair the + /// date side decides, whatever the time side holds: a time of day with no date does not + /// name an instant. + MissingTimestamp, + /// An id expression evaluated to `Null` or to a value with no + /// [`canonical_string`](super::value::Value::canonical_string). + NullOrUnrenderableId, + /// The mapping's `when` excluded the row. + PredicateExcluded, + /// The row named an entity whose id is already taken by an entity of a different type. + /// Only reachable under [`IdRendering::Raw`](super::blueprint::IdRendering::Raw), where two + /// types can render the same id. `TypePrefixed` makes it impossible. Deliberately not + /// [`MappingStats::deduplicated`]: nothing was deduplicated, two distinct entities collided. + IdTypeCollision, + /// The row named an object the sink already had and + /// [`DuplicateObjectPolicy::Error`](super::blueprint::DuplicateObjectPolicy::Error) is in + /// force, so none of the row's attributes were written onto it. The same repeat counts as + /// [`MappingStats::deduplicated`] under `FirstWins`, where it is not a loss. + DuplicateObjectRejected, +} + +/// Counts for one mapping's run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct MappingStats { + /// Which mapping. + pub mapping: MappingRef, + /// Rows the mapping's node produced, before `when` was applied. + pub rows_read: u64, + /// Entities or relations this mapping handed to the sink, which is not the same as + /// "survived the run" for a sink that defers resolution. + /// + /// An eager sink refuses a dangling relation at the call site, so it lands in + /// [`DropReason::UnresolvedEndpoint`] and never here. A deferring sink writes it, counts it + /// here, and deletes it at [`finalize`](super::sink::ExtractionSink::finalize), reporting it + /// in [`FinalizeReport::unresolved_endpoints`](super::sink::FinalizeReport) instead. To count + /// what a run produced, subtract `ExtractionReport::finalize.unresolved_endpoints` from the + /// total. + pub entities_emitted: u64, + /// Rows that tried to create an entity the sink already had. Not a loss: an object mapping + /// at event grain names the same object on every row by design. See + /// [`DuplicateObjectPolicy::Error`](super::blueprint::DuplicateObjectPolicy::Error) for what + /// turns a repeat into a loss instead. + /// + /// One increment per row whose entity-creating call found the entity already present, across + /// mappings, since the sink is what answers. + /// + /// Resolving a relation endpoint is never counted, so an `E2O`/`O2O` mapping reports zero + /// however often its rows repeat an id: finding an existing endpoint is the normal successful + /// case. A blueprint that wants its inline references' repeats counted can name the objects + /// with their own [`Target::Object`](super::blueprint::Target::Object) mapping. + pub deduplicated: u64, + /// Rows dropped, by reason. A row that matches several reasons at once (rare) is counted + /// once, under the first one detected. + pub dropped: BTreeMap, + /// Attribute values that would not convert to their attribute's declared type, stored as + /// `Null`. Not a dropped row: the entity was written, with one of its attributes empty. + #[serde(default)] + pub uncoercible_attributes: u64, +} + +impl MappingStats { + /// Zeroed stats for `mapping`. + #[must_use] + pub(crate) fn new(mapping: MappingRef) -> Self { + Self { + mapping, + rows_read: 0, + entities_emitted: 0, + deduplicated: 0, + dropped: BTreeMap::new(), + uncoercible_attributes: 0, + } + } + + /// Increment `reason`'s count by one. + pub(crate) fn drop(&mut self, reason: DropReason) { + *self.dropped.entry(reason).or_insert(0) += 1; + } +} + +/// The most [`ExtractionError`]s one run keeps. Past this only `ErrorLog::suppressed` grows. +/// +/// Capped because a policy configured to error reports one error per offending row, making this +/// the only per-run structure whose size is a function of the data rather than of the blueprint. +pub const MAX_REPORTED_ERRORS: usize = 1000; + +/// A bounded [`ExtractionError`] collector: the first [`MAX_REPORTED_ERRORS`] are kept in full, +/// the rest only counted. +#[derive(Debug, Default)] +pub(crate) struct ErrorLog { + errors: Vec, + suppressed: u64, +} + +impl ErrorLog { + /// An empty log. + pub(crate) fn new() -> Self { + Self::default() + } + + /// Record `error`, or count it as suppressed once the cap is reached. + pub(crate) fn push(&mut self, error: ExtractionError) { + if self.errors.len() < MAX_REPORTED_ERRORS { + self.errors.push(error); + } else { + self.suppressed += 1; + } + } + + /// The errors kept, and how many were not. + pub(crate) fn into_parts(self) -> (Vec, u64) { + (self.errors, self.suppressed) + } +} + +/// One node the compiler refused to push down to its source, and why. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct PushdownDeclined { + /// The node id, as the blueprint names it. + pub node: String, + /// What the compiler objected to. + pub reason: RejectReason, +} + +/// What [`extract`](super::extract::extract) produced, beyond the OCEL itself. +/// +/// Serializable but not deserializable: [`ExtractionError`] carries `&'static str` fields (a +/// borrow no deserializer can manufacture), so this only ever crosses a bindings boundary +/// outbound, as a `#[register_binding]` return value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct ExtractionReport { + /// One entry per mapping executed, in desugared blueprint order: the order the author wrote + /// the mappings in, with each ordered group expanded in place. Not execution order, + /// which is multi-pass and grouped by node. Each entry's [`MappingRef::path`] names the + /// authored entry. + pub per_mapping: Vec, + /// Non-fatal problems collected while running: a policy configured to error + /// (`on_duplicate_object: Error`, `on_missing_endpoint: Error`), or an attribute type two + /// mappings disagreed on. Extraction continues past these. See [`ExtractionError`] for what + /// aborts it instead. + /// + /// Capped at [`MAX_REPORTED_ERRORS`], with the remainder counted in + /// [`errors_suppressed`](Self::errors_suppressed). + pub errors: Vec, + /// Non-fatal problems the run hit past [`MAX_REPORTED_ERRORS`], which + /// [`errors`](Self::errors) therefore does not name. Zero for every run under the cap. + pub errors_suppressed: u64, + /// Rows every `Join`/`Union` materialisation this run performed produced, summed across + /// materialisations rather than peaked: an upper bound on peak buffered rows. A cached + /// materialisation is counted once, when computed. + /// + /// Zero when no mapping's node graph contains a `Join` or `Union`, since a pure + /// `Source -> Filter` chain streams. Zero is therefore a witness that the run streamed. + pub rows_materialized: u64, + /// Nodes whose source could have executed the whole node but the compiler declined to build + /// the query for, paired with the reason, and deduplicated per node. + /// + /// Always safe, since the executor runs the node itself. Reported because falling back on a + /// `Join` is the one execution path whose memory grows with the data, so this explains a + /// non-zero [`rows_materialized`](Self::rows_materialized). + pub pushdown_declined: Vec, + /// What the sink did at [`ExtractionSink::finalize`](super::sink::ExtractionSink::finalize). + /// + /// All zero for a sink that resolves relation endpoints eagerly, which reports everything + /// through [`per_mapping`](Self::per_mapping) instead. See + /// [`Resolution`](super::sink::Resolution). + pub finalize: FinalizeReport, + /// Where the run's wall-clock time went. + /// + /// `None` from [`extract`](super::extract::extract) itself, which is handed open providers + /// and cannot know what they cost to obtain. The runner that owns the connections fills this + /// in, as the `extraction-dbcon` bindings do. Also kept out of `extract` because + /// `std::time::Instant` panics on `wasm32-unknown-unknown`. + #[serde(skip_serializing_if = "Option::is_none")] + pub timing: Option, +} + +/// How long a run spent, split by phase, in milliseconds. Schema discovery is a fixed cost a +/// caller holding a catalog can skip, so it is reported apart from the row reading. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, JsonSchema)] +pub struct ExtractionTiming { + /// Connecting to each source and reading its schema. Zero when the caller supplied a catalog. + pub discovery_ms: u64, + /// Reading rows and emitting entities: `extract` itself. + pub extraction_ms: u64, +} + +/// Why [`extract`](super::extract::extract) could not run at all, or a non-fatal problem +/// recorded in [`ExtractionReport::errors`] while it did. +/// +/// Serializable but not deserializable: some variants carry `&'static str` fields (a borrow no +/// deserializer can manufacture), so this only ever crosses a bindings boundary outbound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub enum ExtractionError { + /// The blueprint failed [`validate`](super::validate::validate), so `extract` refuses to run + /// it. Fatal. + Invalid(Vec), + /// A `Source` node names a `source_id` with no entry in the `providers` map `extract` was + /// given. Fatal. + MissingProvider { + /// The missing source id. + source_id: String, + }, + /// A predicate's or split's regular expression failed to compile. Fatal. `validate` checks + /// this, so it should not happen when `extract` is called on a validated blueprint. + InvalidRegex { + /// The offending pattern. + pattern: String, + /// The compiler's message. + message: String, + }, + /// A `Join`'s `on` clause named a key column that its input's rows do not carry. Fatal: + /// dropping the column from the key instead would shorten it, silently turning the join into + /// a partial cross product. + JoinKeyColumnMissing { + /// The `Join` node. + node: String, + /// `"left"` or `"right"`. + side: &'static str, + /// The key column, as named on that side. + column: String, + }, + /// A [`RowProvider`](super::provider::RowProvider) call failed. Fatal: the rows a mapping + /// needs are simply not available. + Provider { + /// The node being read when the failure happened. + node: String, + /// The underlying error. + source: ProviderError, + }, + /// An [`ExtractionSink`](super::sink::ExtractionSink) call failed for a reason other than a + /// policy below. Fatal: this is the storage layer failing, and carrying on would leave a + /// half-written OCEL that reports success. + Sink { + /// What was being added when the failure happened. + context: String, + /// The underlying error. + source: SinkError, + }, + /// Two mappings, or two rows, produced the same rendered id under two different types. + /// Only reachable under [`IdRendering::Raw`](super::blueprint::IdRendering::Raw). Non-fatal: + /// the row is dropped (see [`DropReason::IdTypeCollision`]) and the run continues, since + /// merging the two would fold two distinct entities into one. + IdTypeCollision { + /// The mapping whose row collided. + mapping: MappingRef, + /// The contested id. + id: String, + /// The type this row wanted the id for. The type that already holds it is whatever the + /// sink reports for that id. + requested_type: String, + }, + /// Two mappings declared the same attribute of the same entity type under different value + /// types. Non-fatal: the declaration is widened to a type covering both (see + /// [`OCELAttributeType::coalesce`](crate::core::event_data::object_centric::OCELAttributeType::coalesce)) + /// and every row converted to it. Reported because the resulting type is then decided by a + /// coincidence of two mappings rather than by either author's intent. Names no mapping: the + /// conflict is a property of the pair. + ConflictingAttributeType { + /// `"event"` or `"object"`. + kind: &'static str, + /// The entity type. + type_name: String, + /// The attribute. + attribute: String, + /// The type it was declared with first. + declared: OCELAttributeType, + /// The type the later declaration gave it. + conflicting: OCELAttributeType, + }, + /// `on_duplicate_object: Error` fired: `id` had already been added by `mapping`. Non-fatal. + DuplicateObject { + /// The mapping whose row named the repeat. + mapping: MappingRef, + /// The repeated id. + id: String, + }, + /// `on_missing_endpoint: Error` fired: `endpoint` named `id`, which could not be resolved. + /// Non-fatal. + MissingEndpoint { + /// The mapping whose row named the endpoint. + mapping: MappingRef, + /// Which endpoint (`"event"`, `"object"`, `"source"`, `"target"`, ...). + endpoint: &'static str, + /// The unresolved id. + id: String, + }, + /// `on_missing_endpoint: Error` fired for a sink that answered + /// [`Resolution::Deferred`](super::sink::Resolution::Deferred): the same policy violation as + /// [`MissingEndpoint`](Self::MissingEndpoint), reported once for the whole run instead of + /// once per endpoint. Non-fatal. + /// + /// Such a sink detects the violation at [`finalize`](super::sink::ExtractionSink::finalize), + /// where the mapping and row that named the endpoint are gone, so this names neither. + MissingEndpointsAtFinalize { + /// How many relations the sink could not resolve. Usually equals an eager sink's + /// [`MissingEndpoint`](Self::MissingEndpoint) count for the same run, but can exceed it: + /// a deferring sink also stages the inline references of a `Target::Event` whose event + /// this run dropped, which an eager sink never asks about. + count: u64, + }, +} + +impl std::fmt::Display for ExtractionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExtractionError::Invalid(errs) => write!(f, "blueprint did not validate: {errs:?}"), + ExtractionError::MissingProvider { source_id } => { + write!(f, "no provider registered for source '{source_id}'") + } + ExtractionError::InvalidRegex { pattern, message } => { + write!(f, "invalid regular expression '{pattern}': {message}") + } + ExtractionError::JoinKeyColumnMissing { node, side, column } => write!( + f, + "join '{node}': {side} input has no key column '{column}'" + ), + ExtractionError::Provider { node, source } => { + write!(f, "reading node '{node}' failed: {source}") + } + ExtractionError::Sink { context, source } => { + write!(f, "{context}: {source}") + } + ExtractionError::ConflictingAttributeType { + kind, + type_name, + attribute, + declared, + conflicting, + } => write!( + f, + "{kind} type '{type_name}': attribute '{attribute}' declared as '{}' and as '{}'", + declared.to_type_string(), + conflicting.to_type_string() + ), + ExtractionError::IdTypeCollision { + mapping, + id, + requested_type, + } => write!( + f, + "mapping {}: id '{id}' is already taken by an entity of another type, \ + so no '{requested_type}' could take it", + mapping.title() + ), + ExtractionError::DuplicateObject { mapping, id } => { + write!(f, "mapping {}: duplicate object id '{id}'", mapping.title()) + } + ExtractionError::MissingEndpoint { + mapping, + endpoint, + id, + } => write!( + f, + "mapping {}: unresolved {endpoint} '{id}'", + mapping.title() + ), + ExtractionError::MissingEndpointsAtFinalize { count } => { + write!(f, "{count} relation(s) had an endpoint that never resolved") + } + } + } +} + +impl std::error::Error for ExtractionError {} diff --git a/process_mining/src/core/event_data/object_centric/extraction/row.rs b/process_mining/src/core/event_data/object_centric/extraction/row.rs new file mode 100644 index 00000000..6a42e254 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/row.rs @@ -0,0 +1,67 @@ +//! Column-indexed access to a row of [`Value`]s. +//! +//! Internal to the module: rows are a processing detail, not part of the blueprint's shape. + +use std::collections::HashMap; + +use super::value::Value; + +/// Maps a column name to its position in a row. +pub(crate) type ColumnIndex<'a> = HashMap<&'a str, usize>; + +/// One row, addressable by column name. +#[derive(Debug)] +pub(crate) struct Row<'a> { + /// Cell values, positionally aligned with `index`. + pub(crate) values: &'a [Value], + /// Column name to position. + pub(crate) index: &'a ColumnIndex<'a>, +} + +impl<'a> Row<'a> { + /// The value in `column`, or `None` if the row has no such column. + pub(crate) fn get(&self, column: &str) -> Option<&'a Value> { + self.index.get(column).and_then(|&i| self.values.get(i)) + } +} + +/// Build a [`ColumnIndex`] from an ordered column list. +pub(crate) fn build_column_index<'a>(columns: &[&'a str]) -> ColumnIndex<'a> { + columns + .iter() + .enumerate() + .map(|(i, &name)| (name, i)) + .collect() +} + +/// Build a [`Row`] from name/value pairs and hand it to `f`. +/// +/// Test-only: a `Row` borrows its values and index, so tests would otherwise repeat three +/// lines of setup each. +#[cfg(test)] +pub(crate) fn with_row(pairs: &[(&str, Value)], f: impl FnOnce(&Row<'_>) -> R) -> R { + let names: Vec<&str> = pairs.iter().map(|(n, _)| *n).collect(); + let values: Vec = pairs.iter().map(|(_, v)| v.clone()).collect(); + let index = build_column_index(&names); + f(&Row { + values: &values, + index: &index, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_returns_the_value_and_none_for_unknown_columns() { + with_row( + &[("a", Value::Integer(1)), ("b", Value::Text("x".into()))], + |row| { + assert_eq!(row.get("a"), Some(&Value::Integer(1))); + assert_eq!(row.get("b"), Some(&Value::Text("x".into()))); + assert_eq!(row.get("missing"), None); + }, + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/schema.rs b/process_mining/src/core/event_data/object_centric/extraction/schema.rs new file mode 100644 index 00000000..cea8849c --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/schema.rs @@ -0,0 +1,416 @@ +//! Per-node column resolution: what a node's rows look like, and which of its columns are actually +//! read anywhere downstream. +//! +//! Internal to the executor. One traversal produces both the full column shape of every node +//! (needed for [`Predicate::prepare`](super::predicate::Predicate::prepare)'s literal coercion and +//! to disambiguate a `Join`'s `right_` columns) and the subset of each node's columns +//! anything downstream reads, so a `Source` node only asks its +//! [`RowProvider`](super::provider::RowProvider) for what a mapping or `Filter` will use. +//! +//! Demand flows downward everywhere except a `Filter`, whose demand must equal its input's: +//! [`GraphExecutor::stream`](super::graph::GraphExecutor::stream) hands a filter's consumers the +//! input's row verbatim, so a narrower demand would shift every position after a dropped column. + +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, HashMap, HashSet}; + +use super::blueprint::{Blueprint, Mapping, Node, NodeOp}; +use super::catalog::{Catalog, ColumnSchema, TableSchema, UNTYPED_COL_TYPE}; + +/// The full, statically known column shape of every node, keyed by node id. +/// +/// The single resolution of a node's columns: `validate` derives the names it checks against from +/// this, so a blueprint cannot pass validation against one set of columns and run against another. +pub(crate) fn full_node_schemas<'a>( + blueprint: &'a Blueprint, + catalog: &dyn Catalog, +) -> HashMap<&'a str, TableSchema> { + let mut out: HashMap<&str, TableSchema> = HashMap::new(); + // Nodes may appear in any order, so a node whose inputs are not resolved yet is retried on a + // later pass. + let mut changed = true; + while changed { + changed = false; + for node in &blueprint.nodes { + if out.contains_key(node.id.as_str()) { + continue; + } + if let Some(schema) = resolve_one(node, &out, catalog) { + out.insert(node.id.as_str(), schema); + changed = true; + } + } + } + out +} + +fn resolve_one( + node: &Node, + out: &HashMap<&str, TableSchema>, + catalog: &dyn Catalog, +) -> Option { + match &node.op { + NodeOp::Source { source_id, table } => catalog.table(source_id, table).cloned(), + NodeOp::Filter { input, .. } => out.get(input.as_str()).cloned(), + NodeOp::Union { inputs } => { + let mut columns: BTreeMap = BTreeMap::new(); + let mut declared_by: BTreeMap = BTreeMap::new(); + for input in inputs { + let s = out.get(input.as_str())?; + for (name, col) in &s.columns { + *declared_by.entry(name.clone()).or_insert(0) += 1; + match columns.entry(name.clone()) { + Entry::Vacant(slot) => { + slot.insert(col.clone()); + } + Entry::Occupied(mut slot) => reconcile_union_column(slot.get_mut(), col), + } + } + } + // A column only some inputs declare is Null on every row a non-declaring input + // contributes, regardless of what each declaring input says about nullability. + for (name, col) in &mut columns { + if declared_by[name] < inputs.len() { + col.nullable = true; + } + } + Some(TableSchema { + name: node.id.clone(), + columns, + }) + } + NodeOp::Join { left, right, .. } => { + let l = out.get(left.as_str())?; + let r = out.get(right.as_str())?; + let mut columns = l.columns.clone(); + for (name, col) in &r.columns { + if l.columns.contains_key(name) { + let renamed = format!("right_{name}"); + columns.insert( + renamed.clone(), + ColumnSchema { + name: renamed, + col_type: col.col_type.clone(), + nullable: col.nullable, + }, + ); + } else { + columns.insert(name.clone(), col.clone()); + } + } + Some(TableSchema { + name: node.id.clone(), + columns, + }) + } + } +} + +/// Fold a second input's declaration of one `Union` column into the first input's. +/// +/// Inputs disagreeing about a column's kind leave it [`UNTYPED_COL_TYPE`] rather than picking a +/// side, which would coerce a guard literal to a kind half the rows do not have. +fn reconcile_union_column(into: &mut ColumnSchema, other: &ColumnSchema) { + if into.declared_kind() != other.declared_kind() { + into.col_type = UNTYPED_COL_TYPE.to_string(); + } + into.nullable |= other.nullable; +} + +/// Which side of a `Join` one of its output columns comes from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum JoinSide { + /// The join's left input. + Left, + /// The join's right input. + Right, +} + +/// Resolve one `Join` output column name to the input it comes from and the name it has there. +/// +/// The single rule both this module (deciding which input to demand a column from) and +/// [`GraphExecutor`](super::graph::GraphExecutor) (deciding which input row to read it out of) +/// use, so the two cannot disagree. The precedence mirrors how [`full_node_schemas`] builds a +/// `Join`'s schema: left columns first, then each right column either under its own name or, +/// where the left already has that name, under `right_`. +/// +/// `None` means the name resolves to no single column: neither side has one, or both a left +/// column literally called `right_` and the rename of the right's `` claim it. +pub(crate) fn join_column_source<'a>( + name: &'a str, + left: &TableSchema, + right: &TableSchema, +) -> Option<(JoinSide, &'a str)> { + if let Some(stripped) = name.strip_prefix("right_") { + if left.columns.contains_key(stripped) && right.columns.contains_key(stripped) { + if left.columns.contains_key(name) { + return None; + } + return Some((JoinSide::Right, stripped)); + } + } + if left.columns.contains_key(name) { + return Some((JoinSide::Left, name)); + } + if right.columns.contains_key(name) { + return Some((JoinSide::Right, name)); + } + None +} + +/// The columns each node must produce: the union of what a mapping reading it needs, what a +/// downstream `Filter`/`Join` condition needs, and (recursively) what a downstream node's own +/// demand requires of it. A `Source` node's entry becomes the `columns` argument to +/// [`RowProvider::scan`](super::provider::RowProvider::scan). +/// +/// `full` is [`full_node_schemas`]'s result, needed to resolve which side of a `Join` a demanded +/// column (possibly `right_`-prefixed) actually belongs to, see [`join_column_source`]. A +/// `Filter`'s demand equals its input's rather than being narrower, see this module's docs. +pub(crate) fn demanded_columns<'a>( + blueprint: &'a Blueprint, + full: &HashMap<&str, TableSchema>, + mappings: &[(String, Mapping)], +) -> HashMap<&'a str, HashSet> { + let mut demand: HashMap<&str, HashSet> = blueprint + .nodes + .iter() + .map(|n| (n.id.as_str(), HashSet::new())) + .collect(); + + for (_, m) in mappings { + let Some(entry) = demand.get_mut(m.node.as_str()) else { + continue; + }; + let mut cols: HashSet<&str> = HashSet::new(); + if let Some(when) = &m.when { + when.referenced_columns(&mut cols); + } + super::validate::collect_target_columns(&m.target, &mut cols); + entry.extend(cols.into_iter().map(str::to_string)); + } + + for node in &blueprint.nodes { + if let NodeOp::Filter { condition, .. } = &node.op { + let mut cols: HashSet<&str> = HashSet::new(); + condition.referenced_columns(&mut cols); + if let Some(entry) = demand.get_mut(node.id.as_str()) { + entry.extend(cols.into_iter().map(str::to_string)); + } + } + } + + // Demand only grows and is bounded by each node's full column set, so this terminates. A + // bounded number of passes would not do: a `Filter` moves demand both down and back up, so one + // demand can travel arbitrarily many edges in either direction before settling. + let mut changed = true; + while changed { + changed = false; + for node in &blueprint.nodes { + let here = demand.get(node.id.as_str()).cloned().unwrap_or_default(); + match &node.op { + NodeOp::Source { .. } => {} + NodeOp::Filter { input, .. } => { + // Equal, not narrower. See this function's docs. + let mut from_input = HashSet::new(); + if let Some(entry) = demand.get_mut(input.as_str()) { + changed |= extend_demand(entry, here.iter().cloned()); + from_input.clone_from(entry); + } + if let Some(entry) = demand.get_mut(node.id.as_str()) { + changed |= extend_demand(entry, from_input); + } + } + NodeOp::Union { inputs } => { + for input in inputs { + if let Some(entry) = demand.get_mut(input.as_str()) { + changed |= extend_demand(entry, here.iter().cloned()); + } + } + } + NodeOp::Join { left, right, on } => { + for (l, r) in on { + if let Some(entry) = demand.get_mut(left.as_str()) { + changed |= entry.insert(l.clone()); + } + if let Some(entry) = demand.get_mut(right.as_str()) { + changed |= entry.insert(r.clone()); + } + } + let (Some(l_full), Some(r_full)) = + (full.get(left.as_str()), full.get(right.as_str())) + else { + continue; + }; + for col in &here { + let Some((side, source_column)) = join_column_source(col, l_full, r_full) + else { + continue; + }; + let input = match side { + JoinSide::Left => left, + JoinSide::Right => right, + }; + if let Some(entry) = demand.get_mut(input.as_str()) { + changed |= entry.insert(source_column.to_string()); + } + } + } + } + } + } + + demand +} + +/// Add every name in `more` to `into`, reporting whether anything was actually new. +fn extend_demand(into: &mut HashSet, more: impl IntoIterator) -> bool { + let mut changed = false; + for name in more { + changed |= into.insert(name); + } + changed +} + +/// A node's execution-time schema: [`full_node_schemas`]'s entry for that node, restricted to +/// [`demanded_columns`]'s entry, i.e. exactly the columns anything downstream reads, with their +/// declared types for [`Predicate::prepare`](super::predicate::Predicate::prepare). +pub(crate) fn projected_schema( + full: &TableSchema, + demanded: &HashSet, + node_id: &str, +) -> TableSchema { + TableSchema { + name: node_id.to_string(), + columns: full + .columns + .iter() + .filter(|(name, _)| demanded.contains(name.as_str())) + .map(|(n, c)| (n.clone(), c.clone())) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::blueprint::{IdRendering, Node}; + use crate::core::event_data::object_centric::extraction::catalog::ExtractionCatalog; + use crate::core::event_data::object_centric::extraction::value::ValueKind; + use crate::core::event_data::object_centric::extraction::MODEL_VERSION; + + fn blueprint(nodes: Vec) -> Blueprint { + Blueprint { + version: MODEL_VERSION, + id_rendering: IdRendering::Raw, + nodes, + mappings: vec![], + on_missing_endpoint: super::super::MissingEndpointPolicy::default(), + on_duplicate_object: super::super::DuplicateObjectPolicy::default(), + } + } + + fn source(id: &str, table: &str) -> Node { + Node { + id: id.to_string(), + label: None, + op: NodeOp::Source { + source_id: "db".into(), + table: table.to_string(), + }, + } + } + + #[test] + fn inputs_disagreeing_about_a_union_column_declare_no_kind() { + let bp = blueprint(vec![ + source("new", "orders"), + source("old", "legacy_orders"), + Node { + id: "all".into(), + label: None, + op: NodeOp::Union { + inputs: vec!["new".into(), "old".into()], + }, + }, + ]); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("state", "TEXT", false)], + ), + ) + .with_table( + "db", + TableSchema::new( + "legacy_orders", + [("id", "VARCHAR", true), ("state", "TEXT", false)], + ), + ); + + let full = full_node_schemas(&bp, &catalog); + let all = full.get("all").expect("union resolves"); + assert_eq!(all.columns["id"].declared_kind(), None); + assert!(all.columns["id"].nullable, "either input may be null here"); + assert_eq!(all.columns["state"].declared_kind(), Some(ValueKind::Text)); + } + + #[test] + fn a_union_column_declared_by_only_one_input_is_nullable() { + let bp = blueprint(vec![ + source("new", "orders"), + source("old", "legacy_orders"), + Node { + id: "all".into(), + label: None, + op: NodeOp::Union { + inputs: vec!["new".into(), "old".into()], + }, + }, + ]); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("discount", "INTEGER", false)], + ), + ) + .with_table( + "db", + TableSchema::new("legacy_orders", [("id", "INTEGER", false)]), + ); + + let full = full_node_schemas(&bp, &catalog); + let all = full.get("all").expect("union resolves"); + // `legacy_orders` has no `discount` column, so a row from it contributes Null there -- + // nullable even though the sole declaring input marks it non-null. + assert!( + all.columns["discount"].nullable, + "column missing from one union input must be nullable" + ); + assert!( + !all.columns["id"].nullable, + "column declared non-null by every input stays non-null" + ); + } + + #[test] + fn a_join_column_name_claimed_by_both_sides_resolves_to_neither() { + let left = TableSchema::new("l", [("id", "INTEGER", false), ("right_id", "TEXT", false)]); + let right = TableSchema::new("r", [("id", "INTEGER", false)]); + assert_eq!(join_column_source("right_id", &left, &right), None); + assert_eq!( + join_column_source("id", &left, &right), + Some((JoinSide::Left, "id")) + ); + + // Without a left column of that name the rename is unambiguous. + let plain = TableSchema::new("l", [("id", "INTEGER", false)]); + assert_eq!( + join_column_source("right_id", &plain, &right), + Some((JoinSide::Right, "id")) + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/sink.rs b/process_mining/src/core/event_data/object_centric/extraction/sink.rs new file mode 100644 index 00000000..998a7f8f --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/sink.rs @@ -0,0 +1,317 @@ +//! Where extracted entities and relations go. + +use std::fmt::Debug; + +use chrono::{DateTime, FixedOffset}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::blueprint::MissingEndpointPolicy; +use crate::core::event_data::object_centric::{OCELAttributeValue, OCELTypeAttribute}; + +/// A handle to an event, passed back to [`ExtractionSink::add_e2o`]. +/// +/// Opaque to the extractor, which only ever stores and replays it. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum EventRef { + /// An in-memory sink's own event index. + Index(u32), + /// The id a streaming sink was given, echoed back as its handle. + Id(String), +} + +/// A handle to an object. Mirrors [`EventRef`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ObjectRef { + /// An in-memory sink's own object index. + Index(u32), + /// The id a streaming sink was given, echoed back as its handle. + Id(String), +} + +/// What a sink can say about an entity id a relation names. +/// +/// [`Resolution::Deferred`] is what lets a sink avoid keeping a full in-memory id index: it may +/// decline to answer now and resolve the relation in bulk at +/// [`finalize`](ExtractionSink::finalize). `on_missing_endpoint` decides the same fate for a +/// relation either way, but a deferring sink can only report a count, not which row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Resolution { + /// The entity exists. Here is its handle. + Exists(R), + /// The entity does not exist. The caller applies `on_missing_endpoint` now. + Missing, + /// The sink cannot say, and has undertaken to resolve it at + /// [`finalize`](ExtractionSink::finalize). The handle is valid for writing a relation + /// against, but carries no promise that the entity exists: if it turns out not to, + /// `finalize` applies `on_missing_endpoint` to the relation then. + Deferred(R), +} + +impl Resolution { + /// The handle, for `Exists` and `Deferred` alike. `None` for `Missing`. + #[must_use] + pub fn into_ref(self) -> Option { + match self { + Resolution::Exists(r) | Resolution::Deferred(r) => Some(r), + Resolution::Missing => None, + } + } +} + +/// What a sink did during [`ExtractionSink::finalize`], folded into +/// [`ExtractionReport::finalize`](super::report::ExtractionReport::finalize). +/// +/// All zero for a sink that resolves eagerly. Nonzero entries carry a deferring sink's share of +/// the per-mapping counters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FinalizeReport { + /// Relations written against a [`Resolution::Deferred`] endpoint that resolved to a real + /// entity at finalize. + pub resolved_relations: u64, + /// Relations whose deferred endpoint did not resolve. An eager sink counts these per mapping + /// as [`DropReason::UnresolvedEndpoint`](super::report::DropReason::UnresolvedEndpoint). + pub unresolved_endpoints: u64, + /// Objects synthesised at finalize under `on_missing_endpoint: Create`, for deferred + /// endpoints that turned out not to exist. + pub objects_created: u64, + /// Repeated entity ids removed at finalize: a deferring sink's share of + /// [`MappingStats::deduplicated`](super::report::MappingStats::deduplicated). + pub duplicates_removed: u64, +} + +/// Why an [`ExtractionSink`] call failed. +/// +/// Serializable but not deserializable: [`SinkError::UnknownType`] carries a `&'static str` (a +/// borrow no deserializer can manufacture), so this only ever crosses a bindings boundary +/// outbound, nested inside [`ExtractionError::Sink`](super::report::ExtractionError::Sink). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub enum SinkError { + /// [`ExtractionSink::add_event`] was called with an id already present. + DuplicateEvent { + /// The repeated id. + id: String, + }, + /// [`ExtractionSink::add_object`] was called with an id already present. + DuplicateObject { + /// The repeated id. + id: String, + }, + /// [`ExtractionSink::add_object`] was called for `id`, but `id` is already present under a + /// different object type, i.e. two distinct entities collided on one id rather than the plain + /// repeat [`SinkError::DuplicateObject`] reports. Only reachable under + /// [`IdRendering::Raw`](super::blueprint::IdRendering::Raw). A deferring sink cannot make that + /// distinction at [`resolve_object`](ExtractionSink::resolve_object) time and reports it + /// here. + IdTypeCollision { + /// The contested id. + id: String, + }, + /// An entity was added, or a relation named an endpoint, under a type that was never + /// declared via [`ExtractionSink::declare_event_type`] / [`declare_object_type`](ExtractionSink::declare_object_type). + UnknownType { + /// `"event"` or `"object"`. + kind: &'static str, + /// The undeclared type name. + name: String, + }, + /// A relation named a ref this sink instance did not itself hand out. Cannot happen through + /// [`extract`](super::extract::extract). + InvalidRef, + /// The backend failed (I/O, driver error), as its message. + Backend(String), +} + +impl std::fmt::Display for SinkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SinkError::DuplicateEvent { id } => write!(f, "duplicate event id '{id}'"), + SinkError::DuplicateObject { id } => write!(f, "duplicate object id '{id}'"), + SinkError::IdTypeCollision { id } => { + write!(f, "id '{id}' is already taken by an object of another type") + } + SinkError::UnknownType { kind, name } => { + write!(f, "undeclared {kind} type '{name}'") + } + SinkError::InvalidRef => write!(f, "ref not recognised by this sink"), + SinkError::Backend(message) => write!(f, "sink backend error: {message}"), + } + } +} + +impl std::error::Error for SinkError {} + +/// Where [`extract`](super::extract::extract) sends the entities and relations it produces. +/// +/// Two implementations ship with this crate: [`SlimOcelSink`](super::slim_sink::SlimOcelSink) +/// (in memory, always available) and a `DuckDB`-backed streaming sink (feature `ocel-duckdb`). +pub trait ExtractionSink: Debug { + /// Declare an event type with these attributes, merging with any earlier declaration of + /// the same name (new attribute names are added, already-declared ones are left alone). + /// Idempotent: declaring the same type again is not an error. + /// + /// # Errors + /// Returns [`SinkError`] if the backend fails to record the declaration. + fn declare_event_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError>; + + /// Declare an object type. See [`declare_event_type`](ExtractionSink::declare_event_type). + /// + /// # Errors + /// Returns [`SinkError`] if the backend fails to record the declaration. + fn declare_object_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError>; + + /// Add an event of a previously declared `event_type`, whose declaration must already name + /// every attribute in `attributes`. + /// + /// # Errors + /// Returns [`SinkError::UnknownType`] if `event_type` was never declared, + /// [`SinkError::DuplicateEvent`] if `id` is already present, or [`SinkError::Backend`] if an + /// attribute name is not one the type declares (as + /// [`add_object_attribute`](ExtractionSink::add_object_attribute) reports the same condition) + /// or the backend failed. + fn add_event( + &mut self, + event_type: &str, + time: DateTime, + id: &str, + attributes: &[(String, OCELAttributeValue)], + ) -> Result; + + /// Add an object of a previously declared `object_type`, with its initial timed attribute + /// values, subject to the same one-value-per-`(id, name, time)` rule + /// [`add_object_attribute`](ExtractionSink::add_object_attribute) describes. + /// + /// # Errors + /// Returns [`SinkError::UnknownType`] if `object_type` was never declared, + /// [`SinkError::DuplicateObject`] if `id` is already present under this same type, + /// [`SinkError::IdTypeCollision`] if `id` is already present under a different type, or + /// [`SinkError::Backend`] if an attribute name is not one the type declares (as + /// [`add_object_attribute`](ExtractionSink::add_object_attribute) reports the same condition) + /// or the backend failed. + fn add_object( + &mut self, + object_type: &str, + id: &str, + attributes: &[(String, DateTime, OCELAttributeValue)], + ) -> Result; + + /// Append one more timed value to an already-added object's named attribute. This is how a + /// change-tracked [`Target::Object`](super::blueprint::Target::Object) (`timestamp: Some`) + /// records a later row naming the same object id. + /// + /// # At most one value per `(id, name, time)`, and the first one wins + /// + /// A repeated `(object, name, time)` is not an error and does not append a second + /// entry. The same rule applies to [`add_object`](ExtractionSink::add_object)'s initial + /// values, so an object's history does not depend on whether the row that wrote an attribute + /// also created the object. It is what makes a static object attribute single-valued: a + /// `timestamp: None` [`Target::Object`](super::blueprint::Target::Object) writes every row's + /// value at the epoch, so a case-level column repeated across a case's events is stored once. + /// + /// Where two writes at one `(id, name, time)` carry different values, scan order decides + /// which survives, and [`extract`](super::extract::extract) issues no `ORDER BY`. + /// + /// # Errors + /// Returns [`SinkError::InvalidRef`] if `object` was not handed out by this sink, or + /// [`SinkError::Backend`] on a backend failure. + fn add_object_attribute( + &mut self, + object: &ObjectRef, + name: &str, + time: DateTime, + value: OCELAttributeValue, + ) -> Result<(), SinkError>; + + /// Announce the `on_missing_endpoint` policy this run uses, before any entity is added. + /// + /// Only a sink that answers [`Resolution::Deferred`] needs this: it applies the policy itself + /// at [`finalize`](ExtractionSink::finalize). For an eager sink the extractor applies it at + /// the call site, so the default is to ignore it. + /// + /// # Errors + /// Returns [`SinkError`] if the backend fails to record it. + fn set_missing_endpoint_policy( + &mut self, + policy: MissingEndpointPolicy, + ) -> Result<(), SinkError> { + let _ = policy; + Ok(()) + } + + /// Resolve an event id a relation names, without adding anything. + /// + /// `event_type`, when the endpoint declares one, is the type the caller expects: a sink that + /// can check it must answer [`Resolution::Missing`] when an event of that id exists under a + /// different type. Otherwise, under + /// [`IdRendering::Raw`](super::blueprint::IdRendering::Raw), two types collide on one id and + /// are silently merged. + /// + /// Takes `&mut self` so a sink may record that it owes an answer, see + /// [`Resolution::Deferred`]. + fn resolve_event(&mut self, id: &str, event_type: Option<&str>) -> Resolution; + + /// Resolve an object id. See [`resolve_event`](ExtractionSink::resolve_event). + fn resolve_object(&mut self, id: &str, object_type: Option<&str>) -> Resolution; + + /// Finish the run: resolve everything this sink deferred, and leave the backing store in a + /// readable state. Called exactly once by [`extract`](super::extract::extract), after the + /// last row. + /// + /// # Errors + /// Returns [`SinkError`] if the backend fails. Fatal to the run: the log is incomplete. + fn finalize(&mut self) -> Result; + + /// Add an event-to-object relation. + /// + /// # Calling contract: one `resolve_object` immediately before, for this row's own object + /// + /// Every `add_e2o` call must be immediately preceded by exactly one + /// [`resolve_object`](ExtractionSink::resolve_object) call for the very `object` argument + /// passed here, with no other `resolve_object` call in between. + /// [`extract`](super::extract::extract) satisfies this by construction. A third-party driver + /// that batches its resolutions does not, and a deferring sink cannot detect the violation. + /// + /// The adjacency is how a deferring sink links each staged ask to the `add_e2o` that consumed + /// it, so it can discard an ask belonging to a row whose event never materialised. Violate it + /// and, under [`MissingEndpointPolicy::Create`], + /// an object id can be synthesised under the wrong declared type, silently. + /// + /// # Errors + /// Returns [`SinkError::InvalidRef`] if either ref was not handed out by this sink, or + /// [`SinkError::Backend`] on a backend failure. + fn add_e2o( + &mut self, + event: &EventRef, + object: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError>; + + /// Add an object-to-object relation, from `source` to `target`. + /// + /// # Calling contract: one `resolve_object` immediately before, for this row's own target + /// + /// The same adjacency [`add_e2o`](ExtractionSink::add_e2o) requires, on the target side. + /// The `source` carries no such requirement: one resolved source may back several targets, + /// which is how [`extract`](super::extract::extract) drives it. + /// + /// It lets a deferring sink discard a target ask staged for a row whose source never + /// resolved. See `add_e2o` for what a violation costs. + /// + /// # Errors + /// Returns [`SinkError::InvalidRef`] if either ref was not handed out by this sink, or + /// [`SinkError::Backend`] on a backend failure. + fn add_o2o( + &mut self, + source: &ObjectRef, + target: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError>; +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/slim_sink.rs b/process_mining/src/core/event_data/object_centric/extraction/slim_sink.rs new file mode 100644 index 00000000..2c4a411f --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/slim_sink.rs @@ -0,0 +1,298 @@ +//! In-memory [`ExtractionSink`], backed by a [`SlimLinkedOCEL`]. + +use std::collections::HashMap; + +use chrono::{DateTime, FixedOffset}; + +use crate::core::event_data::object_centric::linked_ocel::slim_linked_ocel::{ + EventIndex, ObjectIndex, +}; +use crate::core::event_data::object_centric::linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}; +use crate::core::event_data::object_centric::{OCELAttributeValue, OCELTypeAttribute}; + +use super::sink::{EventRef, ExtractionSink, FinalizeReport, ObjectRef, Resolution, SinkError}; + +/// In-memory [`ExtractionSink`], backed by a [`SlimLinkedOCEL`]. +/// +/// The handles this sink hands out are always [`EventRef::Index`] / [`ObjectRef::Index`], +/// wrapping the same [`EventIndex`]/[`ObjectIndex`] `SlimLinkedOCEL` itself uses, so relation +/// wiring is a direct index operation with no extra lookup. +/// +/// `SlimLinkedOCEL::add_event`/`add_object` require attribute values positioned to match the +/// declared type's attribute order exactly (by position, not by name), so this sink keeps its +/// own name -> position mirror per type, built as [`declare_event_type`](ExtractionSink::declare_event_type) +/// / [`declare_object_type`](ExtractionSink::declare_object_type) are called, and uses it to +/// reorder the name/value pairs `add_event`/`add_object` are given. +#[derive(Debug, Default)] +pub struct SlimOcelSink { + locel: SlimLinkedOCEL, + event_attr_order: HashMap, + object_attr_order: HashMap, +} + +/// A value offered under a name the entity's type never declared. Refused rather than dropped, so +/// a driver that adds an entity before declaring every attribute does not lose values silently. +fn undeclared(name: &str) -> SinkError { + SinkError::Backend(format!( + "attribute '{name}' was never declared for this entity's type" + )) +} + +impl SlimOcelSink { + /// An empty sink. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// The OCEL built so far. + #[must_use] + pub fn ocel(&self) -> &SlimLinkedOCEL { + &self.locel + } + + /// Consume the sink, returning the OCEL it built. + #[must_use] + pub fn into_ocel(self) -> SlimLinkedOCEL { + self.locel + } + + fn event_index(r: &EventRef) -> Result { + match r { + EventRef::Index(i) => Ok(EventIndex::from(*i)), + EventRef::Id(_) => Err(SinkError::InvalidRef), + } + } + + fn object_index(r: &ObjectRef) -> Result { + match r { + ObjectRef::Index(i) => Ok(ObjectIndex::from(*i)), + ObjectRef::Id(_) => Err(SinkError::InvalidRef), + } + } +} + +/// Whether an endpoint's declared type (`None` = "any") accepts an entity of type `actual`. +fn type_matches(declared: Option<&str>, actual: &str) -> bool { + declared.is_none_or(|t| t == actual) +} + +/// Extend `order` with any name in `attrs` not already present, preserving first-seen order -- +/// mirrors the merge [`SlimLinkedOCEL::add_event_type`]/`add_object_type` themselves apply, so +/// positions agree. +fn merge_order(order: &mut AttributeOrder, attrs: &[OCELTypeAttribute]) { + for a in attrs { + order.push(&a.name); + } +} + +/// One type's attribute names, in declared order, with the name -> position map a `(name, value)` +/// pair resolves through. +/// +/// The map is what keeps a wide type linear: scanning `names` for each position instead is +/// quadratic in the type's attribute count, paid once per entity. +#[derive(Debug, Default)] +struct AttributeOrder { + names: Vec, + positions: HashMap, +} + +impl AttributeOrder { + fn push(&mut self, name: &str) { + if !self.positions.contains_key(name) { + self.positions.insert(name.to_string(), self.names.len()); + self.names.push(name.to_string()); + } + } + + fn len(&self) -> usize { + self.names.len() + } + + fn position(&self, name: &str) -> Option { + self.positions.get(name).copied() + } +} + +/// Insert `(time, value)` into an attribute's history unless that instant is already taken, +/// keeping the history sorted by time so the test is a binary search rather than a scan of every +/// change recorded so far. +fn record_at( + history: &mut Vec<(DateTime, OCELAttributeValue)>, + time: DateTime, + value: OCELAttributeValue, +) { + if let Err(pos) = history.binary_search_by(|(t, _)| t.cmp(&time)) { + history.insert(pos, (time, value)); + } +} + +impl ExtractionSink for SlimOcelSink { + fn declare_event_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + merge_order( + self.event_attr_order.entry(name.to_string()).or_default(), + attrs, + ); + self.locel.add_event_type(name, attrs.to_vec()); + Ok(()) + } + + fn declare_object_type( + &mut self, + name: &str, + attrs: &[OCELTypeAttribute], + ) -> Result<(), SinkError> { + merge_order( + self.object_attr_order.entry(name.to_string()).or_default(), + attrs, + ); + self.locel.add_object_type(name, attrs.to_vec()); + Ok(()) + } + + fn add_event( + &mut self, + event_type: &str, + time: DateTime, + id: &str, + attributes: &[(String, OCELAttributeValue)], + ) -> Result { + let order = + self.event_attr_order + .get(event_type) + .ok_or_else(|| SinkError::UnknownType { + kind: "event", + name: event_type.to_string(), + })?; + let mut values = vec![OCELAttributeValue::Null; order.len()]; + for (name, value) in attributes { + let pos = order.position(name).ok_or_else(|| undeclared(name))?; + values[pos] = value.clone(); + } + self.locel + .add_event(event_type, time, Some(id.to_string()), values, Vec::new()) + .map(|idx| EventRef::Index(idx.into_inner())) + .ok_or_else(|| SinkError::DuplicateEvent { id: id.to_string() }) + } + + fn add_object( + &mut self, + object_type: &str, + id: &str, + attributes: &[(String, DateTime, OCELAttributeValue)], + ) -> Result { + let order = + self.object_attr_order + .get(object_type) + .ok_or_else(|| SinkError::UnknownType { + kind: "object", + name: object_type.to_string(), + })?; + let mut values: Vec, OCELAttributeValue)>> = + vec![Vec::new(); order.len()]; + for (name, time, value) in attributes { + let pos = order.position(name).ok_or_else(|| undeclared(name))?; + // The same first-wins rule `add_object_attribute` applies, for the case where one + // call carries two values for one `(name, time)`. Both paths must agree, or an + // object's attribute history would depend on whether the row that wrote it also + // created the object. + record_at(&mut values[pos], *time, value.clone()); + } + self.locel + .add_object(object_type, Some(id.to_string()), values, Vec::new()) + .map(|idx| ObjectRef::Index(idx.into_inner())) + .ok_or_else(|| SinkError::DuplicateObject { id: id.to_string() }) + } + + /// First-wins on `(name, time)`, this sink's answer to the contract on + /// [`ExtractionSink::add_object_attribute`]. + fn add_object_attribute( + &mut self, + object: &ObjectRef, + name: &str, + time: DateTime, + value: OCELAttributeValue, + ) -> Result<(), SinkError> { + let idx = Self::object_index(object)?; + match idx.get_attribute_value_mut(name, &mut self.locel) { + Some(history) => { + record_at(history, time, value); + Ok(()) + } + None => Err(undeclared(name)), + } + } + + /// Always [`Resolution::Exists`] or [`Resolution::Missing`], never + /// [`Resolution::Deferred`]: the whole log is in memory, so there is nothing to defer. + /// + /// An event of this id under a different type answers `Missing`, not `Exists`: no event of + /// the requested type carries that id, and answering `Exists` would silently merge two types + /// that happen to share an id under `IdRendering::Raw`. + fn resolve_event(&mut self, id: &str, event_type: Option<&str>) -> Resolution { + match self.locel.get_ev_by_id(id) { + Some(i) => { + if type_matches(event_type, i.get_ev_type(&self.locel)) { + Resolution::Exists(EventRef::Index(i.into_inner())) + } else { + Resolution::Missing + } + } + None => Resolution::Missing, + } + } + + /// See [`resolve_event`](ExtractionSink::resolve_event). + fn resolve_object(&mut self, id: &str, object_type: Option<&str>) -> Resolution { + match self.locel.get_ob_by_id(id) { + Some(i) => { + if type_matches(object_type, i.get_ob_type(&self.locel)) { + Resolution::Exists(ObjectRef::Index(i.into_inner())) + } else { + Resolution::Missing + } + } + None => Resolution::Missing, + } + } + + /// Nothing to do: this sink defers nothing, so every counter is zero and the caller's + /// per-mapping stats already hold the whole story. + fn finalize(&mut self) -> Result { + Ok(FinalizeReport::default()) + } + + fn add_e2o( + &mut self, + event: &EventRef, + object: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let e = Self::event_index(event)?; + let o = Self::object_index(object)?; + if self.locel.add_e2o(e, o, qualifier.to_string()) { + Ok(()) + } else { + Err(SinkError::InvalidRef) + } + } + + fn add_o2o( + &mut self, + source: &ObjectRef, + target: &ObjectRef, + qualifier: &str, + ) -> Result<(), SinkError> { + let s = Self::object_index(source)?; + let t = Self::object_index(target)?; + if self.locel.add_o2o(s, t, qualifier.to_string()) { + Ok(()) + } else { + Err(SinkError::InvalidRef) + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/sqlite_provider.rs b/process_mining/src/core/event_data/object_centric/extraction/sqlite_provider.rs new file mode 100644 index 00000000..2ff3346f --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/sqlite_provider.rs @@ -0,0 +1,541 @@ +//! `SQLite`-backed [`RowProvider`]. +#![cfg(feature = "ocel-sqlite")] + +use std::ops::ControlFlow; +use std::path::Path; + +use chrono::{DateTime, FixedOffset, NaiveDateTime}; +use rusqlite::types::ValueRef; +use rusqlite::Connection; + +use super::catalog::{ExtractionCatalog, TableSchema}; +use super::provider::{ProviderError, RowProvider}; +use super::value::Value; + +/// Streams rows out of a `SQLite` database via `rusqlite`. +/// +/// One `SELECT FROM ` per [`RowProvider::scan`] call, read through `rusqlite`'s +/// own row iterator, never collected into a `Vec` first, so a `Source -> Filter` chain over this +/// provider streams. +pub struct SqliteRowProvider { + con: Connection, +} + +impl std::fmt::Debug for SqliteRowProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SqliteRowProvider").finish_non_exhaustive() + } +} + +impl SqliteRowProvider { + /// Open a `SQLite` database file. + /// + /// # Errors + /// Returns the underlying `rusqlite::Error` if the file cannot be opened. + pub fn open>(path: P) -> Result { + Ok(Self { + con: Connection::open(path)?, + }) + } + + /// Wrap an already-open connection. + #[must_use] + pub fn from_connection(con: Connection) -> Self { + Self { con } + } + + /// Read a `SQLite` database held in memory. + /// + /// Needs no filesystem: the bytes go straight to `sqlite3_deserialize`, which is what makes + /// extraction work on `wasm32` and on a browser `File`. + /// + /// # Errors + /// Returns `SQLITE_NOTADB` if `bytes` do not begin with a `SQLite` file header, and the + /// underlying `rusqlite::Error` if the handover fails. Corruption past the header surfaces on + /// the first read, as it does for a file on disk. + pub fn from_slice(bytes: &[u8]) -> Result { + let mut con = Connection::open_in_memory()?; + deserialize_slice(&mut con, bytes)?; + Ok(Self { con }) + } + + /// Every table in this database, as an [`ExtractionCatalog`] under `source_id`. + /// + /// Views are included: a blueprint reads them exactly as it reads a table. `sqlite_*` internal + /// tables are not. + /// + /// # Errors + /// Returns the underlying `rusqlite::Error` if the schema cannot be read. + pub fn discover_catalog(&self, source_id: &str) -> Result { + let mut catalog = ExtractionCatalog::new(); + let mut tables = self.con.prepare( + "SELECT name FROM sqlite_master WHERE type IN ('table', 'view') \ + AND name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + let names: Vec = tables + .query_map([], |r| r.get::<_, String>(0))? + .collect::>()?; + drop(tables); + + for name in names { + // `PRAGMA table_info` takes no bind parameter, and the name comes from sqlite_master + // rather than from a caller, so it cannot carry anything a bind would protect against. + let mut cols = self.con.prepare(&format!( + "PRAGMA table_info('{}')", + name.replace('\'', "''") + ))?; + let columns: Vec<(String, String, bool)> = cols + .query_map([], |r| { + let col: String = r.get(1)?; + let decl: String = r.get(2).unwrap_or_default(); + let notnull: i64 = r.get(3)?; + Ok((col, decl, notnull == 0)) + })? + .collect::>()?; + drop(cols); + catalog = catalog.with_table(source_id, TableSchema::new(&name, columns)); + } + Ok(catalog) + } +} + +/// The magic every `SQLite` database file starts with, trailing NUL included. +const SQLITE_HEADER_MAGIC: &[u8] = b"SQLite format 3\0"; +/// A `SQLite` file header is 100 bytes, so anything shorter cannot be a database. +const SQLITE_HEADER_LEN: usize = 100; +/// Offsets of the file-format write and read version bytes within that header. +const WRITE_VERSION_OFFSET: usize = 18; +const READ_VERSION_OFFSET: usize = 19; +/// The value those bytes carry for a write-ahead log, and for a rollback journal. +const FORMAT_WAL: u8 = 2; +const FORMAT_ROLLBACK: u8 = 1; + +/// Load `data` into `con` as its `main` database, via `sqlite3_deserialize`. +fn deserialize_slice(con: &mut Connection, data: &[u8]) -> Result<(), rusqlite::Error> { + let schema = std::ffi::CString::new("main")?; + // Checked here rather than left to the first query. `sqlite3_deserialize` accepts any bytes + // and only reports a bad database when something reads one, which attributes the failure to + // whichever table happened to be scanned first instead of to the file that is not a database. + if data.len() < SQLITE_HEADER_LEN || !data.starts_with(SQLITE_HEADER_MAGIC) { + return Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_NOTADB), + Some("not a SQLite database: the file header is missing or truncated".to_string()), + )); + } + let sz = i64::try_from(data.len()).map_err(|_| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_TOOBIG), + Some("database is too large to deserialize".to_string()), + ) + })?; + // SQLite takes ownership of a buffer it may resize, so it must be one sqlite allocated. + let buf = unsafe { rusqlite::ffi::sqlite3_malloc64(sz as u64) }.cast::(); + if buf.is_null() { + return Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_NOMEM), + Some("sqlite3_malloc64 failed".to_string()), + )); + } + unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len()) }; + // A deserialized database has no `-wal` sidecar to open, so SQLite answers every read of one + // whose header claims WAL mode with SQLITE_CANTOPEN, so `from_slice` would appear to succeed + // and then fail on the first scan. `sqlite3_deserialize`'s own documented workaround is to + // set the file-format version bytes to 1 before handing the buffer over. The buffer is this + // function's private copy, so the caller's bytes and the file they came from are untouched; + // what is read is the database as of its last checkpoint, which is all that is present. + unsafe { + for offset in [WRITE_VERSION_OFFSET, READ_VERSION_OFFSET] { + let byte = buf.add(offset); + if byte.read() == FORMAT_WAL { + byte.write(FORMAT_ROLLBACK); + } + } + } + let rc = unsafe { + rusqlite::ffi::sqlite3_deserialize( + con.handle(), + schema.as_ptr(), + buf, + sz, + sz, + rusqlite::ffi::SQLITE_DESERIALIZE_RESIZEABLE + | rusqlite::ffi::SQLITE_DESERIALIZE_FREEONCLOSE, + ) + }; + // No `sqlite3_free(buf)` on this path: with SQLITE_DESERIALIZE_FREEONCLOSE set, + // `sqlite3_deserialize` frees the buffer itself before returning a failure, and freeing it + // again here would be a double free. + if rc != rusqlite::ffi::SQLITE_OK { + return Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rc), + Some("sqlite3_deserialize failed".to_string()), + )); + } + Ok(()) +} + +impl RowProvider for SqliteRowProvider { + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[Value]) -> ControlFlow<()>, + ) -> Result<(), ProviderError> { + let quoted_table = quote_ident(table); + // An empty projection means "no columns, one callback per row", not "every column": see + // `RowProvider::scan`. `SELECT 1` gives exactly that row count with nothing to read. + let column_list = if columns.is_empty() { + "1".to_string() + } else { + columns + .iter() + .map(|c| quote_ident(c)) + .collect::>() + .join(", ") + }; + let sql = format!("SELECT {column_list} FROM {quoted_table}"); + + let mut stmt = self.con.prepare(&sql).map_err(|e| map_err(table, &e))?; + // SQLite stores a type per cell, so a TIMESTAMP column arrives as Text and its declared + // type is the only thing saying what that text means. Read once, before the scan, and use + // it to re-tag: otherwise every row pays the multi-format chrono cascade to rediscover + // what the schema already stated. + let declared: Vec> = stmt + .columns() + .iter() + .map(|c| c.decl_type().and_then(declared_kind)) + .collect(); + let mut rows = stmt.query([]).map_err(|e| map_err(table, &e))?; + + let mut buf = vec![Value::Null; columns.len()]; + while let Some(row) = rows.next().map_err(|e| map_err(table, &e))? { + for (i, slot) in buf.iter_mut().enumerate() { + let value_ref = row.get_ref(i).map_err(|e| map_err(table, &e))?; + *slot = convert(value_ref, declared[i]); + } + if f(&buf).is_break() { + return Ok(()); + } + } + Ok(()) + } +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// What a column's declared type says its cells mean, where the storage class alone is ambiguous. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeclaredKind { + Timestamp, + Boolean, +} + +/// `SQLite` type affinity is by name, so this matches by name too. Only the two kinds whose +/// storage class is ambiguous are worth recovering; an INTEGER column already arrives as an +/// integer. +/// +/// Matched per word, not by substring: `SQLite` accepts any declared type, so `CANDIDATE` +/// contains "date" and `RUNTIME_MS` contains "time" while neither holds an instant. Splitting on +/// non-alphanumerics still keeps `TIMESTAMP(3) WITHOUT TIME ZONE`, `TIMESTAMPTZ`, `DATETIME2` and +/// `SMALLDATETIME`. +fn declared_kind(decl: &str) -> Option { + let d = decl.to_ascii_lowercase(); + let words = || d.split(|c: char| !c.is_ascii_alphanumeric()); + if words() + .any(|w| w == "date" || w == "time" || w.contains("timestamp") || w.contains("datetime")) + { + Some(DeclaredKind::Timestamp) + } else if words().any(|w| w.contains("bool")) { + Some(DeclaredKind::Boolean) + } else { + None + } +} + +fn convert(v: ValueRef<'_>, declared: Option) -> Value { + match v { + ValueRef::Null => Value::Null, + // 0/1 in a BOOLEAN column is the value SQLite writes for a bool; anything else is a + // number that happens to live there, and is left alone. + ValueRef::Integer(i) if declared == Some(DeclaredKind::Boolean) && (i == 0 || i == 1) => { + Value::Boolean(i == 1) + } + ValueRef::Integer(i) => Value::Integer(i), + ValueRef::Real(f) => Value::Float(f), + ValueRef::Text(t) => { + let text = String::from_utf8_lossy(t).into_owned(); + if declared == Some(DeclaredKind::Timestamp) { + // Parsed here once, or left as text for the timestamp cascade to try, but never + // silently turned into a wrong instant. + if let Some(ts) = parse_declared_timestamp(&text) { + return Value::Timestamp(ts); + } + } + Value::Text(text) + } + // Not text, and lossy UTF-8 would invent characters. A blob has no place in a value the + // model can render, so it is absent rather than corrupt. + ValueRef::Blob(_) => Value::Null, + } +} + +/// The two spellings `SQLite` itself writes for a timestamp, plus RFC 3339. Deliberately narrow: +/// anything else stays `Text` and reaches the timestamp parser's full cascade, which is where +/// format guessing belongs. +fn parse_declared_timestamp(text: &str) -> Option> { + if let Ok(dt) = DateTime::parse_from_rfc3339(text) { + return Some(dt); + } + let utc = FixedOffset::east_opt(0)?; + for fmt in ["%Y-%m-%d %H:%M:%S%.f", "%Y-%m-%dT%H:%M:%S%.f"] { + if let Ok(naive) = NaiveDateTime::parse_from_str(text, fmt) { + return Some(DateTime::from_naive_utc_and_offset(naive, utc)); + } + } + None +} + +/// Translate a `rusqlite` error into a [`ProviderError`], recognising the two cases `SQLite`'s +/// own error text names literally (`"no such table"`, `"no such column"`) and falling back to +/// [`ProviderError::Backend`] for everything else. +fn map_err(table: &str, e: &rusqlite::Error) -> ProviderError { + let message = e.to_string(); + super::provider::sqlite_message_error(table, &message).unwrap_or(ProviderError::Backend { + table: table.to_string(), + message, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::Catalog; + + /// `SQLite` accepts any word at all as a declared type, so matching "date" or "time" anywhere + /// in one claims columns that hold no instant: `CANDIDATE` and `RUNTIME_MS` are both legal + /// declarations, and a text cell in either would have been re-tagged as a timestamp. + #[test] + fn a_declared_type_that_merely_spells_date_or_time_is_not_a_timestamp() { + for decl in [ + "CANDIDATE", + "UPDATE_COUNT", + "RUNTIME_MS INTEGER", + "VALIDATED", + "MANDATE", + "TEXT", + "VARCHAR(45)", + ] { + assert_eq!(declared_kind(decl), None, "{decl} is not an instant"); + } + for decl in [ + "TIMESTAMP", + "timestamptz", + "TIMESTAMP(3) WITHOUT TIME ZONE", + "DATETIME", + "datetime2", + "SMALLDATETIME", + "DATE", + "TIME", + ] { + assert_eq!( + declared_kind(decl), + Some(DeclaredKind::Timestamp), + "{decl} is an instant" + ); + } + assert_eq!(declared_kind("BOOLEAN"), Some(DeclaredKind::Boolean)); + assert_eq!(declared_kind("bool"), Some(DeclaredKind::Boolean)); + } + + /// The same rule end to end: a date-shaped string in a `CANDIDATE` column stays text. + #[test] + fn a_date_shaped_string_in_a_candidate_column_stays_text() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("candidate.sqlite"); + { + let con = Connection::open(&path).expect("open"); + con.execute_batch( + "CREATE TABLE t (c CANDIDATE); INSERT INTO t VALUES ('2024-01-02 03:04:05');", + ) + .expect("seed"); + } + let provider = + SqliteRowProvider::from_slice(&std::fs::read(&path).expect("read")).expect("load"); + let mut seen = Vec::new(); + provider + .scan("t", &["c"], &mut |row| { + seen = row.to_vec(); + ControlFlow::Continue(()) + }) + .expect("scan"); + assert_eq!(seen[0], Value::Text("2024-01-02 03:04:05".to_string())); + } + + /// A deserialized database has no `-wal` sidecar, so one whose header claims WAL mode is + /// unreadable unless the header handed to `sqlite3_deserialize` says rollback journal. Every + /// database left in WAL mode, which is the default for many applications, lands here and used + /// to load and then fail every scan with "unable to open database file". + #[test] + fn a_database_left_in_wal_mode_still_reads() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("wal.sqlite"); + { + let con = Connection::open(&path).expect("open"); + con.pragma_update(None, "journal_mode", "WAL") + .expect("switch to WAL"); + con.execute_batch("CREATE TABLE t (a TEXT); INSERT INTO t VALUES ('x'), ('y');") + .expect("seed"); + } + let bytes = std::fs::read(&path).expect("read"); + assert_eq!( + (bytes[WRITE_VERSION_OFFSET], bytes[READ_VERSION_OFFSET]), + (FORMAT_WAL, FORMAT_WAL), + "the fixture must really be a WAL-mode file for this to test anything" + ); + + let provider = SqliteRowProvider::from_slice(&bytes).expect("load"); + assert!( + provider + .discover_catalog("db") + .expect("discover") + .table("db", "t") + .is_some(), + "a WAL-mode database's schema is readable" + ); + let mut rows = 0; + provider + .scan("t", &["a"], &mut |_| { + rows += 1; + ControlFlow::Continue(()) + }) + .expect("scan"); + assert_eq!(rows, 2); + // The caller's bytes are its own; only the private copy was rewritten. + assert_eq!(bytes[WRITE_VERSION_OFFSET], FORMAT_WAL); + } + + /// Bytes that are not a database are rejected where they are handed over, not on whichever + /// table is scanned first. `sqlite3_deserialize` accepts anything, so this is checked here. + #[test] + fn bytes_that_are_not_a_database_are_rejected_up_front() { + for bad in [ + Vec::new(), + b"not a database at all".to_vec(), + vec![0u8; 4096], + // A valid header, truncated below the 100 bytes one occupies. + SQLITE_HEADER_MAGIC.to_vec(), + ] { + let err = SqliteRowProvider::from_slice(&bad) + .expect_err("bytes that are not a database must not load"); + assert!( + err.to_string().contains("not a SQLite database"), + "unhelpful error for {} bytes: {err}", + bad.len() + ); + } + } + + /// `SQLite` stores a type per cell, so only the declared type says what a `TIMESTAMP` column's + /// text means. Recovering it here is what keeps every row from re-running the format cascade. + #[test] + fn declared_types_recover_what_the_storage_class_loses() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("typed.sqlite"); + { + let con = Connection::open(&path).expect("open"); + con.execute_batch( + "CREATE TABLE t (at TIMESTAMP, flag BOOLEAN, n INTEGER, label TEXT, raw BLOB); + INSERT INTO t VALUES ('2024-01-02 03:04:05', 1, 7, '2024-01-02 03:04:05', x'00ff');", + ) + .expect("seed"); + } + let provider = + SqliteRowProvider::from_slice(&std::fs::read(&path).expect("read")).expect("load"); + + let mut seen = Vec::new(); + provider + .scan("t", &["at", "flag", "n", "label", "raw"], &mut |row| { + seen = row.to_vec(); + ControlFlow::Continue(()) + }) + .expect("scan"); + + assert!( + matches!(seen[0], Value::Timestamp(_)), + "a TIMESTAMP column is an instant, not text: {:?}", + seen[0] + ); + assert_eq!(seen[1], Value::Boolean(true)); + assert_eq!(seen[2], Value::Integer(7)); + // Identical text, undeclared: still text, so the timestamp cascade decides later. + assert_eq!(seen[3], Value::Text("2024-01-02 03:04:05".to_string())); + // A blob is absent rather than lossy-decoded into invented characters. + assert_eq!(seen[4], Value::Null); + } + + /// A number in a BOOLEAN column that is not 0/1 is a number, not a bool. + #[test] + fn only_zero_and_one_read_as_boolean() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("odd.sqlite"); + { + let con = Connection::open(&path).expect("open"); + con.execute_batch("CREATE TABLE t (flag BOOLEAN); INSERT INTO t VALUES (42);") + .expect("seed"); + } + let provider = + SqliteRowProvider::from_slice(&std::fs::read(&path).expect("read")).expect("load"); + let mut seen = Vec::new(); + provider + .scan("t", &["flag"], &mut |row| { + seen = row.to_vec(); + ControlFlow::Continue(()) + }) + .expect("scan"); + assert_eq!(seen[0], Value::Integer(42)); + } + + #[test] + fn a_database_held_in_memory_scans_and_discovers() { + // Built on disk, then read as bytes: only `from_slice` is under test, not how the bytes + // were produced. + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("fixture.sqlite"); + { + let con = Connection::open(&path).expect("open"); + con.execute_batch( + "CREATE TABLE actor (actor_id numeric NOT NULL, first_name VARCHAR(45)); + INSERT INTO actor VALUES (1, 'PENELOPE'), (2, 'NICK'); + CREATE VIEW recent AS SELECT * FROM actor;", + ) + .expect("seed"); + } + let bytes = std::fs::read(&path).expect("read bytes"); + + let provider = SqliteRowProvider::from_slice(&bytes).expect("load from bytes"); + + let catalog = provider.discover_catalog("db").expect("discover"); + let actor = catalog.table("db", "actor").expect("actor table"); + assert_eq!(actor.columns["actor_id"].col_type, "numeric"); + assert!(!actor.columns["actor_id"].nullable); + assert!(actor.columns["first_name"].nullable); + // A view is readable exactly as a table is, so it belongs in the catalog. + assert!(catalog.table("db", "recent").is_some()); + // Internal bookkeeping is not something a blueprint maps. + assert!(catalog.tables["db"] + .keys() + .all(|t| !t.starts_with("sqlite_"))); + + let mut seen = Vec::new(); + provider + .scan("actor", &["actor_id"], &mut |row| { + seen.push(row[0].canonical_string()); + ControlFlow::Continue(()) + }) + .expect("scan"); + // `numeric` decodes as Float; a whole one is still an identity. + assert_eq!(seen, vec![Some("1".to_string()), Some("2".to_string())]); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/tests.rs b/process_mining/src/core/event_data/object_centric/extraction/tests.rs new file mode 100644 index 00000000..c1733d82 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/tests.rs @@ -0,0 +1,4681 @@ +//! End-to-end tests: `RowProvider`, node graph execution, `ExtractionSink`, mapping execution +//! and `ExtractionReport`. +#![cfg(all(test, feature = "ocel-sqlite"))] + +use std::collections::HashMap; + +use rusqlite::{params, Connection}; +use tempfile::tempdir; + +use super::blueprint::{ + Blueprint, DuplicateObjectPolicy, EventEndpoint, IdRendering, InlineObjectRef, Mapping, + MappingEntry, MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Target, +}; +use super::case_centric::FlatEventTable; +use super::catalog::{ExtractionCatalog, TableSchema}; +use super::expr::{AttributeMapping, SplitKind, SplitSpec, TimestampSource, ValueExpression}; +use super::extract::extract; +use super::predicate::{CompareOp, Literal, Operand, Predicate}; +use super::provider::RowProvider; +use super::report::{DropReason, ExtractionError}; +use super::slim_sink::SlimOcelSink; +use super::sqlite_provider::SqliteRowProvider; +use super::validate::validate; +use crate::core::event_data::object_centric::linked_ocel::LinkedOCELAccess; +use crate::core::event_data::object_centric::utils::flatten::flatten_ocel_on; +use crate::core::event_data::object_centric::OCELAttributeType; +use crate::core::event_data::object_centric::OCELAttributeValue; + +/// A fresh `SQLite` file in its own temp directory, and the still-open build connection used to +/// populate it. Every test gets its own directory, since a shared fixture path corrupts under +/// concurrent test runs. +struct Fixture { + _dir: tempfile::TempDir, + path: std::path::PathBuf, +} + +impl Fixture { + fn new() -> Self { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fixture.sqlite"); + Self { _dir: dir, path } + } + + fn build(&self) -> Connection { + Connection::open(&self.path).expect("open sqlite file") + } + + fn provider(&self) -> SqliteRowProvider { + SqliteRowProvider::open(&self.path).expect("reopen sqlite file") + } +} + +fn providers_of<'a>( + source_id: &str, + provider: &'a SqliteRowProvider, +) -> HashMap { + let mut m: HashMap = HashMap::new(); + m.insert(source_id.to_string(), provider); + m +} + +fn source_node(id: &str, source_id: &str, table: &str) -> Node { + Node { + id: id.to_string(), + label: None, + op: NodeOp::Source { + source_id: source_id.to_string(), + table: table.to_string(), + }, + } +} + +fn col(name: &str) -> ValueExpression { + ValueExpression::Column { + column: name.to_string(), + } +} + +fn constant(value: &str) -> ValueExpression { + ValueExpression::Constant { + value: value.to_string(), + } +} + +fn blank_blueprint(nodes: Vec, mappings: Vec) -> Blueprint { + Blueprint { + version: super::MODEL_VERSION, + id_rendering: IdRendering::Raw, + nodes, + mappings, + on_missing_endpoint: MissingEndpointPolicy::Drop, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } +} + +// flat event table, case-centric. + +/// Fixture + blueprint + catalog for case 1: flat, case-centric event table. Factored out so +/// the differential test can run the identical blueprint against the identical data +/// through both sinks. +fn case1_fixture_and_blueprint() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE events (case_id TEXT, activity TEXT, ts TEXT, region TEXT);", + ) + .unwrap(); + let rows = [ + ("A", "create", "2020-01-01T00:00:00Z", "EU"), + ("A", "approve", "2020-01-02T00:00:00Z", "EU"), + ("A", "close", "2020-01-03T00:00:00Z", "EU"), + ("B", "create", "2020-01-01T00:00:00Z", "US"), + ("B", "close", "2020-01-02T00:00:00Z", "US"), + ("C", "create", "2020-01-01T00:00:00Z", "US"), + ]; + for (case_id, activity, ts, region) in rows { + con.execute( + "INSERT INTO events (case_id, activity, ts, region) VALUES (?1, ?2, ?3, ?4)", + params![case_id, activity, ts, region], + ) + .unwrap(); + } + } + + let bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![], + event_attributes: vec![], + }); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("region", "TEXT", true), + ], + ), + ); + (fx, bp, catalog) +} + +#[test] +fn case_1_flat_event_table_case_centric() { + let (fx, bp, catalog) = case1_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + let ocel = sink.ocel(); + assert_eq!( + ocel.get_obs_of_type("Case").count(), + 3, + "one object per distinct case" + ); + assert_eq!(ocel.get_all_evs().count(), 6, "one event per row"); + let e2o_total: usize = ocel.get_all_evs().map(|e| ocel.get_e2o(e).count()).sum(); + assert_eq!(e2o_total, 6, "one E2O per row"); + + assert_eq!(report.per_mapping.len(), 1); + let stats = &report.per_mapping[0]; + // Zero, not `rows - distinct cases`. This blueprint's single mapping creates its case objects + // through an inline object reference, and resolving a relation endpoint is not a + // deduplication (see `MappingStats::deduplicated`). Counting only the repeats among those + // resolutions is what the per-mapping id set used to buy, and a deferring sink could never + // have agreed with it. The three repeated case ids still produce three objects, not six; only + // the counter changed. + assert_eq!( + stats.deduplicated, 0, + "an inline object reference resolves an endpoint; it does not deduplicate an entity" + ); + assert!(stats.dropped.is_empty(), "zero drops: {:?}", stats.dropped); + + let log = flatten_ocel_on(ocel, "Case"); + assert_eq!(log.traces.len(), 3); + let trace_a = log + .traces + .iter() + .find(|t| t.events.len() == 3) + .expect("case A's trace"); + assert_eq!(trace_a.events.len(), 3); +} + +// discriminated table (zero-match type still declared). + +#[test] +fn case_2_discriminated_table_declares_zero_match_type() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE docs (id INTEGER, kind TEXT);") + .unwrap(); + con.execute("INSERT INTO docs (id, kind) VALUES (1, 'invoice')", []) + .unwrap(); + con.execute("INSERT INTO docs (id, kind) VALUES (2, 'invoice')", []) + .unwrap(); + con.execute("INSERT INTO docs (id, kind) VALUES (3, 'bill')", []) + .unwrap(); + } + + let node = source_node("docs", "db", "docs"); + let object_mapping = |label: &str, kind: &str, object_type: &str| { + MappingEntry::Single(Mapping { + node: "docs".into(), + label: Some(label.into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant(object_type), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }) + }; + let bp = blank_blueprint( + vec![node], + vec![ + object_mapping("invoices", "invoice", "Invoice"), + object_mapping("bills", "bill", "Bill"), + object_mapping("credit_notes", "credit_note", "CreditNote"), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("docs", [("id", "INTEGER", false), ("kind", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + let ocel = sink.ocel(); + let mut types: Vec<&str> = ocel.get_ob_types().collect(); + types.sort_unstable(); + assert_eq!( + types, + vec!["Bill", "CreditNote", "Invoice"], + "all three declared, even CreditNote" + ); + assert_eq!(ocel.get_obs_of_type("Invoice").count(), 2); + assert_eq!(ocel.get_obs_of_type("Bill").count(), 1); + assert_eq!(ocel.get_obs_of_type("CreditNote").count(), 0); + + let credit_notes = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("credit_notes")) + .expect("credit_notes stats"); + assert_eq!(credit_notes.rows_read, 3); + assert_eq!(credit_notes.entities_emitted, 0); + assert_eq!( + credit_notes.dropped.get(&DropReason::PredicateExcluded), + Some(&3) + ); +} + +// ordered group, first-match-wins, equals hand-desugared. + +#[test] +fn case_3_ordered_group_first_match_wins() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE changes (id INTEGER, new_status TEXT);") + .unwrap(); + for (id, status) in [(1, "C"), (2, "P"), (3, "C"), (4, "X")] { + con.execute( + "INSERT INTO changes (id, new_status) VALUES (?1, ?2)", + params![id, status], + ) + .unwrap(); + } + } + + let node = source_node("changes", "db", "changes"); + let event_target = |event_type: &str| Target::Event { + event_type: constant(event_type), + id: Some(col("id")), + timestamp: TimestampSource::constant("2020-01-01T00:00:00Z"), + attributes: vec![], + objects: vec![], + }; + let completed = Mapping { + node: "changes".into(), + label: Some("completed".into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "new_status".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("C".into()), + }, + }), + target: event_target("Completed"), + }; + let changed = Mapping { + node: "changes".into(), + label: Some("changed".into()), + when: None, + target: event_target("Changed"), + }; + let ordered_bp = blank_blueprint( + vec![node.clone()], + vec![MappingEntry::Ordered { + mappings: vec![completed, changed], + }], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "changes", + [("id", "INTEGER", false), ("new_status", "TEXT", false)], + ), + ); + assert_eq!(validate(&ordered_bp, &catalog), vec![]); + + let desugared_mappings = super::desugar::desugar(&ordered_bp); + let hand_bp = blank_blueprint( + vec![node], + desugared_mappings + .into_iter() + .map(MappingEntry::Single) + .collect(), + ); + + let run = |bp: &Blueprint| { + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.into_ocel(); + let mut ids_by_type: Vec<(String, String)> = ocel + .get_all_evs() + .map(|e| (e.get_ev_type(&ocel).clone(), e.get_ev(&ocel).id.clone())) + .collect(); + ids_by_type.sort_unstable(); + ids_by_type + }; + + let ordered_result = run(&ordered_bp); + let hand_result = run(&hand_bp); + assert_eq!( + ordered_result, hand_result, + "ordered sugar must equal hand-desugared" + ); + assert_eq!( + ordered_result, + vec![ + ("Changed".to_string(), "2".to_string()), + ("Changed".to_string(), "4".to_string()), + ("Completed".to_string(), "1".to_string()), + ("Completed".to_string(), "3".to_string()), + ], + "first-match-wins: only rows without new_status='C' fall through to 'changed'" + ); +} + +// join, right_ rule agrees with `validate`. + +/// Fixture + blueprint + catalog for case 4: two tables joined on a shared column name. Factored +/// out so the differential test can reuse it. See [`case1_fixture_and_blueprint`]. +fn case4_fixture_and_blueprint() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE orders (id INTEGER, amount INTEGER); + CREATE TABLE meta (id INTEGER, region TEXT);", + ) + .unwrap(); + for (id, amount) in [(1, 100), (2, 200), (3, 300)] { + con.execute( + "INSERT INTO orders (id, amount) VALUES (?1, ?2)", + params![id, amount], + ) + .unwrap(); + } + for (id, region) in [(1, "EU"), (2, "US")] { + con.execute( + "INSERT INTO meta (id, region) VALUES (?1, ?2)", + params![id, region], + ) + .unwrap(); + } + } + + let left = source_node("orders", "db", "orders"); + let right = source_node("meta", "db", "meta"); + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "orders".into(), + right: "meta".into(), + on: vec![("id".into(), "id".into())], + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("right_id"), + timestamp: None, + attributes: vec![ + AttributeMapping { + source_column: "amount".into(), + name: "amount".into(), + value_type: None, + }, + AttributeMapping { + source_column: "region".into(), + name: "region".into(), + value_type: None, + }, + ], + }, + }); + let bp = blank_blueprint(vec![left, right, join], vec![mapping]); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("amount", "INTEGER", false)], + ), + ) + .with_table( + "db", + TableSchema::new( + "meta", + [("id", "INTEGER", false), ("region", "TEXT", false)], + ), + ); + (fx, bp, catalog) +} + +#[test] +fn case_4_join_right_prefix_matches_validate_prediction() { + let (fx, bp, catalog) = case4_fixture_and_blueprint(); + // If graph.rs's runtime column resolution disagreed with validate's node_columns + // prediction, `right_id`/`region` would be reported UnknownColumn here. + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + assert_eq!( + ocel.get_obs_of_type("Order").count(), + 2, + "inner join drops unmatched id=3" + ); + + for (id, amount, region) in [("1", 100i64, "EU"), ("2", 200i64, "US")] { + let ob = ocel.get_ob_by_id(id).expect("object by right_id"); + let amount_val = &ob.get_attribute_value("amount", ocel).expect("amount attr")[0].1; + assert_eq!(amount_val, &OCELAttributeValue::Integer(amount)); + let region_val = &ob.get_attribute_value("region", ocel).expect("region attr")[0].1; + assert_eq!(region_val, &OCELAttributeValue::String(region.to_string())); + } +} + +/// C1: a `Filter` hands its consumers the input's row verbatim, so its projected schema must be +/// the input's, not a narrower one. `demanded_columns` used to propagate demand downward only, +/// so a source read by both a mapping and a filter produced rows wider than the filter's own +/// schema, and every mapping reading the filter then silently indexed the wrong slot. +/// +/// The dropped column is deliberately not the last one: `b` sits between `a` and `c`, so the +/// filter's narrower schema `{a, c}` puts `c` at slot 1, where the row actually carries `b`. +#[test] +fn c1_a_filter_reads_the_same_row_layout_its_consumers_index_it_with() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE t (a INTEGER, b TEXT, c TEXT);") + .unwrap(); + con.execute("INSERT INTO t VALUES (1, 'X', 'Y')", []) + .unwrap(); + } + + let source = source_node("s", "db", "t"); + let filter = Node { + id: "f".into(), + label: None, + op: NodeOp::Filter { + input: "s".into(), + condition: Predicate::Compare { + left: Operand::Column { column: "a".into() }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Integer(1), + }, + }, + }, + }; + // Two consumers of `s`: this mapping (which needs `b`) and the filter (which needs `a`). + let via_source = MappingEntry::Single(Mapping { + node: "s".into(), + label: Some("via_source".into()), + when: None, + target: Target::Object { + object_type: constant("FromSource"), + id: col("b"), + timestamp: None, + attributes: vec![], + }, + }); + let via_filter = MappingEntry::Single(Mapping { + node: "f".into(), + label: Some("via_filter".into()), + when: None, + target: Target::Object { + object_type: constant("FromFilter"), + id: col("c"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![source, filter], vec![via_source, via_filter]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "t", + [ + ("a", "INTEGER", false), + ("b", "TEXT", false), + ("c", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + + let from_filter: Vec = ocel + .get_obs_of_type("FromFilter") + .map(|o| o.get_ob(ocel).id.clone()) + .collect(); + assert_eq!( + from_filter, + vec!["Y".to_string()], + "the filter's mapping must read column c, not whatever sits at c's position in a \ + narrower projection" + ); + let from_source: Vec = ocel + .get_obs_of_type("FromSource") + .map(|o| o.get_ob(ocel).id.clone()) + .collect(); + assert_eq!(from_source, vec!["X".to_string()]); +} + +/// The `right_` rule is one rule, shared by demand routing and the runtime row assembly. +/// A left table with a real column literally named `right_foo` used to route demand to the right +/// input, so it was never fetched from the left and materialised as `Null`, while the runtime +/// rule resolved it from the left. +#[test] +fn a_left_column_literally_named_right_foo_resolves_from_the_left() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE l (id INTEGER, right_foo TEXT); + CREATE TABLE r (id INTEGER, other TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO l VALUES (1, 'from-left')", []) + .unwrap(); + con.execute("INSERT INTO r VALUES (1, 'x')", []).unwrap(); + } + + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("id".into(), "id".into())], + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Thing"), + id: col("right_foo"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint( + vec![ + source_node("l", "db", "l"), + source_node("r", "db", "r"), + join, + ], + vec![mapping], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "l", + [("id", "INTEGER", false), ("right_foo", "TEXT", false)], + ), + ) + .with_table( + "db", + TableSchema::new("r", [("id", "INTEGER", false), ("other", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert!( + sink.ocel().get_ob_by_id("from-left").is_some(), + "left's own right_foo column must be fetched from the left, not materialised as Null; \ + dropped: {:?}", + report.per_mapping[0].dropped + ); +} + +/// The `right_` rule's second disagreement, which the test above cannot reach because +/// there `right_id`'s unprefixed twin is the join key and so is demanded from the left anyway. +/// +/// Here `foo` exists on both sides but is neither a join key nor read by any mapping, so nothing +/// demands it from the left. The runtime rule tested "does the left have `foo`?" against the +/// projected left schema, which by then did not, so `right_foo` materialised as `Null` and the +/// row was dropped for an unusable id. Both rules now ask both sides' full schemas. +#[test] +fn a_right_prefixed_column_resolves_when_its_unprefixed_twin_is_not_a_join_key() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE l (id INTEGER, foo TEXT); + CREATE TABLE r (id INTEGER, foo TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO l VALUES (1, 'left-foo')", []) + .unwrap(); + con.execute("INSERT INTO r VALUES (1, 'right-foo')", []) + .unwrap(); + } + + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("id".into(), "id".into())], + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Thing"), + // The only column any mapping reads: `foo` itself is never demanded from the left. + id: col("right_foo"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint( + vec![ + source_node("l", "db", "l"), + source_node("r", "db", "r"), + join, + ], + vec![mapping], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("l", [("id", "INTEGER", false), ("foo", "TEXT", false)]), + ) + .with_table( + "db", + TableSchema::new("r", [("id", "INTEGER", false), ("foo", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert!( + sink.ocel().get_ob_by_id("right-foo").is_some(), + "right_foo must be read from the right input, not materialised as Null; dropped: {:?}", + report.per_mapping[0].dropped + ); +} + +/// I-e: a join key column that is a `Float` used to render through `Value::canonical_string`, +/// which is `None` for floats, so the join silently produced zero rows where SQL joins them. +#[test] +fn a_join_on_a_float_key_matches_rows_instead_of_silently_producing_none() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE l (k REAL, name TEXT); + CREATE TABLE r (k REAL, note TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO l VALUES (1.5, 'a')", []).unwrap(); + con.execute("INSERT INTO r VALUES (1.5, 'b')", []).unwrap(); + } + + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("k".into(), "k".into())], + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Pair"), + id: col("name"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint( + vec![ + source_node("l", "db", "l"), + source_node("r", "db", "r"), + join, + ], + vec![mapping], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("l", [("k", "REAL", false), ("name", "TEXT", false)]), + ) + .with_table( + "db", + TableSchema::new("r", [("k", "REAL", false), ("note", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + sink.ocel().get_obs_of_type("Pair").count(), + 1, + "a float join key must join, as it does in SQL" + ); +} + +/// The projection contract: an empty `columns` means "no columns, one callback per row", not +/// "every column". Falling back to `SELECT *` handed the caller rows wider than the index it +/// reads them with. +#[test] +fn an_empty_projection_yields_empty_rows_not_every_column() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE t (a INTEGER, b TEXT, c TEXT);") + .unwrap(); + for i in 0..3 { + con.execute("INSERT INTO t VALUES (?1, 'x', 'y')", params![i]) + .unwrap(); + } + } + let provider = fx.provider(); + let mut widths = Vec::new(); + provider + .scan("t", &[], &mut |vals| { + widths.push(vals.len()); + std::ops::ControlFlow::Continue(()) + }) + .expect("scan"); + assert_eq!(widths, vec![0, 0, 0], "one empty row per table row"); +} + +// C3: endpoint resolution is staged, so mapping order cannot change the result. + +/// Fixture, catalog and the three mappings of the C3 test, so the test can assemble them in +/// either order. +fn c3_fixture() -> (Fixture, ExtractionCatalog, Vec, Vec) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE objs (id TEXT, kind TEXT); + CREATE TABLE rels (src TEXT, dst TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO objs VALUES ('a', 'A')", []) + .unwrap(); + con.execute("INSERT INTO objs VALUES ('b', 'B')", []) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('a', 'b')", []) + .unwrap(); + } + let nodes = vec![ + source_node("objs", "db", "objs"), + source_node("rels", "db", "rels"), + ]; + let of_kind = |kind: &str, object_type: &str| { + MappingEntry::Single(Mapping { + node: "objs".into(), + label: Some(object_type.to_string()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant(object_type), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }) + }; + let o2o = MappingEntry::Single(Mapping { + node: "rels".into(), + label: Some("o2o".into()), + when: None, + target: Target::O2O { + source: ObjectEndpoint { + id: col("src"), + object_type: Some(constant("A")), + split: None, + }, + target: ObjectEndpoint { + id: col("dst"), + object_type: Some(constant("B")), + split: None, + }, + qualifier: Some(constant("links")), + }, + }); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("objs", [("id", "TEXT", false), ("kind", "TEXT", false)]), + ) + .with_table( + "db", + TableSchema::new("rels", [("src", "TEXT", false), ("dst", "TEXT", false)]), + ); + ( + fx, + catalog, + vec![o2o, of_kind("A", "A"), of_kind("B", "B")], + nodes, + ) +} + +/// C3: a blueprint whose `O2O` mapping is written before the mappings creating its endpoints +/// must produce the same log as the reverse order. +/// +/// Resolution used to see only what had been emitted so far, under a first-seen-node execution +/// order the author never wrote, so this legal blueprint silently dropped its one relation. SQL +/// cannot reproduce that either: a compiled relation view joins against all objects. +#[test] +fn c3_mapping_order_does_not_change_the_result() { + use super::differential::snapshot; + + let run = |mappings: Vec| { + let (fx, catalog, _, nodes) = c3_fixture(); + let bp = blank_blueprint(nodes, mappings); + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + (snapshot(sink.ocel()), report) + }; + + let (_, _, mappings, _) = c3_fixture(); + let relation_first = mappings.clone(); + let mut endpoints_first = mappings; + endpoints_first.rotate_left(1); // [A, B, o2o] + + let (relation_first_snapshot, report) = run(relation_first); + let (endpoints_first_snapshot, _) = run(endpoints_first); + + assert_eq!( + relation_first_snapshot, endpoints_first_snapshot, + "the order the mappings are written in must not change the log" + ); + assert_eq!( + relation_first_snapshot + .objects + .get("a") + .expect("object a") + .o2o, + vec![("links".to_string(), "b".to_string())], + "the relation must be emitted, not dropped: {:?}", + report.per_mapping + ); +} + +/// C3, the other half: an event's inline object reference must also resolve against every +/// object the blueprint produces, not merely those emitted before it. +/// +/// A `Target::Event` with `id: None` cannot emit its references in the relations pass, since a +/// run-minted UUID cannot be re-derived there, so they are emitted in the events pass. That used +/// to be the same pass as objects, where a `Target::Object` mapping written after the event +/// mapping had not run yet. With +/// `on_missing_endpoint: Drop` the first row of every case then lost its relation, and writing +/// the two mappings the other way round kept it. `Blueprint::from_flat_event_table` generates +/// precisely this shape, and escapes only because it hardcodes `Create`. +/// +/// Event ids are minted per run, so the two runs are compared on everything except them. +#[test] +fn c3_an_inline_object_reference_resolves_against_every_object_whatever_the_order() { + use super::differential::{snapshot, OcelSnapshot}; + + let build = || { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE events (case_id TEXT, activity TEXT, ts TEXT, region TEXT);", + ) + .unwrap(); + for (case_id, activity, ts, region) in [ + ("A", "create", "2020-01-01T00:00:00Z", "EU"), + ("A", "approve", "2020-01-02T00:00:00Z", "EU"), + ("A", "close", "2020-01-03T00:00:00Z", "EU"), + ("B", "create", "2020-01-01T00:00:00Z", "US"), + ("B", "close", "2020-01-02T00:00:00Z", "US"), + ("C", "create", "2020-01-01T00:00:00Z", "US"), + ] { + con.execute( + "INSERT INTO events (case_id, activity, ts, region) VALUES (?1, ?2, ?3, ?4)", + params![case_id, activity, ts, region], + ) + .unwrap(); + } + } + let mut bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![AttributeMapping { + source_column: "region".into(), + name: "region".into(), + value_type: None, + }], + event_attributes: vec![], + }); + // The generator hardcodes `Create`, which hides the ordering question by synthesising + // whatever has not been emitted yet. `Drop` exposes it, and is a setting an author may + // legitimately choose for the same blueprint. + bp.on_missing_endpoint = MissingEndpointPolicy::Drop; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("region", "TEXT", false), + ], + ), + ); + (fx, bp, catalog) + }; + + // Everything a minted event id does not make incomparable between two runs. + let comparable = |s: &OcelSnapshot| { + let mut events: Vec<_> = s + .events + .values() + .map(|e| (e.event_type.clone(), e.time, e.e2o.clone())) + .collect(); + events.sort(); + (events, s.objects.clone()) + }; + + let run = |event_mapping_first: bool| { + let (fx, mut bp, catalog) = build(); + assert_eq!(bp.mappings.len(), 2, "one event mapping, one case mapping"); + if !event_mapping_first { + bp.mappings.reverse(); + } + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + comparable(&snapshot(sink.ocel())) + }; + + let event_first = run(true); + let objects_first = run(false); + assert_eq!( + event_first, objects_first, + "the order the mappings are written in must not change the log" + ); + assert_eq!( + event_first + .0 + .iter() + .filter(|(_, _, e2o)| !e2o.is_empty()) + .count(), + 6, + "every row's case object exists, so every row keeps its relation" + ); +} + +/// `add_event` deduplicates on id, and the dropped one must not be counted as emitted. (The +/// invariant is named after a bug in the OCPQ extractor, which discarded `add_event`'s return +/// value while checking `add_object`'s, so `total_events` overreported.) +#[test] +fn i4_a_duplicate_event_id_is_counted_as_deduplicated_not_as_emitted() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id TEXT, activity TEXT, ts TEXT);") + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('e1', 'A', '2020-01-01T00:00:00Z')", + [], + ) + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('e1', 'A', '2020-01-02T00:00:00Z')", + [], + ) + .unwrap(); + } + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let bp = blank_blueprint(vec![source_node("events", "db", "events")], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!(sink.ocel().get_all_evs().count(), 1, "one event survives"); + let stats = &report.per_mapping[0]; + assert_eq!(stats.rows_read, 2); + assert_eq!( + stats.entities_emitted, 1, + "the deduplicated row must not be counted as emitted" + ); + assert_eq!(stats.deduplicated, 1); + assert!(stats.dropped.is_empty(), "a repeat is not a drop"); +} + +/// I-d: a row whose event id repeats an earlier row's used to return before its inline object +/// references were processed, so those relations vanished with no `DropReason`. They belong to +/// the row, not to the event insertion, and a compiled relation view emits one per row regardless +/// of how many distinct events the id column names. +#[test] +fn i_d_a_duplicate_event_id_keeps_that_row_s_inline_object_relations() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id TEXT, activity TEXT, ts TEXT, obj TEXT);") + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('e1', 'A', '2020-01-01T00:00:00Z', 'o1')", + [], + ) + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('e1', 'A', '2020-01-02T00:00:00Z', 'o2')", + [], + ) + .unwrap(); + } + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![InlineObjectRef { + object: ObjectEndpoint { + id: col("obj"), + object_type: Some(constant("Thing")), + split: None, + }, + qualifier: Some(constant("uses")), + }], + }, + }); + let mut bp = blank_blueprint(vec![source_node("events", "db", "events")], vec![mapping]); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("obj", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + let ocel = sink.ocel(); + assert_eq!(ocel.get_all_evs().count(), 1); + let e2o_total: usize = ocel.get_all_evs().map(|e| ocel.get_e2o(e).count()).sum(); + assert_eq!( + e2o_total, 2, + "both rows' inline relations survive; dropped: {:?}", + report.per_mapping[0].dropped + ); + assert!(ocel.get_ob_by_id("o2").is_some(), "the second row's object"); +} + +/// I-c: `deduplicated` used to count every relation endpoint that resolved, so an `E2O` mapping +/// over n rows with every endpoint present and every id distinct reported n deduplications while +/// nothing had been deduplicated. +#[test] +fn i_c_resolving_an_existing_endpoint_is_not_a_deduplication() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE objs (id TEXT); + CREATE TABLE evs (id TEXT, activity TEXT, ts TEXT); + CREATE TABLE rels (ev TEXT, ob TEXT);", + ) + .unwrap(); + for i in 1..=3 { + con.execute("INSERT INTO objs VALUES (?1)", params![format!("o{i}")]) + .unwrap(); + con.execute( + "INSERT INTO evs VALUES (?1, 'A', '2020-01-01T00:00:00Z')", + params![format!("e{i}")], + ) + .unwrap(); + con.execute( + "INSERT INTO rels VALUES (?1, ?2)", + params![format!("e{i}"), format!("o{i}")], + ) + .unwrap(); + } + } + let objects = MappingEntry::Single(Mapping { + node: "objs".into(), + label: Some("objects".into()), + when: None, + target: Target::Object { + object_type: constant("Thing"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let events = MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let e2o = MappingEntry::Single(Mapping { + node: "rels".into(), + label: Some("e2o".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("ev"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(constant("Thing")), + split: None, + }, + qualifier: Some(constant("uses")), + }, + }); + let bp = blank_blueprint( + vec![ + source_node("objs", "db", "objs"), + source_node("evs", "db", "evs"), + source_node("rels", "db", "rels"), + ], + vec![objects, events, e2o], + ); + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("objs", [("id", "TEXT", false)])) + .with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new("rels", [("ev", "TEXT", false), ("ob", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + let stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("e2o")) + .expect("e2o stats"); + assert_eq!(stats.entities_emitted, 3, "three relations"); + assert!(stats.dropped.is_empty(), "no drops: {:?}", stats.dropped); + assert_eq!( + stats.deduplicated, 0, + "nothing was deduplicated: three distinct endpoints, each resolved once" + ); +} + +/// I-a: two object types sharing one rendered id under `IdRendering::Raw` (the default). The +/// lookup ignored the type, so the second type's object silently merged into the first's, and +/// writing the second's attributes onto it failed as soon as the first type had not declared that +/// attribute name, aborting the whole extraction with `?`. +#[test] +fn i_a_a_cross_type_id_collision_is_reported_not_merged_and_does_not_abort() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE things (id TEXT, kind TEXT, note TEXT);") + .unwrap(); + con.execute("INSERT INTO things VALUES ('1', 'order', NULL)", []) + .unwrap(); + con.execute("INSERT INTO things VALUES ('1', 'item', 'n')", []) + .unwrap(); + } + let of_kind = |kind: &str, object_type: &str, attributes: Vec| { + MappingEntry::Single(Mapping { + node: "things".into(), + label: Some(object_type.to_string()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant(object_type), + id: col("id"), + timestamp: None, + attributes, + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("things", "db", "things")], + vec![ + of_kind("order", "Order", vec![]), + of_kind( + "item", + "Item", + vec![AttributeMapping { + source_column: "note".into(), + name: "note".into(), + value_type: None, + }], + ), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "things", + [ + ("id", "TEXT", false), + ("kind", "TEXT", false), + ("note", "TEXT", true), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("must not abort the run"); + + let ocel = sink.ocel(); + assert_eq!(ocel.get_obs_of_type("Order").count(), 1); + assert_eq!( + ocel.get_obs_of_type("Item").count(), + 0, + "the colliding Item is not merged into the Order" + ); + let item = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("Item")) + .expect("Item stats"); + assert_eq!( + item.dropped.get(&DropReason::IdTypeCollision), + Some(&1), + "a collision has its own reason: {:?}", + item.dropped + ); + assert_eq!(item.deduplicated, 0, "a collision is not a deduplication"); + assert!( + report.errors.iter().any(|e| matches!( + e, + ExtractionError::IdTypeCollision { id, requested_type, .. } + if id == "1" && requested_type == "Item" + )), + "collision reported: {:?}", + report.errors + ); +} + +/// Fixture, blueprint and catalog for the lazy-declaration case: two object mappings over one row, naming the same +/// object id, where only the second carries an attribute. Factored out so case 11 can run the +/// identical blueprint through a deferring sink, the shape whose two sinks disagreed. +fn dynamic_type_fixture_and_blueprint() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE rows_ (id TEXT, kind TEXT, note TEXT);") + .unwrap(); + con.execute("INSERT INTO rows_ VALUES ('x', 'T', 'hello')", []) + .unwrap(); + } + // Both mappings name the object type dynamically, so neither is declared up front by the + // static pass. The first declares the type with no attributes and adds the object; the + // second then grows the type and writes to the object that already exists. + let bare = MappingEntry::Single(Mapping { + node: "rows_".into(), + label: Some("bare".into()), + when: None, + target: Target::Object { + object_type: col("kind"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let with_attr = MappingEntry::Single(Mapping { + node: "rows_".into(), + label: Some("with_attr".into()), + when: None, + target: Target::Object { + object_type: col("kind"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "note".into(), + name: "note".into(), + value_type: None, + }], + }, + }); + let bp = blank_blueprint( + vec![source_node("rows_", "db", "rows_")], + vec![bare, with_attr], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "rows_", + [ + ("id", "TEXT", false), + ("kind", "TEXT", false), + ("note", "TEXT", true), + ], + ), + ); + (fx, bp, catalog) +} + +/// A type that gains an attribute after an object of it was added must not make the next +/// attribute write fail. Reachable through any dynamic object type, which declares lazily, one row +/// at a time. +#[test] +fn dynamic_type_a_type_gaining_an_attribute_after_an_object_exists_does_not_abort() { + let (fx, bp, catalog) = dynamic_type_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("must not abort the run"); + + let ocel = sink.ocel(); + let obj = ocel.get_ob_by_id("x").expect("object x"); + let note = obj + .get_attribute_value("note", ocel) + .expect("note history on the grown type"); + assert_eq!(note[0].1, OCELAttributeValue::String("hello".into())); +} + +/// I-f: an empty id cell is not an identity. `''` is how an ERP export routinely writes "no id"; +/// accepting it collapsed every such row into one entity whose id is the empty string, counted as +/// deduplication rather than as the loss it is. +#[test] +fn i_f_an_empty_id_is_dropped_not_treated_as_an_identity() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE objs (id TEXT);").unwrap(); + for id in ["", "", "real"] { + con.execute("INSERT INTO objs VALUES (?1)", params![id]) + .unwrap(); + } + } + let mapping = MappingEntry::Single(Mapping { + node: "objs".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Thing"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![source_node("objs", "db", "objs")], vec![mapping]); + let catalog = + ExtractionCatalog::new().with_table("db", TableSchema::new("objs", [("id", "TEXT", true)])); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!( + sink.ocel().get_obs_of_type("Thing").count(), + 1, + "only the row with a real id produces an object" + ); + assert!(sink.ocel().get_ob_by_id("").is_none()); + let stats = &report.per_mapping[0]; + assert_eq!( + stats.dropped.get(&DropReason::NullOrUnrenderableId), + Some(&2), + "both empty ids are counted as unusable ids: {:?}", + stats.dropped + ); + assert_eq!(stats.deduplicated, 0, "they are not one entity seen twice"); +} + +/// I-g: two mappings declaring one attribute of one type under different value types. The sink +/// merges last-wins while each mapping converts its own rows against the type it declared, so the +/// attribute ends up holding values of two types under one declaration and nothing noticed. +#[test] +fn i_g_conflicting_attribute_declarations_are_reported_and_widened() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE rows_ (id TEXT, kind TEXT, amount TEXT);") + .unwrap(); + con.execute("INSERT INTO rows_ VALUES ('a', 'x', '1')", []) + .unwrap(); + con.execute("INSERT INTO rows_ VALUES ('b', 'y', 'text')", []) + .unwrap(); + } + let typed = |label: &str, kind: &str, value_type: OCELAttributeType| { + MappingEntry::Single(Mapping { + node: "rows_".into(), + label: Some(label.into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant("Thing"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "amount".into(), + name: "amount".into(), + value_type: Some(value_type), + }], + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("rows_", "db", "rows_")], + vec![ + typed("as_integer", "x", OCELAttributeType::Integer), + typed("as_string", "y", OCELAttributeType::String), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "rows_", + [ + ("id", "TEXT", false), + ("kind", "TEXT", false), + ("amount", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert!( + report.errors.iter().any(|e| matches!( + e, + ExtractionError::ConflictingAttributeType { type_name, attribute, .. } + if type_name == "Thing" && attribute == "amount" + )), + "the conflict must be reported: {:?}", + report.errors + ); + use crate::core::event_data::object_centric::readable::ReadableOCEL; + let declared = sink + .ocel() + .object_types() + .iter() + .find(|t| t.name == "Thing") + .expect("Thing declared") + .clone(); + let amount = declared + .attributes + .iter() + .find(|a| a.name == "amount") + .expect("amount declared"); + assert_eq!( + amount.value_type, + OCELAttributeType::String.to_type_string(), + "the declaration is widened to a type covering both, not whichever ran last" + ); + + // Widening the declaration is only half of it: the values stored under it must be of the + // widened type as well, or the attribute still holds two types at once, which was the + // problem. `a` is the row written by the mapping that declared `integer`. + let ocel = sink.ocel(); + for (id, expected) in [("a", "1"), ("b", "text")] { + let obj = ocel.get_ob_by_id(id).expect("object"); + let value = &obj + .get_attribute_value("amount", ocel) + .expect("amount history")[0] + .1; + assert_eq!( + value, + &OCELAttributeValue::String(expected.to_string()), + "object '{id}' must hold a value of the type 'amount' is declared with" + ); + } +} + +// `on_missing_endpoint` at every endpoint kind, not just an O2O's target. + +/// One event, one existing object, and a relation row naming a missing object plus a relation +/// row naming a missing event. `kind` selects which mapping the blueprint carries. +/// +/// The two rows name different missing objects on purpose. Sharing one id hid a divergence: +/// `run_e2o` gives up before it ever looks at the object endpoint of the row whose event is +/// missing, so under `Create` the eager sink never creates that object, while a deferring sink, +/// which cannot fail an event endpoint at all, staged it and synthesised it at finalize. +fn i5_fixture(target: Target, node: &str) -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE evs (id TEXT, activity TEXT, ts TEXT); + CREATE TABLE rels (ev TEXT, ob TEXT);", + ) + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e1', 'A', '2020-01-01T00:00:00Z')", + [], + ) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('e1', 'ghost')", []) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('no-such-event', 'ghost2')", []) + .unwrap(); + } + let events = MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let under_test = MappingEntry::Single(Mapping { + node: node.into(), + label: Some("under_test".into()), + when: None, + target, + }); + let bp = blank_blueprint( + vec![ + source_node("evs", "db", "evs"), + source_node("rels", "db", "rels"), + ], + vec![events, under_test], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new("rels", [("ev", "TEXT", false), ("ob", "TEXT", false)]), + ); + (fx, bp, catalog) +} + +fn e2o_target() -> Target { + Target::E2O { + event: EventEndpoint { + id: col("ev"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(constant("Thing")), + split: None, + }, + qualifier: Some(constant("uses")), + } +} + +fn i5_run( + bp: &Blueprint, + catalog: &ExtractionCatalog, + fx: &Fixture, +) -> (SlimOcelSink, super::report::ExtractionReport) { + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(bp, catalog, &providers, &mut sink).expect("extract"); + (sink, report) +} + +/// `E2O`'s object side: the policy applies exactly as it does to an `O2O`'s target. +#[test] +fn i5_e2o_object_side_honours_every_missing_endpoint_policy() { + for (policy, expect_objects, expect_drops, expect_error) in [ + (MissingEndpointPolicy::Drop, 0, 2u64, false), + // One relation is still dropped: its event endpoint is missing, and `Create` cannot + // synthesise an event. + (MissingEndpointPolicy::Create, 1, 1, false), + (MissingEndpointPolicy::Error, 0, 2, true), + ] { + let (fx, mut bp, catalog) = i5_fixture(e2o_target(), "rels"); + bp.on_missing_endpoint = policy; + assert_eq!(validate(&bp, &catalog), vec![]); + let (sink, report) = i5_run(&bp, &catalog, &fx); + let stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("under_test")) + .expect("stats"); + assert_eq!( + sink.ocel().get_obs_of_type("Thing").count(), + expect_objects, + "{policy:?}: object synthesis" + ); + assert_eq!( + stats.dropped.get(&DropReason::UnresolvedEndpoint).copied(), + Some(expect_drops), + "{policy:?}: drops {:?}", + stats.dropped + ); + assert_eq!( + report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::MissingEndpoint { .. })), + expect_error, + "{policy:?}: errors {:?}", + report.errors + ); + } +} + +/// `E2O`'s event side, which behaves differently on purpose: `Create` cannot synthesise an +/// event, since there is no timestamp to give it, so it degrades to `Drop` there while still +/// creating objects. +#[test] +fn i5_create_cannot_synthesise_an_event_endpoint_and_degrades_to_drop() { + let (fx, mut bp, catalog) = i5_fixture(e2o_target(), "rels"); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + assert_eq!(validate(&bp, &catalog), vec![]); + let (sink, report) = i5_run(&bp, &catalog, &fx); + + assert_eq!( + sink.ocel().get_all_evs().count(), + 1, + "'no-such-event' is not synthesised" + ); + assert_eq!( + sink.ocel().get_obs_of_type("Thing").count(), + 1, + "the object endpoint, by contrast, is created" + ); + let stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("under_test")) + .expect("stats"); + assert_eq!( + stats.dropped.get(&DropReason::UnresolvedEndpoint), + Some(&1), + "the row with the missing event endpoint is dropped and counted" + ); +} + +/// an event's inline object reference: same policy, same code path, same counts as the `E2O` +/// object side above. +#[test] +fn i5_inline_object_references_honour_every_missing_endpoint_policy() { + for (policy, expect_objects, expect_drops, expect_error) in [ + (MissingEndpointPolicy::Drop, 0, 1u64, false), + (MissingEndpointPolicy::Create, 1, 0, false), + (MissingEndpointPolicy::Error, 0, 1, true), + ] { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE evs (id TEXT, activity TEXT, ts TEXT, ob TEXT);") + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e1', 'A', '2020-01-01T00:00:00Z', 'ghost')", + [], + ) + .unwrap(); + } + let mapping = MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("under_test".into()), + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![InlineObjectRef { + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(constant("Thing")), + split: None, + }, + qualifier: Some(constant("uses")), + }], + }, + }); + let mut bp = blank_blueprint(vec![source_node("evs", "db", "evs")], vec![mapping]); + bp.on_missing_endpoint = policy; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("ob", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + let (sink, report) = i5_run(&bp, &catalog, &fx); + let stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("under_test")) + .expect("stats"); + assert_eq!( + sink.ocel().get_obs_of_type("Thing").count(), + expect_objects, + "{policy:?}: object synthesis" + ); + assert_eq!( + stats.dropped.get(&DropReason::UnresolvedEndpoint).copied(), + Some(expect_drops).filter(|d| *d > 0), + "{policy:?}: drops {:?}", + stats.dropped + ); + assert_eq!( + report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::MissingEndpoint { .. })), + expect_error, + "{policy:?}: errors {:?}", + report.errors + ); + } +} + +// multi-value split: delimiter and regex. + +#[test] +fn case_5_multi_value_split_delimiter() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id INTEGER, activity TEXT, tags TEXT);") + .unwrap(); + con.execute("INSERT INTO events VALUES (1, 'A', 'a,b,c')", []) + .unwrap(); + con.execute("INSERT INTO events VALUES (2, 'B', 'b,d')", []) + .unwrap(); + } + + let node = source_node("events", "db", "events"); + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::constant("2020-01-01T00:00:00Z"), + attributes: vec![], + objects: vec![InlineObjectRef { + object: ObjectEndpoint { + id: col("tags"), + object_type: Some(constant("Tag")), + split: Some(SplitSpec { + kind: SplitKind::Delimiter { + delimiter: ",".into(), + }, + trim: true, + }), + }, + qualifier: Some(constant("tag")), + }], + }, + }); + let mut bp = blank_blueprint(vec![node], vec![mapping]); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "INTEGER", false), + ("activity", "TEXT", false), + ("tags", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + + assert_eq!( + ocel.get_obs_of_type("Tag").count(), + 4, + "distinct tags a,b,c,d" + ); + let e2o_total: usize = ocel.get_all_evs().map(|e| ocel.get_e2o(e).count()).sum(); + assert_eq!(e2o_total, 5, "3 tags for event 1 + 2 tags for event 2"); +} + +#[test] +fn case_5_multi_value_split_regex() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id INTEGER, activity TEXT, tags TEXT);") + .unwrap(); + con.execute("INSERT INTO events VALUES (1, 'A', 'x1;y22')", []) + .unwrap(); + } + + let node = source_node("events", "db", "events"); + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::constant("2020-01-01T00:00:00Z"), + attributes: vec![], + objects: vec![InlineObjectRef { + object: ObjectEndpoint { + id: col("tags"), + object_type: Some(constant("Field")), + split: Some(SplitSpec { + kind: SplitKind::Regex { + pattern: "[a-z][0-9]+".into(), + }, + trim: true, + }), + }, + qualifier: Some(constant("field")), + }], + }, + }); + let mut bp = blank_blueprint(vec![node], vec![mapping]); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "INTEGER", false), + ("activity", "TEXT", false), + ("tags", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + assert_eq!(ocel.get_obs_of_type("Field").count(), 2, "x1 and y22"); +} + +// object attributes: static (single value) and change-tracked (timed history). + +#[test] +fn case_6_static_object_attribute_is_single_valued_not_per_row() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE events (case_id TEXT, activity TEXT, ts TEXT, region TEXT);", + ) + .unwrap(); + for (i, activity) in ["create", "approve", "close"].iter().enumerate() { + con.execute( + "INSERT INTO events VALUES ('A', ?1, ?2, 'EU')", + params![activity, format!("2020-01-0{}T00:00:00Z", i + 1)], + ) + .unwrap(); + } + } + + let bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![AttributeMapping { + source_column: "region".into(), + name: "region".into(), + value_type: None, + }], + event_attributes: vec![], + }); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("region", "TEXT", true), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + let case = ocel.get_ob_by_id("A").expect("case A"); + let history = case + .get_attribute_value("region", ocel) + .expect("region history"); + assert_eq!(history.len(), 1, "one value, not one per row"); + assert_eq!(history[0].1, OCELAttributeValue::String("EU".into())); +} + +#[test] +fn case_6_change_tracked_object_attribute_accumulates_a_timed_history() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE status_changes (doc_id TEXT, ts TEXT, status TEXT);") + .unwrap(); + for (ts, status) in [ + ("2020-01-01T00:00:00Z", "draft"), + ("2020-01-02T00:00:00Z", "open"), + ("2020-01-03T00:00:00Z", "closed"), + ] { + con.execute( + "INSERT INTO status_changes VALUES ('D1', ?1, ?2)", + params![ts, status], + ) + .unwrap(); + } + } + + let node = source_node("status_changes", "db", "status_changes"); + let mapping = MappingEntry::Single(Mapping { + node: "status_changes".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Doc"), + id: col("doc_id"), + timestamp: Some(TimestampSource::column("ts")), + attributes: vec![AttributeMapping { + source_column: "status".into(), + name: "status".into(), + value_type: None, + }], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "status_changes", + [ + ("doc_id", "TEXT", false), + ("ts", "TEXT", false), + ("status", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + + assert_eq!(ocel.get_obs_of_type("Doc").count(), 1); + let doc = ocel.get_ob_by_id("D1").expect("doc D1"); + let mut history: Vec = doc + .get_attribute_value("status", ocel) + .expect("status history") + .iter() + .map(|(_, v)| v.to_string()) + .collect(); + history.sort_unstable(); + assert_eq!(history, vec!["closed", "draft", "open"]); + assert_eq!( + report.per_mapping[0].deduplicated, 2, + "2nd and 3rd rows repeat the id" + ); +} + +// event attributes, typed from the catalog and explicitly overridden. + +#[test] +fn case_7_event_attributes_typed_and_overridden() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id INTEGER, activity TEXT, ts TEXT, amount TEXT);") + .unwrap(); + con.execute( + "INSERT INTO events VALUES (1, 'A', '2020-01-01T00:00:00Z', '100.5')", + [], + ) + .unwrap(); + } + + let node = source_node("events", "db", "events"); + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![ + AttributeMapping { + source_column: "amount".into(), + name: "amount_num".into(), + value_type: None, + }, + AttributeMapping { + source_column: "amount".into(), + name: "amount_raw".into(), + value_type: Some(OCELAttributeType::String), + }, + ], + objects: vec![], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + // amount declared NUMERIC in the catalog, so amount_num infers Float; amount_raw overrides + // to String explicitly. + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "INTEGER", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("amount", "NUMERIC", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + let ocel = sink.ocel(); + let ev = ocel.get_ev_by_id("1").expect("event 1"); + assert_eq!( + ev.get_attribute_value("amount_num", ocel), + Some(&OCELAttributeValue::Float(100.5)), + "inferred from the catalog's NUMERIC column type" + ); + assert_eq!( + ev.get_attribute_value("amount_raw", ocel), + Some(&OCELAttributeValue::String("100.5".into())), + "explicit String override" + ); +} + +// every DropReason, one fixture each. + +#[test] +fn case_8_unresolved_endpoint_is_dropped_and_counted() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE rels (event_id TEXT, object_id TEXT);") + .unwrap(); + con.execute("INSERT INTO rels VALUES ('nope', 'thing-1')", []) + .unwrap(); + } + let node = source_node("rels", "db", "rels"); + let mapping = MappingEntry::Single(Mapping { + node: "rels".into(), + label: None, + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("event_id"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("object_id"), + object_type: Some(constant("Thing")), + split: None, + }, + qualifier: None, + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "rels", + [("event_id", "TEXT", false), ("object_id", "TEXT", false)], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + report.per_mapping[0] + .dropped + .get(&DropReason::UnresolvedEndpoint), + Some(&1) + ); +} + +#[test] +fn case_8_unparseable_timestamp_is_dropped_and_counted() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE events (id INTEGER, activity TEXT, ts TEXT);") + .unwrap(); + con.execute("INSERT INTO events VALUES (1, 'A', 'not-a-date')", []) + .unwrap(); + } + let node = source_node("events", "db", "events"); + let mapping = MappingEntry::Single(Mapping { + node: "events".into(), + label: None, + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("id", "INTEGER", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + report.per_mapping[0] + .dropped + .get(&DropReason::UnparseableTimestamp), + Some(&1) + ); + assert_eq!(sink.ocel().get_all_evs().count(), 0); +} + +#[test] +fn case_8_null_id_is_dropped_and_counted() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE objs (id TEXT);").unwrap(); + con.execute("INSERT INTO objs (id) VALUES (NULL)", []) + .unwrap(); + } + let node = source_node("objs", "db", "objs"); + let mapping = MappingEntry::Single(Mapping { + node: "objs".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Thing"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = + ExtractionCatalog::new().with_table("db", TableSchema::new("objs", [("id", "TEXT", true)])); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + report.per_mapping[0] + .dropped + .get(&DropReason::NullOrUnrenderableId), + Some(&1) + ); + assert_eq!(sink.ocel().get_obs_of_type("Thing").count(), 0); +} + +#[test] +fn case_8_predicate_excluded_is_dropped_and_counted() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE objs (id INTEGER, kind TEXT);") + .unwrap(); + con.execute("INSERT INTO objs VALUES (1, 'no')", []) + .unwrap(); + } + let node = source_node("objs", "db", "objs"); + let mapping = MappingEntry::Single(Mapping { + node: "objs".into(), + label: None, + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("yes".into()), + }, + }), + target: Target::Object { + object_type: constant("Thing"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("objs", [("id", "INTEGER", false), ("kind", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + report.per_mapping[0] + .dropped + .get(&DropReason::PredicateExcluded), + Some(&1) + ); +} + +// policy matrix. + +fn o2o_policy_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE sources (id TEXT); + CREATE TABLE rels (src TEXT, dst TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO sources VALUES ('s1')", []) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('s1', 'd1')", []) + .unwrap(); + } + let sources_node = source_node("sources", "db", "sources"); + let rels_node = source_node("rels", "db", "rels"); + let source_mapping = MappingEntry::Single(Mapping { + node: "sources".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("A"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let o2o_mapping = MappingEntry::Single(Mapping { + node: "rels".into(), + label: Some("o2o".into()), + when: None, + target: Target::O2O { + source: ObjectEndpoint { + id: col("src"), + object_type: Some(constant("A")), + split: None, + }, + target: ObjectEndpoint { + id: col("dst"), + object_type: Some(constant("B")), + split: None, + }, + qualifier: None, + }, + }); + let bp = blank_blueprint( + vec![sources_node, rels_node], + vec![source_mapping, o2o_mapping], + ); + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("sources", [("id", "TEXT", false)])) + .with_table( + "db", + TableSchema::new("rels", [("src", "TEXT", false), ("dst", "TEXT", false)]), + ); + (fx, bp, catalog) +} + +#[test] +fn case_9_missing_endpoint_drop() { + let (fx, mut bp, catalog) = o2o_policy_fixture(); + bp.on_missing_endpoint = MissingEndpointPolicy::Drop; + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + sink.ocel().get_obs_of_type("B").count(), + 0, + "target never created" + ); + let o2o_stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("o2o")) + .unwrap(); + assert_eq!( + o2o_stats.dropped.get(&DropReason::UnresolvedEndpoint), + Some(&1) + ); + assert!(report.errors.is_empty()); +} + +#[test] +fn case_9_missing_endpoint_create() { + let (fx, mut bp, catalog) = o2o_policy_fixture(); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + sink.ocel().get_obs_of_type("B").count(), + 1, + "target synthesised" + ); + assert!(sink.ocel().get_ob_by_id("d1").is_some()); + let o2o_stats = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("o2o")) + .unwrap(); + assert!(o2o_stats.dropped.is_empty()); +} + +#[test] +fn case_9_missing_endpoint_error() { + let (fx, mut bp, catalog) = o2o_policy_fixture(); + bp.on_missing_endpoint = MissingEndpointPolicy::Error; + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!(sink.ocel().get_obs_of_type("B").count(), 0); + assert!(report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::MissingEndpoint { id, .. } if id == "d1"))); +} + +fn duplicate_object_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE dups (id TEXT);").unwrap(); + con.execute("INSERT INTO dups VALUES ('x')", []).unwrap(); + con.execute("INSERT INTO dups VALUES ('x')", []).unwrap(); + } + let node = source_node("dups", "db", "dups"); + let mapping = MappingEntry::Single(Mapping { + node: "dups".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Dup"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![node], vec![mapping]); + let catalog = ExtractionCatalog::new() + .with_table("db", TableSchema::new("dups", [("id", "TEXT", false)])); + (fx, bp, catalog) +} + +#[test] +fn case_9_duplicate_object_first_wins() { + let (fx, mut bp, catalog) = duplicate_object_fixture(); + bp.on_duplicate_object = DuplicateObjectPolicy::FirstWins; + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!(sink.ocel().get_obs_of_type("Dup").count(), 1); + assert_eq!(report.per_mapping[0].deduplicated, 1); + assert!(report.errors.is_empty()); +} + +#[test] +fn case_9_duplicate_object_error() { + let (fx, mut bp, catalog) = duplicate_object_fixture(); + bp.on_duplicate_object = DuplicateObjectPolicy::Error; + assert_eq!(validate(&bp, &catalog), vec![]); + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + sink.ocel().get_obs_of_type("Dup").count(), + 1, + "the first still succeeds" + ); + assert!(report + .errors + .iter() + .any(|e| matches!(e, ExtractionError::DuplicateObject { id, .. } if id == "x"))); +} + +// Literal coercion actually runs. + +#[test] +fn case_10_text_literal_coerces_against_integer_column() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE docs (id INTEGER, docstatus INTEGER);") + .unwrap(); + con.execute("INSERT INTO docs VALUES (1, 1)", []).unwrap(); + con.execute("INSERT INTO docs VALUES (2, 0)", []).unwrap(); + } + let source = source_node("docs", "db", "docs"); + let filter = Node { + id: "filtered".into(), + label: None, + op: NodeOp::Filter { + input: "docs".into(), + // Authored the way an editor's text input would: docstatus = "1", a text literal, + // against an INTEGER column. Without coercion this matches nothing. + condition: Predicate::Compare { + left: Operand::Column { + column: "docstatus".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("1".into()), + }, + }, + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "filtered".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Doc"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![source, filter], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "docs", + [("id", "INTEGER", false), ("docstatus", "INTEGER", false)], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert_eq!( + sink.ocel().get_obs_of_type("Doc").count(), + 1, + "coercion must make docstatus=1 match" + ); + assert!(sink.ocel().get_ob_by_id("1").is_some()); +} + +// both sinks agree, on the case 1 and case 4 fixtures. + +/// Run `bp`/`catalog` against `fx`'s data through both `SlimOcelSink` and `DuckDbSink` -- +/// simultaneously, via [`differential::run_against_both`], so an auto-generated event id (as +/// `Blueprint::from_flat_event_table`'s event mapping produces, having no `id` expression) is +/// minted once and shared rather than independently randomized per sink, and assert their +/// [`differential::snapshot`]s are equal. Reused by both case 11 tests rather than +/// inlined twice; step 3's compiler differential test is expected to build its own `OcelSnapshot` +/// and compare it the same way. +#[cfg(feature = "ocel-duckdb")] +fn assert_both_sinks_agree(fx: &Fixture, bp: &Blueprint, catalog: &ExtractionCatalog) { + use super::differential::{run_against_both, snapshot}; + use super::duckdb_sink::DuckDbSink; + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut slim_sink = SlimOcelSink::new(); + + let db_dir = tempdir().expect("tempdir"); + let db_path = db_dir.path().join("out.duckdb"); + let mut duck_sink = DuckDbSink::new(&db_path).expect("open duckdb sink"); + + run_against_both(&mut slim_sink, &mut duck_sink, |sink| { + extract(bp, catalog, &providers, sink).expect("extract") + }); + + let slim_snapshot = snapshot(slim_sink.ocel()); + let con = duckdb::Connection::open(&db_path).expect("reopen duckdb file"); + let duck_ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(&con) + .expect("read duckdb back"); + let duck_snapshot = snapshot(&duck_ocel); + + assert_eq!( + slim_snapshot, duck_snapshot, + "SlimOcelSink and DuckDbSink must produce the same events, objects and relations" + ); +} + +/// Run `bp`/`catalog` against `fx`'s data through both sinks in separate `extract` calls, so +/// `DuckDbSink` answers every resolution [`Resolution::Deferred`] and the extractor takes its +/// deferral branches, the half of the sink-agreement invariant a fan-out run cannot reach (see +/// [`differential`]'s module docs). Returns both reports, eager first. +#[cfg(feature = "ocel-duckdb")] +fn assert_both_sinks_agree_on_separate_runs( + fx: &Fixture, + bp: &Blueprint, + catalog: &ExtractionCatalog, +) -> ( + super::report::ExtractionReport, + super::report::ExtractionReport, +) { + use super::differential::{extract_separately, snapshot}; + use super::duckdb_sink::DuckDbSink; + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + + let db_dir = tempdir().expect("tempdir"); + let db_path = db_dir.path().join("out.duckdb"); + let mut slim = SlimOcelSink::new(); + let mut duck = DuckDbSink::new(&db_path).expect("open duckdb sink"); + let (slim_report, duck_report) = + extract_separately(bp, catalog, &providers, &mut slim, &mut duck).expect("extract"); + assert_eq!( + slim_report.finalize, + super::sink::FinalizeReport::default(), + "an eager sink defers nothing" + ); + + let con = duckdb::Connection::open(&db_path).expect("reopen duckdb file"); + let duck_ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(&con) + .expect("read duckdb back"); + assert_eq!( + snapshot(slim.ocel()), + snapshot(&duck_ocel), + "deferred resolution must produce the same log as eager resolution" + ); + + // The log snapshot is content, not counters, so it cannot catch two sinks agreeing on what + // they wrote while disagreeing on how they characterise it. `deduplicated` is "the sink + // already had this entity", which an eager sink answers at `resolve_object` and a + // deferring one at `add_object`, so it must be identical per mapping regardless of which sink + // produced it. + assert_eq!( + slim_report + .per_mapping + .iter() + .map(|m| (m.mapping.index, m.deduplicated)) + .collect::>(), + duck_report + .per_mapping + .iter() + .map(|m| (m.mapping.index, m.deduplicated)) + .collect::>(), + "both sinks must agree on each mapping's deduplicated count" + ); + + (slim_report, duck_report) +} + +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_both_sinks_agree_on_flat_event_table() { + let (fx, bp, catalog) = case1_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + assert_both_sinks_agree(&fx, &bp, &catalog); +} + +/// exercising the path the fan-out harness cannot: two separate `extract` calls, one per +/// sink, so `DuckDbSink` answers every endpoint [`Resolution::Deferred`] and resolves it by join +/// at finalize while `SlimOcelSink` resolves eagerly. Only a blueprint whose ids are all +/// author-given is comparable this way, since a minted event id differs between runs by design, +/// which is why this uses the C3 fixture rather than case 1's. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_a_deferring_sink_agrees_with_an_eager_one_on_separate_runs() { + let (fx, catalog, mappings, nodes) = c3_fixture(); + let bp = blank_blueprint(nodes, mappings); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (_, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + assert_eq!( + duck_report.finalize.resolved_relations, 1, + "the O2O relation's deferred endpoints both resolved at finalize" + ); + assert_eq!(duck_report.finalize.unresolved_endpoints, 0); +} + +/// `on_missing_endpoint: Error` must produce errors under a deferring sink too. It produced +/// none at all: the extractor pushes [`ExtractionError::MissingEndpoint`] where it resolves an +/// endpoint, and a deferring sink resolves none: it counts and deletes them at finalize, where +/// the mapping and the row are long gone. The policy silently degraded to `Drop`. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i6_the_error_policy_reports_unresolved_endpoints_under_a_deferring_sink() { + use super::duckdb_sink::DuckDbSink; + + let (fx, mut bp, catalog) = i5_fixture(e2o_target(), "rels"); + bp.on_missing_endpoint = MissingEndpointPolicy::Error; + assert_eq!(validate(&bp, &catalog), vec![]); + + let (eager_sink, eager) = i5_run(&bp, &catalog, &fx); + drop(eager_sink); + let eager_errors = eager + .errors + .iter() + .filter(|e| matches!(e, ExtractionError::MissingEndpoint { .. })) + .count(); + assert_eq!( + eager_errors, 2, + "one per unresolved endpoint: {:?}", + eager.errors + ); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let db_dir = tempdir().expect("tempdir"); + let mut duck = DuckDbSink::new(db_dir.path().join("out.duckdb")).expect("open duckdb sink"); + let deferred = extract(&bp, &catalog, &providers, &mut duck).expect("extract to duckdb"); + + assert_eq!(deferred.finalize.unresolved_endpoints, 2); + assert!( + deferred + .errors + .contains(&ExtractionError::MissingEndpointsAtFinalize { count: 2 }), + "a deferring sink reports the same policy violation in bulk: {:?}", + deferred.errors + ); +} + +/// `on_missing_endpoint: Create` must synthesise the same objects under both sinks. `run_e2o` +/// returns before it looks at the object endpoint when the event endpoint does not resolve, so +/// the eager sink never creates that row's object; a deferring sink resolves no endpoint eagerly, +/// staged it, and created it at finalize even though the relation was deleted immediately after. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_create_synthesises_the_same_objects_under_both_sinks() { + let (fx, mut bp, catalog) = i5_fixture(e2o_target(), "rels"); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + assert_eq!(validate(&bp, &catalog), vec![]); + + let (_, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + assert_eq!( + duck_report.finalize.objects_created, 1, + "only the endpoint of the row whose event resolves is created" + ); +} + +/// `Create`'s `O2O` half, and the case the endpoint gate regressed: a relation row whose source +/// exists nowhere but that row. +/// +/// The eager path creates the source, then resolves the target against a log that now contains +/// it, then creates the target too. A deferring sink cannot: it gates the target's ask on the +/// source id, and evaluated that gate against `objects` from inside the very `INSERT INTO objects` +/// that creates the source. The source was therefore never there, the target ask was ruled +/// unreachable, the target was never created, and the relation was deleted for want of it. The +/// gate has to test the set the statement is about to produce, not the one it reads. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_create_synthesises_both_ends_of_a_relation_only_o2o() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE links (src TEXT, tgt TEXT);") + .unwrap(); + con.execute("INSERT INTO links VALUES ('s1', 't1')", []) + .unwrap(); + } + let o2o = MappingEntry::Single(Mapping { + node: "links".into(), + label: Some("o2o".into()), + when: None, + target: Target::O2O { + source: ObjectEndpoint { + id: col("src"), + object_type: Some(constant("Src")), + split: None, + }, + target: ObjectEndpoint { + id: col("tgt"), + object_type: Some(constant("Tgt")), + split: None, + }, + qualifier: Some(constant("q")), + }, + }); + let mut bp = blank_blueprint(vec![source_node("links", "db", "links")], vec![o2o]); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("links", [("src", "TEXT", false), ("tgt", "TEXT", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (_, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + assert_eq!( + duck_report.finalize.objects_created, 2, + "both endpoints are synthesised, the target no less than the source" + ); + assert_eq!(duck_report.finalize.resolved_relations, 1); + assert_eq!(duck_report.finalize.unresolved_endpoints, 0); +} + +/// The `O2O` half, and the reason the endpoint gate exists at all. Moved here from +/// `duckdb_sink`'s own test module, where it drove `DuckDbSink` alone, pinning the sink under test +/// rather than the reference semantics, and gave its "unresolvable" source a type that `Create` +/// can create, so the eager path did reach the target ask and the asserted type was wrong. +/// +/// A source endpoint whose type expression evaluates to `NULL` is genuinely unresolvable under +/// `Create`: there is nothing to create it as, so `run_o2o` gives up before it looks at any +/// target. The deferring sink still staged that target ask, naming `shared`, an id a real `E2O` +/// also names later and under a different type. Ungated, the earlier, unreachable ask wins +/// `arg_min` and the object is created as `Ghost` where the eager sink makes it an `Order`. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_an_o2o_target_ask_for_an_unresolvable_source_does_not_win_the_type() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE evs (id TEXT, activity TEXT, ts TEXT); + CREATE TABLE links (src TEXT, src_type TEXT, tgt TEXT, tgt_type TEXT); + CREATE TABLE e2orels (ev TEXT, ob TEXT);", + ) + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e1', 'A', '2020-01-01T00:00:00Z')", + [], + ) + .unwrap(); + // `tgt_type` is read from the row rather than written as a constant so that no mapping + // declares `Ghost` statically: an object type declared but left without a single object + // does not survive the `DuckDB` round trip, which would fail the snapshot for a reason + // that has nothing to do with the gate under test. + con.execute( + "INSERT INTO links VALUES ('no-type', NULL, 'shared', 'Ghost')", + [], + ) + .unwrap(); + con.execute("INSERT INTO e2orels VALUES ('e1', 'shared')", []) + .unwrap(); + } + // Mapping order fixes the ask order: `Phase::Relations` runs its nodes in first-seen order, so + // the `O2O`'s target ask is staged before the `E2O`'s and would win an ungated `arg_min`. + let o2o = MappingEntry::Single(Mapping { + node: "links".into(), + label: Some("o2o".into()), + when: None, + target: Target::O2O { + source: ObjectEndpoint { + id: col("src"), + object_type: Some(col("src_type")), + split: None, + }, + target: ObjectEndpoint { + id: col("tgt"), + object_type: Some(col("tgt_type")), + split: None, + }, + qualifier: Some(constant("q")), + }, + }); + let e2o = MappingEntry::Single(Mapping { + node: "e2orels".into(), + label: Some("e2o".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("ev"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(constant("Order")), + split: None, + }, + qualifier: Some(constant("uses")), + }, + }); + let events = MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let mut bp = blank_blueprint( + vec![ + source_node("links", "db", "links"), + source_node("e2orels", "db", "e2orels"), + source_node("evs", "db", "evs"), + ], + vec![o2o, e2o, events], + ); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "links", + [ + ("src", "TEXT", false), + ("src_type", "TEXT", true), + ("tgt", "TEXT", false), + ("tgt_type", "TEXT", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new("e2orels", [("ev", "TEXT", false), ("ob", "TEXT", false)]), + ) + .with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (slim_report, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + assert_eq!( + duck_report.finalize.objects_created, 1, + "only `shared`, and only through the E2O ask" + ); + let slim_sink = { + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + sink + }; + assert_eq!( + slim_sink + .ocel() + .get_ob_by_id("shared") + .map(|o| o.get_ob_type(slim_sink.ocel()).to_string()), + Some("Order".to_string()), + "the eager oracle: the O2O's target ask is never made, so `Ghost` never applies: {:?}", + slim_report.errors + ); +} + +/// Two object mappings naming the same id, the second carrying an attribute (the lazy-declaration shape, and +/// an entirely ordinary blueprint). The second mapping's `resolve_object` answers `Exists` to an +/// eager sink and `Deferred` to a deferring one; the deferring path used to treat the ensuing +/// `add_object` rejection as a deduplication and throw the attributes away, so the same blueprint +/// produced an object with a `note` under one sink and without it under the other. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_a_deferring_sink_writes_a_second_mapping_s_attributes_too() { + let (fx, bp, catalog) = dynamic_type_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + + // That the second mapping's `note` survives is what the helper's snapshot comparison pins: the + // eager run writes it (`dynamic_type_a_type_gaining_an_attribute_after_an_object_exists_does_not_abort`) + // and the deferring run must produce the same log. The counter is asserted for the shape of + // the answer: the `add_object` rejection is a deduplication under both sinks, and taking that + // path must not cost the attributes. + let (_, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + let with_attr = duck_report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("with_attr")) + .expect("stats"); + assert_eq!( + with_attr.deduplicated, 1, + "the sink already had the id, and the attributes are written anyway" + ); +} + +#[cfg(feature = "ocel-duckdb")] +#[test] +fn case_11_both_sinks_agree_on_join() { + let (fx, bp, catalog) = case4_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + assert_both_sinks_agree(&fx, &bp, &catalog); +} + +// a Source -> Filter chain streams and never materialises; a Join does. +// +// Asserted on `ExtractionReport::rows_materialized` (see graph.rs's +// `GraphExecutor::rows_materialized`) rather than on process-wide memory, which is not +// attributable to one `extract` call when tests run concurrently. + +#[test] +fn case_12_source_filter_chain_does_not_materialize_rows() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE small_src (id INTEGER, val INTEGER);") + .unwrap(); + for i in 0..10i64 { + con.execute( + "INSERT INTO small_src (id, val) VALUES (?1, ?2)", + params![i, i], + ) + .unwrap(); + } + } + + let source = source_node("small_src", "db", "small_src"); + let filter = Node { + id: "small".into(), + label: None, + op: NodeOp::Filter { + input: "small_src".into(), + condition: Predicate::Compare { + left: Operand::Column { + column: "val".into(), + }, + op: CompareOp::Lt, + right: Operand::Literal { + value: Literal::Integer(5), + }, + }, + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "small".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Row"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![source, filter], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "small_src", + [("id", "INTEGER", false), ("val", "INTEGER", false)], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!( + sink.ocel().get_obs_of_type("Row").count(), + 5, + "correctness: val < 5" + ); + assert_eq!( + report.rows_materialized, 0, + "a Source -> Filter chain must stream: it should never call \ + GraphExecutor::materialize, so rows_materialized stays zero regardless of table size" + ); +} + +/// A [`RowProvider`] that generates `rows` rows on the fly and counts how many it has handed out +/// so far, so a test can ask "how much of the table had been read when the first entity came out +/// the other end?". +#[derive(Debug)] +struct CountingProvider { + rows: i64, + emitted: std::rc::Rc>, +} + +impl RowProvider for CountingProvider { + fn scan( + &self, + _table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[super::value::Value]) -> std::ops::ControlFlow<()>, + ) -> Result<(), super::provider::ProviderError> { + for i in 0..self.rows { + let vals: Vec = columns + .iter() + .map(|_| super::value::Value::Integer(i)) + .collect(); + self.emitted.set(self.emitted.get() + 1); + if f(&vals).is_break() { + break; + } + } + Ok(()) + } +} + +/// Wraps [`SlimOcelSink`] and records the provider's row counter the first time an object +/// reaches the sink. Every other call is passed straight through. +#[derive(Debug)] +struct FirstEntityWitness { + inner: SlimOcelSink, + emitted: std::rc::Rc>, + rows_read_at_first_object: Option, +} + +impl super::sink::ExtractionSink for FirstEntityWitness { + fn declare_event_type( + &mut self, + name: &str, + attrs: &[crate::core::event_data::object_centric::OCELTypeAttribute], + ) -> Result<(), super::sink::SinkError> { + self.inner.declare_event_type(name, attrs) + } + fn declare_object_type( + &mut self, + name: &str, + attrs: &[crate::core::event_data::object_centric::OCELTypeAttribute], + ) -> Result<(), super::sink::SinkError> { + self.inner.declare_object_type(name, attrs) + } + fn add_event( + &mut self, + event_type: &str, + time: chrono::DateTime, + id: &str, + attributes: &[(String, OCELAttributeValue)], + ) -> Result { + self.inner.add_event(event_type, time, id, attributes) + } + fn add_object( + &mut self, + object_type: &str, + id: &str, + attributes: &[( + String, + chrono::DateTime, + OCELAttributeValue, + )], + ) -> Result { + if self.rows_read_at_first_object.is_none() { + self.rows_read_at_first_object = Some(self.emitted.get()); + } + self.inner.add_object(object_type, id, attributes) + } + fn add_object_attribute( + &mut self, + object: &super::sink::ObjectRef, + name: &str, + time: chrono::DateTime, + value: OCELAttributeValue, + ) -> Result<(), super::sink::SinkError> { + self.inner.add_object_attribute(object, name, time, value) + } + fn resolve_event( + &mut self, + id: &str, + event_type: Option<&str>, + ) -> super::sink::Resolution { + self.inner.resolve_event(id, event_type) + } + fn resolve_object( + &mut self, + id: &str, + object_type: Option<&str>, + ) -> super::sink::Resolution { + self.inner.resolve_object(id, object_type) + } + fn finalize(&mut self) -> Result { + self.inner.finalize() + } + fn add_e2o( + &mut self, + event: &super::sink::EventRef, + object: &super::sink::ObjectRef, + qualifier: &str, + ) -> Result<(), super::sink::SinkError> { + self.inner.add_e2o(event, object, qualifier) + } + fn add_o2o( + &mut self, + source: &super::sink::ObjectRef, + target: &super::sink::ObjectRef, + qualifier: &str, + ) -> Result<(), super::sink::SinkError> { + self.inner.add_o2o(source, target, qualifier) + } +} + +/// witnessed from outside the code under test. +/// +/// `rows_materialized == 0` (asserted by the test above) is a counter the executor maintains +/// about itself: it says "materialize was never called", which is true and useful but is the +/// implementation grading its own homework. This asserts the observable consequence instead, and +/// nothing in the extractor participates in the measurement: a provider counts the rows it has +/// handed out, a sink records that counter when the first object reaches it, and if the chain +/// streams the first object must arrive after exactly one row. A buffering implementation would +/// have had to pull all ten thousand first. +#[test] +fn case_12_the_first_entity_arrives_after_one_row_not_after_the_whole_table() { + const ROWS: i64 = 10_000; + let emitted = std::rc::Rc::new(std::cell::Cell::new(0u64)); + let provider = CountingProvider { + rows: ROWS, + emitted: emitted.clone(), + }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + + let source = source_node("big", "db", "big"); + let filter = Node { + id: "kept".into(), + label: None, + op: NodeOp::Filter { + input: "big".into(), + condition: Predicate::Compare { + left: Operand::Column { + column: "val".into(), + }, + op: CompareOp::Ge, + right: Operand::Literal { + value: Literal::Integer(0), + }, + }, + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "kept".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Row"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![source, filter], vec![mapping]); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("big", [("id", "INTEGER", false), ("val", "INTEGER", false)]), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let mut sink = FirstEntityWitness { + inner: SlimOcelSink::new(), + emitted: emitted.clone(), + rows_read_at_first_object: None, + }; + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!( + sink.rows_read_at_first_object, + Some(1), + "the first object must reach the sink after the provider has handed out exactly one \ + row; a buffering chain would have read all {ROWS} first" + ); + assert_eq!( + emitted.get(), + ROWS as u64, + "and the whole table is still read" + ); + assert_eq!(report.rows_materialized, 0); + assert_eq!( + sink.inner.ocel().get_obs_of_type("Row").count(), + ROWS as usize + ); +} + +#[test] +fn case_12_join_reports_materialized_rows() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE orders (id INTEGER, amount INTEGER); + CREATE TABLE meta (id INTEGER, region TEXT);", + ) + .unwrap(); + for (id, amount) in [(1, 100), (2, 200), (3, 300)] { + con.execute( + "INSERT INTO orders (id, amount) VALUES (?1, ?2)", + params![id, amount], + ) + .unwrap(); + } + for (id, region) in [(1, "EU"), (2, "US")] { + con.execute( + "INSERT INTO meta (id, region) VALUES (?1, ?2)", + params![id, region], + ) + .unwrap(); + } + } + + let left = source_node("orders", "db", "orders"); + let right = source_node("meta", "db", "meta"); + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "orders".into(), + right: "meta".into(), + on: vec![("id".into(), "id".into())], + }, + }; + let mapping = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("right_id"), + timestamp: None, + attributes: vec![], + }, + }); + let bp = blank_blueprint(vec![left, right, join], vec![mapping]); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("amount", "INTEGER", false)], + ), + ) + .with_table( + "db", + TableSchema::new( + "meta", + [("id", "INTEGER", false), ("region", "TEXT", false)], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!( + sink.ocel().get_obs_of_type("Order").count(), + 2, + "inner join drops unmatched id=3" + ); + assert!( + report.rows_materialized > 0, + "a Join executed here holds its right input, so rows_materialized should be nonzero \ + (metric is not trivially always zero)" + ); +} + +// Code review closure: sink-caveats findings. + +/// Fixture for finding 1: an `E2O` whose object type is read from a column, not a constant, so +/// two rows can stage the same object id under two different types. Row 1 names a missing event; +/// row 2 names the same object under a real one. Only row 2's ask is one the eager path ever +/// makes: `run_e2o` gives up on row 1 before it looks at the object endpoint, because the event +/// does not resolve. +#[cfg(feature = "ocel-duckdb")] +fn finding1_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE evs (id TEXT, activity TEXT, ts TEXT); + CREATE TABLE rels (ev TEXT, ob TEXT, obtype TEXT);", + ) + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e1', 'A', '2020-01-01T00:00:00Z')", + [], + ) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('missing', 'x', 'T1')", []) + .unwrap(); + con.execute("INSERT INTO rels VALUES ('e1', 'x', 'T2')", []) + .unwrap(); + } + let events = MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: col("activity"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }); + let rels = MappingEntry::Single(Mapping { + node: "rels".into(), + label: Some("rels".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("ev"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(col("obtype")), + split: None, + }, + qualifier: Some(constant("uses")), + }, + }); + let mut bp = blank_blueprint( + vec![ + source_node("evs", "db", "evs"), + source_node("rels", "db", "rels"), + ], + vec![events, rels], + ); + bp.on_missing_endpoint = MissingEndpointPolicy::Create; + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ) + .with_table( + "db", + TableSchema::new( + "rels", + [ + ("ev", "TEXT", false), + ("ob", "TEXT", false), + ("obtype", "TEXT", false), + ], + ), + ); + (fx, bp, catalog) +} + +/// Finding 1: `DuckDbSink::resolve_deferred`'s `arg_min` used to range over every staged ask for +/// an id, including the ones `finding1_fixture` sets up on purpose, so it picked `T1` (staged +/// first, from the row whose event never resolves) for an id the eager path only ever asks about +/// under `T2`. +/// +/// This runs `DuckDbSink` alone, pinning the type `arg_min` picks. The second divergence this +/// fixture used to trigger, the `rels` mapping's own `deduplicated` count differing between the +/// sinks because `resolve_object_endpoint` recorded the ghost ask at row-processing time and +/// nothing at finalize could undo it, is gone with the per-mapping id set, and +/// `both_sinks_agree_on_deduplicated_where_a_ghost_ask_used_to_split_them` runs this +/// fixture through the full agreement helper to keep it that way. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn finding1_arg_min_ignores_asks_the_eager_path_never_makes() { + use super::duckdb_sink::DuckDbSink; + + let (fx, bp, catalog) = finding1_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let db_dir = tempdir().expect("tempdir"); + let db_path = db_dir.path().join("out.duckdb"); + let mut duck = DuckDbSink::new(&db_path).expect("open duckdb sink"); + let report = extract(&bp, &catalog, &providers, &mut duck).expect("extract to duckdb"); + assert_eq!( + report.finalize.objects_created, 1, + "only the id reachable through the surviving (e1, x) relation is created" + ); + + let con = duckdb::Connection::open(&db_path).expect("reopen duckdb file"); + let ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(&con) + .expect("read duckdb back"); + let x = ocel.objects.iter().find(|o| o.id == "x").expect("object x"); + assert_eq!( + x.object_type, "T2", + "the type must come from the ask reachable through the surviving (e1, x) relation, not \ + the one staged for the row whose event ('missing') never resolves" + ); +} + +/// Finding 2: two `Target::Object` mappings on the same node name the same id under two +/// different types (only possible under `IdRendering::Raw`). An eager sink's `resolve_object` +/// answers `Missing` for the second mapping (the id is taken by another type), so its `add_object` +/// failure is unconditionally an `IdTypeCollision`. A deferring sink cannot answer that at +/// `resolve_object` time, so it always takes the "maybe I already own this id" branch, and used to +/// treat the `add_object` rejection that follows as "the id already exists, append to it", +/// writing the second mapping's attribute onto the first mapping's object instead of reporting +/// a collision. +/// +/// Runs `DuckDbSink` alone rather than through `assert_both_sinks_agree_on_separate_runs`: its +/// full snapshot comparison also flags that a type declared with zero objects (`Item` here, once +/// its one candidate object collides and is dropped) does not round-trip through `DuckDB`, which +/// derives `object_types()` from `DISTINCT ocel_type` over the `objects` table rather than a +/// persisted declaration. That is a separate, pre-existing limitation this fixture happens to +/// trigger, not the content-merge bug this test pins. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn finding2_a_deferring_sink_does_not_merge_a_type_collision() { + use super::duckdb_sink::DuckDbSink; + + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE things (id TEXT, kind TEXT, note TEXT);") + .unwrap(); + con.execute("INSERT INTO things VALUES ('1', 'order', NULL)", []) + .unwrap(); + con.execute("INSERT INTO things VALUES ('1', 'item', 'n')", []) + .unwrap(); + } + let of_kind = |kind: &str, object_type: &str, attributes: Vec| { + MappingEntry::Single(Mapping { + node: "things".into(), + label: Some(object_type.to_string()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant(object_type), + id: col("id"), + timestamp: None, + attributes, + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("things", "db", "things")], + vec![ + of_kind("order", "Order", vec![]), + of_kind( + "item", + "Item", + vec![AttributeMapping { + source_column: "note".into(), + name: "note".into(), + value_type: None, + }], + ), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "things", + [ + ("id", "TEXT", false), + ("kind", "TEXT", false), + ("note", "TEXT", true), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let db_dir = tempdir().expect("tempdir"); + let db_path = db_dir.path().join("out.duckdb"); + let mut duck = DuckDbSink::new(&db_path).expect("open duckdb sink"); + let report = extract(&bp, &catalog, &providers, &mut duck).expect("extract to duckdb"); + + let item = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("Item")) + .expect("Item stats"); + assert_eq!( + item.dropped.get(&DropReason::IdTypeCollision), + Some(&1), + "a deferring sink must report the same collision an eager one does: {:?}", + item.dropped + ); + + let con = duckdb::Connection::open(&db_path).expect("reopen duckdb file"); + let ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(&con) + .expect("read duckdb back"); + let order = ocel.objects.iter().find(|o| o.id == "1").expect("object 1"); + assert_eq!(order.object_type, "Order"); + assert!( + order.attributes.is_empty(), + "the colliding Item mapping's 'note' attribute must not be merged onto the surviving \ + Order object: {:?}", + order.attributes + ); +} + +/// Finding 8: `Blueprint::from_flat_event_table`'s case-object mapping now creates the case +/// objects (the objects pass runs before the events pass), rather than merely finding them +/// already created by the event mapping's inline `Create` reference. Pins the counter the +/// three-phase split corrected: before it, this mapping's `entities_emitted` was 0, because the +/// event mapping's own inline reference (also `Create`) always ran first and created the object, +/// leaving the case mapping nothing to do but append. +#[test] +fn finding8_case_object_mapping_creates_the_case_objects_not_the_event_mapping() { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE events (case_id TEXT, activity TEXT, ts TEXT, region TEXT);", + ) + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('c1', 'A', '2020-01-01T00:00:00Z', 'east')", + [], + ) + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('c1', 'B', '2020-01-01T01:00:00Z', 'east')", + [], + ) + .unwrap(); + con.execute( + "INSERT INTO events VALUES ('c2', 'A', '2020-01-01T02:00:00Z', 'west')", + [], + ) + .unwrap(); + } + let bp = Blueprint::from_flat_event_table(FlatEventTable { + source_id: "db".into(), + table: "events".into(), + case_id: "case_id".into(), + activity: "activity".into(), + timestamp: "ts".into(), + case_object_type: "Case".into(), + case_attributes: vec![AttributeMapping { + source_column: "region".into(), + name: "region".into(), + value_type: None, + }], + event_attributes: vec![], + }); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "events", + [ + ("case_id", "TEXT", false), + ("activity", "TEXT", false), + ("ts", "TEXT", false), + ("region", "TEXT", false), + ], + ), + ); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + let cases = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("cases")) + .expect("cases stats"); + assert_eq!( + cases.entities_emitted, 2, + "the objects pass creates the 2 distinct cases; the event mapping only finds them" + ); + assert_eq!(sink.ocel().get_obs_of_type("Case").count(), 2); +} + +// `compile` refuses an invalid blueprint, exactly as `extract` does. + +/// A one-node, one-mapping blueprint that compiles clean, plus the catalog it needs. The base +/// every case below perturbs into something `validate` rejects. +fn compilable_blueprint() -> (Blueprint, ExtractionCatalog) { + let bp = blank_blueprint( + vec![source_node("docs", "db", "docs")], + vec![MappingEntry::Single(Mapping { + node: "docs".into(), + label: Some("orders".into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("docs", [("id", "TEXT", false), ("kind", "TEXT", false)]), + ); + (bp, catalog) +} + +fn compile_default(bp: &Blueprint, catalog: &ExtractionCatalog) -> super::compile::CompiledOcel { + super::compile::compile( + bp, + catalog, + super::compile::SqlDialect::default(), + super::compile::EmissionShape::PerType, + ) +} + +/// `extraction_compile`'s binding docs promise it "refuses to run an invalid blueprint", and +/// `extract` does exactly that. `compile` did not: the bindings deserialize a `Blueprint` with +/// plain serde rather than `Blueprint::from_json`, so the version check lives only in +/// `validate`, and a `{"version": 2, ...}` blueprint whose new constructs are additive fields +/// was silently compiled under a v1 reading with zero entries in `errors`. +#[test] +fn i1_compile_refuses_a_blueprint_from_a_future_model_version() { + let (mut bp, catalog) = compilable_blueprint(); + bp.version = super::MODEL_VERSION + 1; + assert!(!validate(&bp, &catalog).is_empty()); + + let compiled = compile_default(&bp, &catalog); + assert!( + !compiled.errors().is_empty(), + "a future-version blueprint must be reported, not compiled under a v1 reading" + ); + assert!( + compiled.relations().is_empty(), + "nothing may be emitted for a blueprint this build cannot read: {:?}", + compiled.relations() + ); +} + +/// Duplicate node ids used to read differently in two places: `Blueprint::node` takes the first +/// match, `full_node_schemas` lets the last win. `node_sql_inner` then took the op from one and the +/// columns from another and emitted wrong SQL rather than none. `validate` rejects the duplicate; +/// `compile` has to act on that. +#[test] +fn i1_compile_refuses_duplicate_node_ids() { + let (mut bp, catalog) = compilable_blueprint(); + bp.nodes.push(Node { + id: "docs".into(), + label: None, + op: NodeOp::Source { + source_id: "db".into(), + table: "other".into(), + }, + }); + assert!(!validate(&bp, &catalog).is_empty()); + + let compiled = compile_default(&bp, &catalog); + assert!( + !compiled.errors().is_empty(), + "a duplicate node id must be reported rather than compiled from two different nodes" + ); + assert!( + compiled.relations().is_empty(), + "{:?}", + compiled.relations() + ); +} + +/// A blueprint naming a table the catalog does not have degrades to a complete but permanently +/// empty view set. A caller that does not read `errors()` gets a silent empty log; the failure +/// has to be in `errors`. +#[test] +fn i1_compile_refuses_an_unknown_table() { + let (bp, _) = compilable_blueprint(); + let catalog = ExtractionCatalog::new(); + assert!(!validate(&bp, &catalog).is_empty()); + + let compiled = compile_default(&bp, &catalog); + assert!( + !compiled.errors().is_empty(), + "an unknown table must be reported, not compiled into an empty view set" + ); + assert!( + compiled.relations().is_empty(), + "{:?}", + compiled.relations() + ); +} + +/// The complement: a blueprint that does validate still compiles exactly as before, so adding +/// the precondition check did not turn a working compile into a rejection. +#[test] +fn i1_compile_still_emits_views_for_a_valid_blueprint() { + let (bp, catalog) = compilable_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let compiled = compile_default(&bp, &catalog); + assert!(compiled.errors().is_empty(), "{:?}", compiled.errors()); + assert!(!compiled.relations().is_empty()); +} + +// the two sinks must agree on declared types and attribute values, not only on entities. + +/// A declared type with zero matching rows is part of the log: `extract` declares every +/// statically-named type up front so the declared type set is a function of the blueprint alone, +/// not of which rows happen to match. `case_2`'s `CreditNote` is such a type, and the fixture used +/// to run through `SlimOcelSink` only. +/// +/// This is a known divergence, not closable from this module. The consolidated `DuckDB` layout has +/// nowhere to record a declared object type: `declare_object_type` has no table to write to, and +/// `read_ocel_from_duckdb` rebuilds the type list from `SELECT DISTINCT ocel_type FROM objects`. +/// A type with no rows therefore cannot survive the round trip. Closing it needs an +/// `object_attr_meta(object_type, attr_name, attr_type)` table in +/// `ocel_sql::duckdb::schema::tables` and a declared-types union in that module's reader, i.e. a +/// change to the crate's on-disk OCEL 2.0 `DuckDB` format, which the SQL compiler's +/// `EmissionShape::Consolidated` output would have to emit as well. +/// +/// This test pins the divergence exactly: everything except the declared type set agrees, and +/// the only difference is the zero-entity type. It fails the moment either half changes. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i3_a_declared_type_with_zero_entities_is_lost_by_the_duckdb_layout() { + let (fx, bp, catalog) = case2_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + let (slim, duck) = snapshot_both_sinks(&fx, &bp, &catalog); + + assert_eq!(slim.events, duck.events, "events agree"); + assert_eq!(slim.objects, duck.objects, "objects agree"); + assert_eq!(slim.event_types, duck.event_types); + assert_eq!( + slim.object_types.keys().collect::>(), + vec!["Bill", "CreditNote", "Invoice"], + "the extractor declares every statically-named type, matched or not" + ); + assert_eq!( + duck.object_types.keys().collect::>(), + vec!["Bill", "Invoice"], + "the DuckDB reader rebuilds types from the rows that exist, so CreditNote is lost" + ); +} + +/// One attribute name declared under two different types by two different event types. +/// The extractor reconciles by `(kind, type_name, attr_name)`, so `A.n` stays an integer. +/// +/// Known divergence, not closable from this module. The consolidated layout stores event +/// attributes as *one wide column per attribute name*, shared by every event type, so `A.n` and +/// `B.n` are one `VARCHAR` column and the integer is read back as `String("5")`. The declared +/// types themselves round-trip correctly (`event_attr_meta` is keyed per type); only the value +/// does not. Closing it needs `read_ocel_from_duckdb` to narrow each wide-column value back to +/// its `(event_type, attr_name)` declared type, again a change to the shared reader. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i4_an_attribute_declared_under_two_event_types_is_widened_by_the_duckdb_layout() { + let (fx, bp, catalog) = retyped_event_attribute_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + let (slim, duck) = snapshot_both_sinks(&fx, &bp, &catalog); + + assert_eq!( + slim.event_types, duck.event_types, + "the per-type declarations do round-trip" + ); + assert_eq!(slim.objects, duck.objects); + assert_eq!( + slim.events["e1"].attributes, + vec![("n".to_string(), OCELAttributeValue::Integer(5))], + "the extractor keeps A.n an integer" + ); + assert_eq!( + duck.events["e1"].attributes, + vec![("n".to_string(), OCELAttributeValue::String("5".into()))], + "the shared wide column for 'n' is VARCHAR, because B.n is a string" + ); + assert_eq!( + slim.events["e2"].attributes, duck.events["e2"].attributes, + "the string-typed event type is unaffected" + ); +} + +/// Both sinks' logs, for a test that needs to inspect a specific divergence rather than assert +/// wholesale equality. +#[cfg(feature = "ocel-duckdb")] +fn snapshot_both_sinks( + fx: &Fixture, + bp: &Blueprint, + catalog: &ExtractionCatalog, +) -> ( + super::differential::OcelSnapshot, + super::differential::OcelSnapshot, +) { + use super::differential::{run_against_both, snapshot}; + use super::duckdb_sink::DuckDbSink; + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut slim_sink = SlimOcelSink::new(); + let db_dir = tempdir().expect("tempdir"); + let db_path = db_dir.path().join("out.duckdb"); + let mut duck_sink = DuckDbSink::new(&db_path).expect("open duckdb sink"); + run_against_both(&mut slim_sink, &mut duck_sink, |sink| { + extract(bp, catalog, &providers, sink).expect("extract") + }); + let slim = snapshot(slim_sink.ocel()); + let con = duckdb::Connection::open(&db_path).expect("reopen duckdb file"); + let duck_ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(&con) + .expect("read duckdb back"); + (slim, snapshot(&duck_ocel)) +} + +/// The same type declared by two mappings, so `declare_event_type` arrives twice for it. +/// `SlimOcelSink` keeps the last declaration; the `DuckDB` reader keeps the first row per +/// `(type, name)`, from a query with no `ORDER BY`. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i5_both_sinks_agree_when_one_type_is_declared_by_two_mappings() { + let (fx, bp, catalog) = twice_declared_type_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + assert_both_sinks_agree(&fx, &bp, &catalog); +} + +/// A `NULL` cell written as an object attribute. `to_sql_value` renders `Null` as `("", +/// "string")`, which `from_sql_value` reads back as `String("")` where the eager sink holds +/// `Null`; `DuckDbSink` now stores it under the `"null"` type string instead, so the value +/// round-trips exactly. +/// +/// The declared-type half remains a known divergence. `read_ocel_from_duckdb` derives an +/// object type's declared attributes from the `value_type` of the change rows it observes, not +/// from any declaration, so a name whose values span two types yields two entries for it. Same +/// shared-reader change. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i6_a_null_object_attribute_round_trips_as_null() { + let (fx, bp, catalog) = null_object_attribute_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + let (slim, duck) = snapshot_both_sinks(&fx, &bp, &catalog); + + assert_eq!( + slim.objects, duck.objects, + "every object attribute value, Null included, must survive the round trip" + ); + assert_eq!( + slim.objects["o1"].attributes[0].2, + OCELAttributeValue::Null, + "the NULL cell is a Null attribute, not String(\"\")" + ); + assert_eq!(slim.events, duck.events); + assert_eq!( + slim.object_types["Order"], + vec![("note".to_string(), "string".to_string())], + "the extractor declares note once, as the string the blueprint asked for" + ); + assert_eq!( + duck.object_types["Order"], + vec![ + ("note".to_string(), "null".to_string()), + ("note".to_string(), "string".to_string()) + ], + "the reader derives object attribute types from observed values, so a name whose \ + values span two types yields two entries" + ); +} + +/// `case_2`'s fixture and blueprint, factored out so the differential test above can reuse the +/// exact shape `case_2_discriminated_table_declares_zero_match_type` pins. +#[cfg(feature = "ocel-duckdb")] +fn case2_fixture_and_blueprint() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE docs (id INTEGER, kind TEXT);") + .unwrap(); + for (id, kind) in [(1, "invoice"), (2, "invoice"), (3, "bill")] { + con.execute( + "INSERT INTO docs (id, kind) VALUES (?1, ?2)", + params![id, kind], + ) + .unwrap(); + } + } + let object_mapping = |label: &str, kind: &str, object_type: &str| { + MappingEntry::Single(Mapping { + node: "docs".into(), + label: Some(label.into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Object { + object_type: constant(object_type), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("docs", "db", "docs")], + vec![ + object_mapping("invoices", "invoice", "Invoice"), + object_mapping("bills", "bill", "Bill"), + object_mapping("credit_notes", "credit_note", "CreditNote"), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("docs", [("id", "INTEGER", false), ("kind", "TEXT", false)]), + ); + (fx, bp, catalog) +} + +/// Two event types, `A` and `B`, both declaring an attribute `n`, `A` as an integer and `B` as a +/// string, with one `A` row carrying an integer value for it. +#[cfg(feature = "ocel-duckdb")] +fn retyped_event_attribute_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE evs (id TEXT, kind TEXT, n INTEGER, s TEXT, ts TEXT);") + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e1', 'A', 5, 'five', '2020-01-01T00:00:00Z')", + [], + ) + .unwrap(); + con.execute( + "INSERT INTO evs VALUES ('e2', 'B', 7, 'seven', '2020-01-02T00:00:00Z')", + [], + ) + .unwrap(); + } + let event_mapping = |label: &str, kind: &str, column: &str, value_type: OCELAttributeType| { + MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some(label.into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "kind".into(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(kind.into()), + }, + }), + target: Target::Event { + event_type: constant(kind), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![AttributeMapping { + source_column: column.into(), + name: "n".into(), + value_type: Some(value_type), + }], + objects: vec![], + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("evs", "db", "evs")], + vec![ + event_mapping("a_events", "A", "n", OCELAttributeType::Integer), + event_mapping("b_events", "B", "s", OCELAttributeType::String), + ], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "evs", + [ + ("id", "TEXT", false), + ("kind", "TEXT", false), + ("n", "INTEGER", false), + ("s", "TEXT", false), + ("ts", "TEXT", false), + ], + ), + ); + (fx, bp, catalog) +} + +/// One object type `Order` declared by two mappings on two nodes, each naming attribute `amount`, +/// so `declare_object_type` arrives twice for it, as it does for any type several mappings +/// produce. +#[cfg(feature = "ocel-duckdb")] +fn twice_declared_type_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE a (id TEXT, amount INTEGER); + CREATE TABLE b (id TEXT, amount INTEGER);", + ) + .unwrap(); + con.execute("INSERT INTO a VALUES ('o1', 10)", []).unwrap(); + con.execute("INSERT INTO b VALUES ('o2', 20)", []).unwrap(); + } + let object_mapping = |label: &str, node: &str, value_type: OCELAttributeType| { + MappingEntry::Single(Mapping { + node: node.into(), + label: Some(label.into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "amount".into(), + name: "amount".into(), + value_type: Some(value_type), + }], + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("a", "db", "a"), source_node("b", "db", "b")], + vec![ + object_mapping("from_a", "a", OCELAttributeType::Integer), + object_mapping("from_b", "b", OCELAttributeType::Integer), + ], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("a", [("id", "TEXT", false), ("amount", "INTEGER", false)]), + ) + .with_table( + "db", + TableSchema::new("b", [("id", "TEXT", false), ("amount", "INTEGER", false)]), + ); + (fx, bp, catalog) +} + +/// One object with a `NULL` attribute cell. +#[cfg(feature = "ocel-duckdb")] +fn null_object_attribute_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE obs (id TEXT, note TEXT);") + .unwrap(); + con.execute("INSERT INTO obs VALUES ('o1', NULL)", []) + .unwrap(); + con.execute("INSERT INTO obs VALUES ('o2', 'hello')", []) + .unwrap(); + } + let bp = blank_blueprint( + vec![source_node("obs", "db", "obs")], + vec![MappingEntry::Single(Mapping { + node: "obs".into(), + label: Some("orders".into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: "note".into(), + name: "note".into(), + value_type: Some(OCELAttributeType::String), + }], + }, + })], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new("obs", [("id", "TEXT", true), ("note", "TEXT", true)]), + ); + (fx, bp, catalog) +} + +// `entities_emitted` counts hand-offs to the sink, not survivors. + +/// The counter divergence `assert_both_sinks_agree_on_separate_runs`'s snapshot cannot see: both +/// sinks write the same log, but an eager sink refuses a dangling relation at the call site while +/// a deferring one writes it, counts it, and deletes it at finalize. +/// +/// Pins the exact numbers documented on [`MappingStats::entities_emitted`], and the identity that +/// makes them reconcilable: `emitted - unresolved_endpoints` is the same on both sides. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn i7_entities_emitted_counts_hand_offs_not_survivors_for_a_deferring_sink() { + let (fx, bp, catalog) = dangling_e2o_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (slim_report, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + + let emitted = |r: &super::report::ExtractionReport| { + r.per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("rels")) + .expect("rels stats") + .clone() + }; + let eager = emitted(&slim_report); + let deferring = emitted(&duck_report); + + assert_eq!( + eager.entities_emitted, 0, + "an eager sink refuses the relation at the call site" + ); + assert_eq!( + eager.dropped.get(&DropReason::UnresolvedEndpoint), + Some(&1), + "and reports the loss against the mapping that caused it" + ); + assert_eq!(slim_report.finalize.unresolved_endpoints, 0); + + assert_eq!( + deferring.entities_emitted, 1, + "a deferring sink cannot refuse, so it counts the hand-off" + ); + assert_eq!( + deferring.dropped.get(&DropReason::UnresolvedEndpoint), + None, + "the loss is not attributable to a mapping once it is settled at finalize" + ); + assert_eq!( + duck_report.finalize.unresolved_endpoints, 1, + "it is reported in bulk here instead" + ); + + // The identity that makes the two reconcilable, and the reason the divergence is bounded. + let total = |r: &super::report::ExtractionReport| { + r.per_mapping + .iter() + .map(|m| m.entities_emitted) + .sum::() + - r.finalize.unresolved_endpoints + }; + assert_eq!( + total(&slim_report), + total(&duck_report), + "emitted minus unresolved must agree across sinks" + ); +} + +/// One real event, one real object, and one `E2O` row naming an event that does not exist. +#[cfg(feature = "ocel-duckdb")] +fn dangling_e2o_fixture() -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch( + "CREATE TABLE evs (id TEXT, ts TEXT); + CREATE TABLE obs (id TEXT); + CREATE TABLE rels (ev TEXT, ob TEXT);", + ) + .unwrap(); + con.execute("INSERT INTO evs VALUES ('e1', '2020-01-01T00:00:00Z')", []) + .unwrap(); + con.execute("INSERT INTO obs VALUES ('o1')", []).unwrap(); + con.execute("INSERT INTO rels VALUES ('no-such-event', 'o1')", []) + .unwrap(); + } + let bp = blank_blueprint( + vec![ + source_node("evs", "db", "evs"), + source_node("obs", "db", "obs"), + source_node("rels", "db", "rels"), + ], + vec![ + MappingEntry::Single(Mapping { + node: "evs".into(), + label: Some("events".into()), + when: None, + target: Target::Event { + event_type: constant("Pay"), + id: Some(col("id")), + timestamp: TimestampSource::column("ts"), + attributes: vec![], + objects: vec![], + }, + }), + MappingEntry::Single(Mapping { + node: "obs".into(), + label: Some("objects".into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + }), + MappingEntry::Single(Mapping { + node: "rels".into(), + label: Some("rels".into()), + when: None, + target: Target::E2O { + event: EventEndpoint { + id: col("ev"), + event_type: None, + }, + object: ObjectEndpoint { + id: col("ob"), + object_type: Some(constant("Order")), + split: None, + }, + qualifier: None, + }, + }), + ], + ); + let catalog = ExtractionCatalog::new() + .with_table( + "db", + TableSchema::new("evs", [("id", "TEXT", false), ("ts", "TEXT", false)]), + ) + .with_table("db", TableSchema::new("obs", [("id", "TEXT", false)])) + .with_table( + "db", + TableSchema::new("rels", [("ev", "TEXT", false), ("ob", "TEXT", false)]), + ); + (fx, bp, catalog) +} + +// The extractor holds no per-mapping id set: `deduplicated` is answered by the sink, and so is +// the first-wins rule for a repeated `(id, attribute, time)`. + +/// A [`RowProvider`] generating `rows` rows of a single `id` column, cycling through `distinct` +/// ids. Holds nothing itself. +#[derive(Debug)] +struct GeneratedIds { + rows: usize, + distinct: usize, +} + +impl RowProvider for GeneratedIds { + fn scan( + &self, + table: &str, + columns: &[&str], + f: &mut dyn FnMut(&[super::value::Value]) -> std::ops::ControlFlow<()>, + ) -> Result<(), super::provider::ProviderError> { + if table != "gen" { + return Err(super::provider::ProviderError::UnknownTable { + table: table.to_string(), + }); + } + for c in columns { + if *c != "id" { + return Err(super::provider::ProviderError::UnknownColumn { + table: table.to_string(), + column: (*c).to_string(), + }); + } + } + for i in 0..self.rows { + let cell = super::value::Value::Text(format!("id-{}", i % self.distinct)); + let row: Vec = + columns.iter().map(|_| cell.clone()).collect::>(); + if f(&row).is_break() { + return Ok(()); + } + } + Ok(()) + } +} + +/// One object id on 100k rows extracts cleanly, and every repeat after the first is counted as a +/// deduplication rather than lost. +#[test] +fn one_object_id_on_a_hundred_thousand_rows_extracts_cleanly() { + let bp = blank_blueprint( + vec![source_node("gen", "db", "gen")], + vec![MappingEntry::Single(Mapping { + node: "gen".into(), + label: Some("obs".into()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![], + }, + })], + ); + let catalog = + ExtractionCatalog::new().with_table("db", TableSchema::new("gen", [("id", "TEXT", false)])); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = GeneratedIds { + rows: 100_000, + distinct: 1, + }; + let mut providers: HashMap = HashMap::new(); + providers.insert("db".to_string(), &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + + assert_eq!(sink.ocel().get_obs_of_type("Order").count(), 1); + let stats = &report.per_mapping[0]; + assert_eq!(stats.entities_emitted, 1); + assert_eq!( + stats.deduplicated, 99_999, + "every row after the first named an object the sink already had" + ); + assert!(stats.dropped.is_empty(), "{:?}", stats.dropped); + assert!(report.errors.is_empty(), "{:?}", report.errors); +} + +/// Two static object mappings on one row, writing the same `(id, attribute, time)` with the same +/// value. First-wins now spans every mapping, because it lives in the sink rather than in one +/// mapping's own bookkeeping, so this stores one value, not two, and is not an error. +fn repeated_attribute_fixture(second_value: &str) -> (Fixture, Blueprint, ExtractionCatalog) { + let fx = Fixture::new(); + { + let con = fx.build(); + con.execute_batch("CREATE TABLE rows_ (id TEXT, a TEXT, b TEXT);") + .unwrap(); + con.execute( + "INSERT INTO rows_ VALUES ('x', 'first', ?1)", + params![second_value], + ) + .unwrap(); + } + let mapping = |label: &str, column: &str| { + MappingEntry::Single(Mapping { + node: "rows_".into(), + label: Some(label.to_string()), + when: None, + target: Target::Object { + object_type: constant("Order"), + id: col("id"), + timestamp: None, + attributes: vec![AttributeMapping { + source_column: column.to_string(), + name: "note".into(), + value_type: None, + }], + }, + }) + }; + let bp = blank_blueprint( + vec![source_node("rows_", "db", "rows_")], + vec![mapping("first", "a"), mapping("second", "b")], + ); + let catalog = ExtractionCatalog::new().with_table( + "db", + TableSchema::new( + "rows_", + [ + ("id", "TEXT", false), + ("a", "TEXT", false), + ("b", "TEXT", false), + ], + ), + ); + (fx, bp, catalog) +} + +/// a repeated `(id, attribute, time)` carrying an identical value is not an error, +/// and stores one value. +#[test] +fn a_repeated_attribute_with_the_same_value_stores_one_value() { + let (fx, bp, catalog) = repeated_attribute_fixture("first"); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert!(report.errors.is_empty(), "{:?}", report.errors); + + let ocel = sink.ocel(); + let obj = ocel.get_ob_by_id("x").expect("object x"); + let note = obj.get_attribute_value("note", ocel).expect("note history"); + assert_eq!(note.len(), 1, "one value per (id, attribute, time)"); + assert_eq!(note[0].1, OCELAttributeValue::String("first".into())); +} + +/// the same `(id, attribute, time)` written twice with different values. First +/// wins, and it is not an error. +/// +/// Which write is first is scan order, not a promise: nothing issues an `ORDER BY`, and mapping +/// execution order is `(phase, first-seen node position, mapping index)`. The rule is that exactly +/// one value survives and that a repeat costs neither an error nor a second entry, not that a +/// particular one of two conflicting values is chosen. Here both mappings sit on one node, which +/// is the one shape mapping order does decide (see `extract`'s own docs). +#[test] +fn a_repeated_attribute_with_a_different_value_is_first_wins() { + let (fx, bp, catalog) = repeated_attribute_fixture("second"); + assert_eq!(validate(&bp, &catalog), vec![]); + + let provider = fx.provider(); + let providers = providers_of("db", &provider); + let mut sink = SlimOcelSink::new(); + let report = extract(&bp, &catalog, &providers, &mut sink).expect("extract"); + assert!(report.errors.is_empty(), "{:?}", report.errors); + + let ocel = sink.ocel(); + let obj = ocel.get_ob_by_id("x").expect("object x"); + let note = obj.get_attribute_value("note", ocel).expect("note history"); + assert_eq!(note.len(), 1, "one value per (id, attribute, time)"); + assert_eq!( + note[0].1, + OCELAttributeValue::String("first".into()), + "the first write wins" + ); +} + +/// The first-wins rule has to hold in both sinks or the same blueprint produces two different +/// logs: `SlimOcelSink` scans the attribute's history, `DuckDbSink` lets a unique index reject +/// the insert, and only a differential run proves the two agree. Both the identical-value and the +/// conflicting-value fixtures, since they take different paths through `DuckDB`'s constraint +/// handling only in what the discarded row held. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn both_sinks_keep_exactly_one_value_per_id_attribute_time() { + for second_value in ["first", "second"] { + let (fx, bp, catalog) = repeated_attribute_fixture(second_value); + assert_eq!(validate(&bp, &catalog), vec![]); + assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + } +} + +/// The two sinks must agree on `deduplicated` on the +/// fixture that used to make them disagree. +/// +/// `finding1_fixture` interleaves a repeated object id with a row whose event never resolves. A +/// deferring sink cannot fail `resolve_event` at the call site, so it reached +/// `resolve_object_endpoint` on a row the eager path abandons earlier, and the per-mapping id set +/// that function touched had already recorded the ghost ask by the time `finalize` could tell the +/// relation would not survive. Nothing at finalize could undo it, so the `rels` mapping's +/// `deduplicated` differed between the sinks. With no such set, endpoint resolution counts nothing +/// on either side and the divergence is closed by construction. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn both_sinks_agree_on_deduplicated_where_a_ghost_ask_used_to_split_them() { + let (fx, bp, catalog) = finding1_fixture(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (slim_report, _) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + let rels = slim_report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("rels")) + .expect("rels stats"); + assert_eq!( + rels.deduplicated, 0, + "a relation mapping deduplicates nothing: it emits one relation per row" + ); +} + +/// The other half: a *cross-mapping* repeat, which the old per-mapping set could not see at all. +/// Two object mappings name one id; the second one finds an object the sink already has, which is +/// now a deduplication under both sinks. +#[cfg(feature = "ocel-duckdb")] +#[test] +fn a_second_mapping_finding_the_first_s_object_is_a_deduplication() { + let (fx, bp, catalog) = dynamic_type_fixture_and_blueprint(); + assert_eq!(validate(&bp, &catalog), vec![]); + + let (slim_report, duck_report) = assert_both_sinks_agree_on_separate_runs(&fx, &bp, &catalog); + for report in [&slim_report, &duck_report] { + let with_attr = report + .per_mapping + .iter() + .find(|m| m.mapping.label.as_deref() == Some("with_attr")) + .expect("stats"); + assert_eq!( + with_attr.deduplicated, 1, + "the sink already had this id, which is what `deduplicated` now counts" + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/validate.rs b/process_mining/src/core/event_data/object_centric/extraction/validate.rs new file mode 100644 index 00000000..37a21de8 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/validate.rs @@ -0,0 +1,1819 @@ +//! Check a blueprint against a catalog before anything runs. +//! +//! Every rule here is decidable from the blueprint plus declared schema, with no data access. +//! Catching these up front is what lets the extractor and the compiler agree: a blueprint the +//! two would interpret differently is rejected rather than run. + +use std::collections::{HashMap, HashSet}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::blueprint::{ + Blueprint, IdRendering, Mapping, MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Target, +}; +use super::catalog::{Catalog, ColumnSchema, TableSchema}; +use super::desugar::desugar_with_paths; +use super::expr::{SplitKind, TimestampSource, ValueExpression}; +use super::predicate::{Literal, Operand, Predicate}; +use super::value::ValueKind; +use super::MODEL_VERSION; + +/// A reason a blueprint cannot be executed or compiled. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum ValidationError { + /// The blueprint's `version` is newer than this build understands. + UnsupportedVersion { + /// The blueprint's version. + found: u32, + /// The newest version this build reads. + supported: u32, + }, + /// Two nodes share an id. + DuplicateNodeId { + /// The repeated id. + id: String, + }, + /// A node or mapping names a node that does not exist. + UnknownNodeRef { + /// Who referred to it. + from: String, + /// The missing id. + id: String, + }, + /// The node graph contains a cycle, so no evaluation order exists. + NodeCycle { + /// One node id participating in the cycle. + id: String, + }, + /// A source node names a source with no entry in the catalog. + UnknownSource { + /// The source id. + source_id: String, + }, + /// A source node names a table with no schema in the catalog. + UnknownTable { + /// The source id. + source_id: String, + /// The table name. + table: String, + }, + /// An expression reads a column absent from the declared schema. + UnknownColumn { + /// The node whose rows were being read. + node: String, + /// The column name. + column: String, + }, + /// Type prefixing is on, but a relation endpoint does not declare its type. + MissingTypeForPrefixing { + /// Which mapping, by label or index. + mapping: String, + /// Which endpoint. + endpoint: String, + }, + /// Missing endpoints are created, but an object endpoint does not declare its type. + MissingTypeForCreate { + /// Which mapping, by label or index. + mapping: String, + /// Which endpoint. + endpoint: String, + }, + /// A union has no inputs, so it has no columns to project. + EmptyUnion { + /// The node id. + node: String, + }, + /// A regular expression does not compile. + InvalidRegex { + /// The pattern. + pattern: String, + /// The compiler's message. + message: String, + }, + /// A `Template` expression has a placeholder that is unterminated or empty, either of which + /// drops every row instead of the intended substitution. + InvalidTemplate { + /// The template text. + template: String, + /// What is wrong with it. + reason: String, + }, + /// A `Join` output column resolves to no single input column: the left input has a column + /// literally named `right_` and the right input's `` is renamed onto it. + AmbiguousJoinColumn { + /// The `Join` node. + node: String, + /// The contested output column name. + column: String, + }, + /// A comparison literal cannot be read as the type its column declares, so the comparison + /// matches no row at all, however the data looks. + /// + /// Raised only where the column's declared type names a kind: an unrecognised `col_type` turns + /// coercion off on purpose, and then the literal is compared as authored. + UncoercibleLiteral { + /// The mapping, by label or authored path, or the `Filter` node, the comparison sits in. + location: String, + /// The column compared against. + column: String, + /// That column's declared type, verbatim from the catalog. + col_type: String, + /// The literal, as authored. + literal: String, + /// The kind `col_type` names: `text`, `integer`, `float`, `boolean` or `timestamp`. + expected: String, + }, + /// A comparison's two operands (both columns, or both literals) have different, statically + /// known kinds that [`Value::compare`](super::value::Value::compare) has no rule for (only + /// `integer`/`float` compare across kinds), so it matches no row at all. + IncomparableCompare { + /// The mapping, by label or authored path, or the `Filter` node, the comparison sits in. + location: String, + /// The left operand's declared kind: `text`, `integer`, `float`, `boolean` or `timestamp`. + left_kind: String, + /// The right operand's declared kind. + right_kind: String, + }, +} + +impl std::fmt::Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ValidationError::UnsupportedVersion { found, supported } => { + write!( + f, + "blueprint version {found} is newer than the supported version {supported}" + ) + } + ValidationError::DuplicateNodeId { id } => write!(f, "duplicate node id '{id}'"), + ValidationError::UnknownNodeRef { from, id } => { + write!(f, "'{from}' refers to unknown node '{id}'") + } + ValidationError::NodeCycle { id } => { + write!(f, "node '{id}' takes part in a cycle") + } + ValidationError::UnknownSource { source_id } => { + write!(f, "no catalog entry for source '{source_id}'") + } + ValidationError::UnknownTable { source_id, table } => { + write!(f, "no schema for table '{table}' in source '{source_id}'") + } + ValidationError::UnknownColumn { node, column } => { + write!(f, "node '{node}' has no column '{column}'") + } + ValidationError::MissingTypeForPrefixing { mapping, endpoint } => write!( + f, + "mapping {mapping}: {endpoint} needs a declared type because ids are type-prefixed" + ), + ValidationError::MissingTypeForCreate { mapping, endpoint } => write!( + f, + "mapping {mapping}: {endpoint} needs a declared type because missing endpoints are created" + ), + ValidationError::EmptyUnion { node } => write!(f, "union node '{node}' has no inputs"), + ValidationError::InvalidRegex { pattern, message } => { + write!(f, "invalid regular expression '{pattern}': {message}") + } + ValidationError::InvalidTemplate { template, reason } => { + write!(f, "invalid template '{template}': {reason}") + } + ValidationError::AmbiguousJoinColumn { node, column } => write!( + f, + "join '{node}': column '{column}' could be the left input's own column or the \ + renamed right-hand one" + ), + ValidationError::UncoercibleLiteral { + location, + column, + col_type, + literal, + expected, + } => write!( + f, + "{location}: '{literal}' cannot be read as {expected}, which is what column \ + '{column}' declares ({col_type}), so the comparison matches nothing" + ), + ValidationError::IncomparableCompare { + location, + left_kind, + right_kind, + } => write!( + f, + "{location}: comparing {left_kind} to {right_kind} matches nothing, since only \ + integer and float compare across kinds" + ), + } + } +} + +impl std::error::Error for ValidationError {} + +/// Check a blueprint against a catalog, returning every problem found. +/// +/// An empty result means the blueprint is executable and compilable. Errors are collected rather +/// than short-circuited so an editor can show all of them at once. +#[must_use] +pub fn validate(blueprint: &Blueprint, catalog: &dyn Catalog) -> Vec { + let mut errors = Vec::new(); + + if blueprint.version > MODEL_VERSION { + errors.push(ValidationError::UnsupportedVersion { + found: blueprint.version, + supported: MODEL_VERSION, + }); + return errors; + } + + let mut seen: HashSet<&str> = HashSet::new(); + for node in &blueprint.nodes { + if !seen.insert(node.id.as_str()) { + errors.push(ValidationError::DuplicateNodeId { + id: node.id.clone(), + }); + } + } + + for node in &blueprint.nodes { + for input in node_inputs(&node.op) { + if blueprint.node(input).is_none() { + errors.push(ValidationError::UnknownNodeRef { + from: node.id.clone(), + id: input.to_string(), + }); + } + } + match &node.op { + NodeOp::Source { source_id, table } => { + if !catalog.has_source(source_id) { + errors.push(ValidationError::UnknownSource { + source_id: source_id.clone(), + }); + } else if catalog.table(source_id, table).is_none() { + errors.push(ValidationError::UnknownTable { + source_id: source_id.clone(), + table: table.clone(), + }); + } + } + NodeOp::Union { inputs } if inputs.is_empty() => { + errors.push(ValidationError::EmptyUnion { + node: node.id.clone(), + }); + } + _ => {} + } + } + + errors.extend(cycles(blueprint)); + + let schemas = super::schema::full_node_schemas(blueprint, catalog); + let columns = node_columns(&schemas); + for node in &blueprint.nodes { + errors.extend(check_filter_columns(node, &columns)); + errors.extend(check_join_columns(node, &columns)); + errors.extend(check_join_ambiguity(node, &schemas)); + if let NodeOp::Filter { input, condition } = &node.op { + let location = format!("filter '{}'", node.id); + let input_schema = schemas.get(input.as_str()); + errors.extend(check_literal_kinds(&location, condition, input_schema)); + errors.extend(check_compare_kind_mismatch( + &location, + condition, + input_schema, + )); + } + } + + let desugared = desugar_with_paths(blueprint); + for pattern in all_regexes(blueprint, &desugared) { + if let Err(e) = regex::Regex::new(pattern) { + errors.push(ValidationError::InvalidRegex { + pattern: pattern.to_string(), + message: e.to_string(), + }); + } + } + + for (path, mapping) in &desugared { + errors.extend(check_mapping(blueprint, mapping, path, &columns)); + if let Some(when) = &mapping.when { + let name = mapping.label.clone().unwrap_or_else(|| path.to_string()); + let location = format!("mapping {name}"); + let node_schema = schemas.get(mapping.node.as_str()); + errors.extend(check_literal_kinds(&location, when, node_schema)); + errors.extend(check_compare_kind_mismatch(&location, when, node_schema)); + } + } + + errors +} + +/// The node ids an operation reads from. +fn node_inputs(op: &NodeOp) -> Vec<&str> { + match op { + NodeOp::Source { .. } => Vec::new(), + NodeOp::Filter { input, .. } => vec![input.as_str()], + NodeOp::Join { left, right, .. } => vec![left.as_str(), right.as_str()], + NodeOp::Union { inputs } => inputs.iter().map(String::as_str).collect(), + } +} + +/// Report every node on a cycle, but not the ones merely downstream of one: fixing the cycle +/// fixes those too. +fn cycles(blueprint: &Blueprint) -> Vec { + let deps: HashMap<&str, Vec<&str>> = blueprint + .nodes + .iter() + .map(|n| (n.id.as_str(), node_inputs(&n.op))) + .collect(); + + // Kahn's algorithm: whatever is left once nothing more can be ordered is in or downstream + // of a cycle, but the two are not yet told apart. + let mut remaining = deps.clone(); + loop { + let ready: Vec<&str> = remaining + .iter() + .filter(|(_, ds)| ds.iter().all(|d| !remaining.contains_key(d))) + .map(|(id, _)| *id) + .collect(); + if ready.is_empty() { + break; + } + for id in ready { + remaining.remove(id); + } + } + + let mut ids: Vec<&str> = remaining + .keys() + .copied() + .filter(|id| reaches_itself(id, &deps)) + .collect(); + ids.sort_unstable(); + ids.into_iter() + .map(|id| ValidationError::NodeCycle { id: id.to_string() }) + .collect() +} + +/// Whether following dependency edges from `start` leads back to `start`, i.e. whether `start` is +/// itself on a cycle rather than merely reading from one. +fn reaches_itself(start: &str, deps: &HashMap<&str, Vec<&str>>) -> bool { + let mut stack: Vec<&str> = deps.get(start).cloned().unwrap_or_default(); + let mut seen: HashSet<&str> = HashSet::new(); + while let Some(cur) = stack.pop() { + if cur == start { + return true; + } + if seen.insert(cur) { + stack.extend(deps.get(cur).into_iter().flatten().copied()); + } + } + false +} + +/// The columns each node produces, as far as the catalog can say, taken from +/// [`full_node_schemas`](super::schema::full_node_schemas) so that validation and execution cannot +/// disagree about a `Join`'s `right_` renaming. +/// +/// A node with no entry has unknown columns, and column checks against it are skipped. +fn node_columns<'a>(schemas: &HashMap<&'a str, TableSchema>) -> HashMap<&'a str, HashSet> { + schemas + .iter() + .map(|(id, schema)| (*id, schema.columns.keys().cloned().collect())) + .collect() +} + +/// Report every output column of a `Join` that resolves to neither input. See +/// [`join_column_source`](super::schema::join_column_source), which the executor uses for the same +/// decision. +/// +/// Reported whether or not anything reads the column, so the run cannot fail part way through on a +/// name the author never suspected. +fn check_join_ambiguity(node: &Node, schemas: &HashMap<&str, TableSchema>) -> Vec { + let NodeOp::Join { left, right, .. } = &node.op else { + return Vec::new(); + }; + let (Some(l), Some(r), Some(joined)) = ( + schemas.get(left.as_str()), + schemas.get(right.as_str()), + schemas.get(node.id.as_str()), + ) else { + return Vec::new(); + }; + joined + .columns + .keys() + .filter(|name| super::schema::join_column_source(name, l, r).is_none()) + .map(|column| ValidationError::AmbiguousJoinColumn { + node: node.id.clone(), + column: column.clone(), + }) + .collect() +} + +/// Report every literal a predicate compares against a column whose declared kind cannot hold it. +/// +/// [`Predicate::prepare`] falls back to the literal as authored when coercion fails, and that +/// never equals a cell of the column's kind. Such a comparison is empty on every row, which no +/// runtime signal distinguishes from a genuinely empty result. The shape this catches is +/// `created_at > "2019-01-01"` against a `DATE` column, since +/// [`Value::coerce_to`](super::value::Value::coerce_to) reads a timestamp as strict RFC 3339 only. +fn check_literal_kinds( + location: &str, + predicate: &Predicate, + schema: Option<&TableSchema>, +) -> Vec { + let Some(schema) = schema else { + return Vec::new(); + }; + let mut compared = Vec::new(); + collect_compared_literals(predicate, &mut compared); + compared + .into_iter() + .filter_map(|(column, literal)| { + let col = schema.columns.get(column)?; + // No declared kind means coercion is off for this column, and the literal is compared + // exactly as authored. + let kind = col.declared_kind()?; + let value = literal.as_value(); + if value.coerce_to(kind).is_some() { + return None; + } + Some(ValidationError::UncoercibleLiteral { + location: location.to_string(), + column: column.to_string(), + col_type: col.col_type.clone(), + literal: value.display_string().unwrap_or_default(), + expected: kind_name(kind).to_string(), + }) + }) + .collect() +} + +/// Report every `Compare` whose two operands are both columns, or both literals, with different +/// statically known kinds that do not coerce (only `Integer`/`Float` do). Unlike +/// [`check_literal_kinds`], a mismatched `Column`/`Literal` pair is not reported here: `prepare` +/// coerces the literal to the column's kind there, and [`check_literal_kinds`] already reports +/// when that coercion fails. +fn check_compare_kind_mismatch( + location: &str, + predicate: &Predicate, + schema: Option<&TableSchema>, +) -> Vec { + let mut out = Vec::new(); + collect_compare_kind_mismatches(location, predicate, schema, &mut out); + out +} + +fn collect_compare_kind_mismatches( + location: &str, + predicate: &Predicate, + schema: Option<&TableSchema>, + out: &mut Vec, +) { + match predicate { + Predicate::And { conditions } | Predicate::Or { conditions } => { + for c in conditions { + collect_compare_kind_mismatches(location, c, schema, out); + } + } + Predicate::Not { condition } => { + collect_compare_kind_mismatches(location, condition, schema, out); + } + Predicate::Compare { left, right, .. } => { + let kinds = match (left, right) { + (Operand::Column { column: l }, Operand::Column { column: r }) => Some(( + schema + .and_then(|s| s.columns.get(l)) + .and_then(ColumnSchema::declared_kind), + schema + .and_then(|s| s.columns.get(r)) + .and_then(ColumnSchema::declared_kind), + )), + (Operand::Literal { value: l }, Operand::Literal { value: r }) => { + Some((l.as_value().kind(), r.as_value().kind())) + } + _ => None, + }; + if let Some((Some(l), Some(r))) = kinds { + if l != r + && !matches!( + (l, r), + (ValueKind::Integer, ValueKind::Float) + | (ValueKind::Float, ValueKind::Integer) + ) + { + out.push(ValidationError::IncomparableCompare { + location: location.to_string(), + left_kind: kind_name(l).to_string(), + right_kind: kind_name(r).to_string(), + }); + } + } + } + Predicate::In { .. } + | Predicate::IsNull { .. } + | Predicate::IsEmpty { .. } + | Predicate::Matches { .. } => {} + } +} + +/// Every `(column, literal)` pair a predicate compares, at any depth under `And`/`Or`/`Not`. +fn collect_compared_literals<'a>(predicate: &'a Predicate, out: &mut Vec<(&'a str, &'a Literal)>) { + match predicate { + Predicate::And { conditions } | Predicate::Or { conditions } => { + for c in conditions { + collect_compared_literals(c, out); + } + } + Predicate::Not { condition } => collect_compared_literals(condition, out), + Predicate::Compare { left, right, .. } => match (left, right) { + (Operand::Column { column }, Operand::Literal { value }) + | (Operand::Literal { value }, Operand::Column { column }) => { + out.push((column.as_str(), value)); + } + _ => {} + }, + Predicate::In { column, values } => { + for value in values { + out.push((column.as_str(), value)); + } + } + Predicate::IsNull { .. } | Predicate::IsEmpty { .. } | Predicate::Matches { .. } => {} + } +} + +/// The name [`ValidationError::UncoercibleLiteral`] reports a [`ValueKind`] under. +fn kind_name(kind: ValueKind) -> &'static str { + match kind { + ValueKind::Text => "text", + ValueKind::Integer => "integer", + ValueKind::Float => "float", + ValueKind::Boolean => "boolean", + ValueKind::Timestamp => "timestamp", + } +} + +/// One [`ValidationError::UnknownColumn`] per name in `referenced` that `node`'s columns do not +/// have, in sorted order and once each. The one place that comparison is made. +fn missing_columns<'a>( + node: &str, + referenced: impl IntoIterator, + available: Option<&HashSet>, +) -> Vec { + let Some(available) = available else { + return Vec::new(); + }; + let mut missing: Vec<&str> = referenced + .into_iter() + .filter(|c| !available.contains(*c)) + .collect(); + missing.sort_unstable(); + missing.dedup(); + missing + .into_iter() + .map(|column| ValidationError::UnknownColumn { + node: node.to_string(), + column: column.to_string(), + }) + .collect() +} + +/// Check a `Join` node's key column pairs against each side's columns. +fn check_join_columns( + node: &Node, + columns: &HashMap<&str, HashSet>, +) -> Vec { + let NodeOp::Join { left, right, on } = &node.op else { + return Vec::new(); + }; + let mut errors = missing_columns( + left, + on.iter().map(|(l, _)| l.as_str()), + columns.get(left.as_str()), + ); + errors.extend(missing_columns( + right, + on.iter().map(|(_, r)| r.as_str()), + columns.get(right.as_str()), + )); + errors +} + +/// Check one desugared mapping. +/// +/// Only the column check needs the referenced node to exist, so it alone is skipped when it does +/// not. A bad node reference does not stop the regex, template or endpoint-type checks from +/// running too, so a mapping with several independent problems reports all of them. +fn check_mapping( + blueprint: &Blueprint, + mapping: &Mapping, + path: &str, + columns: &HashMap<&str, HashSet>, +) -> Vec { + let mut errors = Vec::new(); + let name = mapping.label.clone().unwrap_or_else(|| path.to_string()); + + let node_exists = blueprint.node(&mapping.node).is_some(); + if !node_exists { + errors.push(ValidationError::UnknownNodeRef { + from: name.clone(), + id: mapping.node.clone(), + }); + } + + if node_exists { + let mut referenced: HashSet<&str> = HashSet::new(); + if let Some(when) = &mapping.when { + when.referenced_columns(&mut referenced); + } + collect_target_columns(&mapping.target, &mut referenced); + errors.extend(missing_columns( + &mapping.node, + referenced, + columns.get(mapping.node.as_str()), + )); + } + + errors.extend(template_problems(&mapping.target)); + errors.extend(endpoint_rules(blueprint, mapping, &name)); + errors +} + +/// Check a `Filter` node's condition against its input node's columns, exactly as a mapping's +/// referenced columns are checked against the node it reads. +fn check_filter_columns( + node: &Node, + columns: &HashMap<&str, HashSet>, +) -> Vec { + let NodeOp::Filter { input, condition } = &node.op else { + return Vec::new(); + }; + let mut referenced: HashSet<&str> = HashSet::new(); + condition.referenced_columns(&mut referenced); + missing_columns(input, referenced, columns.get(input.as_str())) +} + +/// The `ObjectEndpoint`s a target names, in every position one can appear. +/// +/// `pub(crate)`: the executor's `mapping_exec` walks a target's endpoints in this exact order +/// when preparing (and later executing) each one's `Split`, so the two must agree on order. +pub(crate) fn target_object_endpoints(target: &Target) -> Vec<&ObjectEndpoint> { + match target { + Target::Event { objects, .. } => objects.iter().map(|o| &o.object).collect(), + Target::Object { .. } => Vec::new(), + Target::E2O { object, .. } => vec![object], + Target::O2O { source, target, .. } => vec![source, target], + } +} + +/// Every [`ValueExpression`] a [`TimestampSource`]'s parts read from. +fn timestamp_expressions(timestamp: &TimestampSource) -> Vec<&ValueExpression> { + match timestamp { + TimestampSource::Value(part) => vec![&part.source], + TimestampSource::Components { date, time } => [date, time] + .into_iter() + .flatten() + .map(|p| &p.source) + .collect(), + } +} + +/// Every [`ValueExpression`] position a target names (id, type, qualifier, timestamp), without +/// recursing into `Coalesce`'s parts. Callers needing to look inside one call the expression's own +/// methods. +fn target_value_expressions(target: &Target) -> Vec<&ValueExpression> { + let mut out = Vec::new(); + for endpoint in target_object_endpoints(target) { + out.push(&endpoint.id); + if let Some(t) = &endpoint.object_type { + out.push(t); + } + } + match target { + Target::Event { + event_type, + id, + timestamp, + objects, + .. + } => { + out.push(event_type); + if let Some(id) = id { + out.push(id); + } + out.extend(timestamp_expressions(timestamp)); + for o in objects { + if let Some(q) = &o.qualifier { + out.push(q); + } + } + } + Target::Object { + object_type, + id, + timestamp, + .. + } => { + out.push(object_type); + out.push(id); + if let Some(ts) = timestamp { + out.extend(timestamp_expressions(ts)); + } + } + Target::E2O { + event, qualifier, .. + } => { + out.push(&event.id); + if let Some(t) = &event.event_type { + out.push(t); + } + if let Some(q) = qualifier { + out.push(q); + } + } + Target::O2O { qualifier, .. } => { + if let Some(q) = qualifier { + out.push(q); + } + } + } + out +} + +/// Every `Template` defect in a target: an unterminated or an empty placeholder, both decidable +/// without row data. +fn template_problems(target: &Target) -> Vec { + let mut errors = Vec::new(); + for expr in target_value_expressions(target) { + collect_template_errors(expr, &mut errors); + } + errors +} + +/// Collect `Template` defects from `expr`, recursing into `Coalesce`'s parts. +fn collect_template_errors(expr: &ValueExpression, out: &mut Vec) { + match expr { + ValueExpression::Template { template } => { + if let Some(reason) = template_defect(template) { + out.push(ValidationError::InvalidTemplate { + template: template.clone(), + reason: reason.to_string(), + }); + } + } + ValueExpression::Coalesce { parts } => { + for p in parts { + collect_template_errors(p, out); + } + } + ValueExpression::Column { .. } | ValueExpression::Constant { .. } => {} + } +} + +/// Whether `template` has an unterminated or an empty placeholder. +fn template_defect(template: &str) -> Option<&'static str> { + let mut rest = template; + while let Some(open) = rest.find('{') { + let after = &rest[open + 1..]; + let Some(close) = after.find('}') else { + return Some("unterminated placeholder"); + }; + if after[..close].is_empty() { + return Some("empty placeholder"); + } + rest = &after[close + 1..]; + } + None +} + +/// Every regular expression a predicate contains: a `Matches` pattern, at any depth under +/// `And`/`Or`/`Not`. +fn collect_predicate_regexes<'a>(predicate: &'a Predicate, out: &mut Vec<&'a str>) { + match predicate { + Predicate::And { conditions } | Predicate::Or { conditions } => { + for c in conditions { + collect_predicate_regexes(c, out); + } + } + Predicate::Not { condition } => collect_predicate_regexes(condition, out), + Predicate::Matches { regex, .. } => out.push(regex.as_str()), + Predicate::Compare { .. } + | Predicate::IsNull { .. } + | Predicate::IsEmpty { .. } + | Predicate::In { .. } => {} + } +} + +/// Every regular expression in the blueprint that must compile to run it: a `Matches` pattern in +/// a node's `Filter` condition or a mapping's `when`, and a `Regex` split on any `ObjectEndpoint`. +/// One traversal driven from both node ops and mappings, so a new regex-bearing position only has +/// to be added here to be checked everywhere, rather than risking a second +/// hand-rolled walk that forgets a site the first one covers. +fn all_regexes<'a>(blueprint: &'a Blueprint, desugared: &'a [(String, Mapping)]) -> Vec<&'a str> { + let mut out = Vec::new(); + for node in &blueprint.nodes { + if let NodeOp::Filter { condition, .. } = &node.op { + collect_predicate_regexes(condition, &mut out); + } + } + for (_, m) in desugared { + if let Some(when) = &m.when { + collect_predicate_regexes(when, &mut out); + } + for endpoint in target_object_endpoints(&m.target) { + if let Some(split) = &endpoint.split { + if let SplitKind::Regex { pattern } = &split.kind { + out.push(pattern.as_str()); + } + } + } + } + out +} + +/// Collect every column a target reads into `out`. +/// +/// `pub(crate)`: also used by the executor's demand analysis (`schema::demanded_columns`), so a +/// `Source` node only ever projects the columns a mapping's target actually reads. +pub(crate) fn collect_target_columns<'a>(target: &'a Target, out: &mut HashSet<&'a str>) { + let endpoint = |e: &'a ObjectEndpoint, out: &mut HashSet<&'a str>| { + e.id.referenced_columns(out); + if let Some(t) = &e.object_type { + t.referenced_columns(out); + } + }; + match target { + Target::Event { + event_type, + id, + timestamp, + attributes, + objects, + } => { + event_type.referenced_columns(out); + if let Some(id) = id { + id.referenced_columns(out); + } + timestamp.referenced_columns(out); + for a in attributes { + out.insert(a.source_column.as_str()); + } + for o in objects { + endpoint(&o.object, out); + if let Some(q) = &o.qualifier { + q.referenced_columns(out); + } + } + } + Target::Object { + object_type, + id, + timestamp, + attributes, + } => { + object_type.referenced_columns(out); + id.referenced_columns(out); + if let Some(ts) = timestamp { + ts.referenced_columns(out); + } + for a in attributes { + out.insert(a.source_column.as_str()); + } + } + Target::E2O { + event, + object, + qualifier, + } => { + event.id.referenced_columns(out); + if let Some(t) = &event.event_type { + t.referenced_columns(out); + } + endpoint(object, out); + if let Some(q) = qualifier { + q.referenced_columns(out); + } + } + Target::O2O { + source, + target: tgt, + qualifier, + } => { + endpoint(source, out); + endpoint(tgt, out); + if let Some(q) = qualifier { + q.referenced_columns(out); + } + } + } +} + +/// Endpoints must declare their type when the id rendering or the missing-endpoint policy +/// needs it. Both rules are decidable from the blueprint alone, which is what replaced the +/// order-dependent `prefixed_types` state the original extractor carried. +fn endpoint_rules(blueprint: &Blueprint, mapping: &Mapping, name: &str) -> Vec { + let prefixing = blueprint.id_rendering == IdRendering::TypePrefixed; + let creating = blueprint.on_missing_endpoint == MissingEndpointPolicy::Create; + let mut errors = Vec::new(); + + let object = |e: &ObjectEndpoint, label: &'static str, errors: &mut Vec| { + if e.object_type.is_some() { + return; + } + if prefixing { + errors.push(ValidationError::MissingTypeForPrefixing { + mapping: name.to_string(), + endpoint: label.to_string(), + }); + } + if creating { + errors.push(ValidationError::MissingTypeForCreate { + mapping: name.to_string(), + endpoint: label.to_string(), + }); + } + }; + + match &mapping.target { + Target::E2O { + event, object: obj, .. + } => { + if prefixing && event.event_type.is_none() { + errors.push(ValidationError::MissingTypeForPrefixing { + mapping: name.to_string(), + endpoint: "event".to_string(), + }); + } + object(obj, "object", &mut errors); + } + Target::O2O { source, target, .. } => { + object(source, "source", &mut errors); + object(target, "target", &mut errors); + } + Target::Event { objects, .. } => { + for o in objects { + object(&o.object, "inline object", &mut errors); + } + } + Target::Object { .. } => {} + } + errors +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::extraction::blueprint::*; + use crate::core::event_data::object_centric::extraction::catalog::{ + ExtractionCatalog, TableSchema, + }; + use crate::core::event_data::object_centric::extraction::expr::{ + SplitKind, SplitSpec, ValueExpression, + }; + use crate::core::event_data::object_centric::extraction::predicate::{ + CompareOp, Literal, Operand, Predicate, + }; + + fn source(id: &str, table: &str) -> Node { + Node { + id: id.into(), + label: None, + op: NodeOp::Source { + source_id: "erp".into(), + table: table.into(), + }, + } + } + + fn object_mapping(node: &str, object_type: Option<&str>) -> MappingEntry { + MappingEntry::Single(Mapping { + node: node.into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Constant { + value: object_type.unwrap_or("Order").to_string(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }) + } + + fn bp(nodes: Vec, mappings: Vec) -> Blueprint { + Blueprint { + version: 1, + id_rendering: IdRendering::Raw, + nodes, + mappings, + on_missing_endpoint: MissingEndpointPolicy::Drop, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } + } + + fn catalog() -> ExtractionCatalog { + ExtractionCatalog::new().with_table( + "erp", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("state", "TEXT", true)], + ), + ) + } + + #[test] + fn a_valid_blueprint_reports_nothing() { + let b = bp(vec![source("o", "orders")], vec![object_mapping("o", None)]); + assert_eq!(validate(&b, &catalog()), vec![]); + } + + #[test] + fn rejects_a_future_version() { + let mut b = bp(vec![], vec![]); + b.version = 999; + assert!(matches!( + validate(&b, &catalog())[0], + ValidationError::UnsupportedVersion { .. } + )); + } + + #[test] + fn rejects_duplicate_node_ids() { + let b = bp(vec![source("o", "orders"), source("o", "orders")], vec![]); + assert!(matches!( + validate(&b, &catalog())[0], + ValidationError::DuplicateNodeId { .. } + )); + } + + #[test] + fn rejects_a_mapping_naming_an_unknown_node() { + let b = bp( + vec![source("o", "orders")], + vec![object_mapping("nope", None)], + ); + assert!(matches!( + validate(&b, &catalog())[0], + ValidationError::UnknownNodeRef { .. } + )); + } + + #[test] + fn rejects_a_cycle_in_the_node_graph() { + let a = Node { + id: "a".into(), + label: None, + op: NodeOp::Filter { + input: "b".into(), + condition: Predicate::And { conditions: vec![] }, + }, + }; + let b_node = Node { + id: "b".into(), + label: None, + op: NodeOp::Filter { + input: "a".into(), + condition: Predicate::And { conditions: vec![] }, + }, + }; + let b = bp(vec![a, b_node], vec![]); + assert!(validate(&b, &catalog()) + .iter() + .any(|e| matches!(e, ValidationError::NodeCycle { .. }))); + } + + #[test] + fn rejects_a_column_absent_from_the_catalog() { + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "not_a_column".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![source("o", "orders")], vec![m]); + assert!(validate(&b, &catalog()) + .iter() + .any(|e| matches!(e, ValidationError::UnknownColumn { .. }))); + } + + #[test] + fn type_prefixing_requires_every_relation_endpoint_to_declare_its_type() { + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::E2O { + event: EventEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + event_type: None, + }, + object: ObjectEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + object_type: None, + split: None, + }, + qualifier: None, + }, + }); + let mut b = bp(vec![source("o", "orders")], vec![m]); + assert_eq!(validate(&b, &catalog()), vec![]); + b.id_rendering = IdRendering::TypePrefixed; + let errs = validate(&b, &catalog()); + assert_eq!( + errs.iter() + .filter(|e| matches!(e, ValidationError::MissingTypeForPrefixing { .. })) + .count(), + 2, + "both the event and the object endpoint must be reported" + ); + } + + #[test] + fn create_policy_requires_object_endpoints_to_declare_their_type() { + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::E2O { + event: EventEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + event_type: None, + }, + object: ObjectEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + object_type: None, + split: None, + }, + qualifier: None, + }, + }); + let mut b = bp(vec![source("o", "orders")], vec![m]); + b.on_missing_endpoint = MissingEndpointPolicy::Create; + assert!(validate(&b, &catalog()) + .iter() + .any(|e| matches!(e, ValidationError::MissingTypeForCreate { .. }))); + } + + #[test] + fn rejects_an_uncompilable_regex_before_execution() { + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: Some(Predicate::Matches { + column: "state".into(), + regex: "([".into(), + }), + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![source("o", "orders")], vec![m]); + assert!(validate(&b, &catalog()) + .iter() + .any(|e| matches!(e, ValidationError::InvalidRegex { .. }))); + } + + #[test] + fn a_bad_column_inside_a_filter_condition_is_reported() { + // Regression: validate() used to match only Source and Union in its node loop, so a + // Filter's condition was never checked against its input's columns. + let filter = Node { + id: "recent".into(), + label: None, + op: NodeOp::Filter { + input: "o".into(), + condition: Predicate::Compare { + left: Operand::Column { + column: "stat".into(), // typo for "state" + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text("open".into()), + }, + }, + }, + }; + let b = bp( + vec![source("o", "orders"), filter], + vec![object_mapping("recent", None)], + ); + assert!(validate(&b, &catalog()).iter().any( + |e| matches!(e, ValidationError::UnknownColumn { column, .. } if column == "stat") + )); + } + + #[test] + fn a_bad_regex_inside_a_filter_condition_is_the_only_error() { + // Regression: bad_regexes was called only on mapping.when, never on a Filter condition. + let filter = Node { + id: "orders_filtered".into(), + label: None, + op: NodeOp::Filter { + input: "o".into(), + condition: Predicate::Matches { + column: "state".into(), + regex: "([".into(), + }, + }, + }; + let b = bp( + vec![source("o", "orders"), filter], + vec![object_mapping("orders_filtered", None)], + ); + let errs = validate(&b, &catalog()); + assert_eq!(errs.len(), 1, "got {errs:?}"); + assert!(matches!(errs[0], ValidationError::InvalidRegex { .. })); + } + + #[test] + fn a_bad_regex_inside_an_object_endpoint_split_is_the_only_error() { + // Regression: an uncompilable SplitSpec::Regex pattern inside an ObjectEndpoint.split + // was never visited by bad_regexes nor by collect_target_columns. + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::E2O { + event: EventEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + event_type: None, + }, + object: ObjectEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + object_type: None, + split: Some(SplitSpec { + kind: SplitKind::Regex { + pattern: "([".into(), + }, + trim: true, + }), + }, + qualifier: None, + }, + }); + let b = bp(vec![source("o", "orders")], vec![m]); + let errs = validate(&b, &catalog()); + assert_eq!(errs.len(), 1, "got {errs:?}"); + assert!(matches!(errs[0], ValidationError::InvalidRegex { .. })); + } + + #[test] + fn an_unknown_node_reference_does_not_suppress_other_errors_on_the_same_mapping() { + // D1: a mapping with both a bad node name and a missing endpoint type used to report + // only the first, because check_mapping returned early. + let m = MappingEntry::Single(Mapping { + node: "nope".into(), + label: None, + when: None, + target: Target::E2O { + event: EventEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + event_type: None, + }, + object: ObjectEndpoint { + id: ValueExpression::Column { + column: "id".into(), + }, + object_type: None, + split: None, + }, + qualifier: None, + }, + }); + let mut b = bp(vec![source("o", "orders")], vec![m]); + b.id_rendering = IdRendering::TypePrefixed; + let errs = validate(&b, &catalog()); + assert!( + errs.iter() + .any(|e| matches!(e, ValidationError::UnknownNodeRef { .. })), + "got {errs:?}" + ); + assert!( + errs.iter() + .any(|e| matches!(e, ValidationError::MissingTypeForPrefixing { .. })), + "endpoint-type checks must run even when the node reference is bad: got {errs:?}" + ); + } + + #[test] + fn a_node_only_downstream_of_a_cycle_is_not_reported_as_being_in_one() { + // D2: Kahn's algorithm leaves every node downstream of a cycle unordered too, so the + // old code reported c as "taking part in a cycle" despite only reading from it. + let a = Node { + id: "a".into(), + label: None, + op: NodeOp::Filter { + input: "b".into(), + condition: Predicate::And { conditions: vec![] }, + }, + }; + let b_node = Node { + id: "b".into(), + label: None, + op: NodeOp::Filter { + input: "a".into(), + condition: Predicate::And { conditions: vec![] }, + }, + }; + let c = Node { + id: "c".into(), + label: None, + op: NodeOp::Filter { + input: "a".into(), + condition: Predicate::And { conditions: vec![] }, + }, + }; + let bp = bp(vec![a, b_node, c], vec![]); + let errs = validate(&bp, &catalog()); + let cycle_nodes: Vec<&str> = errs + .iter() + .filter_map(|e| match e { + ValidationError::NodeCycle { id } => Some(id.as_str()), + _ => None, + }) + .collect(); + assert_eq!(cycle_nodes, vec!["a", "b"], "got {errs:?}"); + } + + #[test] + fn a_source_id_typo_reports_unknown_source_not_unknown_table() { + // D3: Catalog::table alone cannot tell a source-id typo apart from a known source with + // an unknown table. + let b = bp( + vec![Node { + id: "o".into(), + label: None, + op: NodeOp::Source { + source_id: "nope".into(), + table: "orders".into(), + }, + }], + vec![], + ); + let errs = validate(&b, &catalog()); + assert!(matches!( + errs[0], + ValidationError::UnknownSource { ref source_id } if source_id == "nope" + )); + } + + #[test] + fn a_known_source_with_an_unknown_table_still_reports_unknown_table() { + let b = bp( + vec![Node { + id: "o".into(), + label: None, + op: NodeOp::Source { + source_id: "erp".into(), + table: "missing".into(), + }, + }], + vec![], + ); + let errs = validate(&b, &catalog()); + assert!(matches!(errs[0], ValidationError::UnknownTable { .. })); + } + + #[test] + fn a_colliding_right_join_column_is_reachable_as_right_prefixed() { + // node_columns must model the same right_ prefix rule Join applies at run + // time, or a mapping reading the documented right_id spuriously fails validation. + let left = source("o", "orders"); + let right = Node { + id: "same_ids".into(), + label: None, + op: NodeOp::Source { + source_id: "erp".into(), + table: "orders".into(), // also has an "id" column: collides with the left side + }, + }; + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "o".into(), + right: "same_ids".into(), + on: vec![("id".into(), "id".into())], + }, + }; + let m = MappingEntry::Single(Mapping { + node: "joined".into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "right_id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![left, right, join], vec![m]); + assert_eq!(validate(&b, &catalog()), vec![]); + } + + #[test] + fn a_join_column_claimed_by_both_the_rename_and_a_real_left_column_is_reported() { + // The left side already has a column literally named `right_id`, and the right side's + // own `id` renames onto it. The executor resolves that to neither, so validation must + // reject it rather than let the run fail on a name the author never wrote. + let catalog = ExtractionCatalog::new() + .with_table( + "erp", + TableSchema::new( + "left_rows", + [("id", "INTEGER", false), ("right_id", "INTEGER", false)], + ), + ) + .with_table( + "erp", + TableSchema::new("right_rows", [("id", "INTEGER", false)]), + ); + let nodes = vec![ + source("l", "left_rows"), + Node { + id: "r".into(), + label: None, + op: NodeOp::Source { + source_id: "erp".into(), + table: "right_rows".into(), + }, + }, + Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "l".into(), + right: "r".into(), + on: vec![("id".into(), "id".into())], + }, + }, + ]; + let b = bp(nodes, vec![]); + assert!(validate(&b, &catalog).iter().any(|e| matches!( + e, + ValidationError::AmbiguousJoinColumn { node, column } + if node == "joined" && column == "right_id" + ))); + } + + #[test] + fn a_literal_that_cannot_be_read_as_its_column_s_type_is_reported() { + // `Value::coerce_to(Timestamp)` takes strict RFC 3339, so this comparison matches no row + // whatever the data holds, and at run time that is indistinguishable from an empty + // result. + let catalog = ExtractionCatalog::new().with_table( + "erp", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("created_at", "DATE", false)], + ), + ); + let guarded = |value: Literal| { + bp( + vec![source("o", "orders")], + vec![MappingEntry::Single(Mapping { + node: "o".into(), + label: Some("orders".into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: "created_at".into(), + }, + op: CompareOp::Gt, + right: Operand::Literal { value }, + }), + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + })], + ) + }; + + let errors = validate(&guarded(Literal::Text("2019-01-01".into())), &catalog); + assert!( + errors.iter().any(|e| matches!( + e, + ValidationError::UncoercibleLiteral { column, literal, expected, .. } + if column == "created_at" && literal == "2019-01-01" && expected == "timestamp" + )), + "{errors:?}" + ); + + // The same comparison written so it does coerce is accepted. + assert_eq!( + validate( + &guarded(Literal::Text("2019-01-01T00:00:00Z".into())), + &catalog + ), + vec![] + ); + } + + #[test] + fn comparing_two_columns_of_incomparable_kinds_is_reported() { + // Neither operand is a literal, so `Predicate::prepare` never coerces either side: at + // run time `Value::compare` sees a `Text` and an `Integer` and returns `None` (false) on + // every row. + let catalog = ExtractionCatalog::new().with_table( + "erp", + TableSchema::new( + "orders", + [ + ("id", "INTEGER", false), + ("code", "TEXT", false), + ("amount", "INTEGER", false), + ("price", "DOUBLE", false), + ], + ), + ); + let guarded = |left: &str, right: &str| { + bp( + vec![source("o", "orders")], + vec![MappingEntry::Single(Mapping { + node: "o".into(), + label: Some("orders".into()), + when: Some(Predicate::Compare { + left: Operand::Column { + column: left.into(), + }, + op: CompareOp::Eq, + right: Operand::Column { + column: right.into(), + }, + }), + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + })], + ) + }; + + let errors = validate(&guarded("code", "amount"), &catalog); + assert!( + errors.iter().any(|e| matches!( + e, + ValidationError::IncomparableCompare { left_kind, right_kind, .. } + if left_kind == "text" && right_kind == "integer" + )), + "{errors:?}" + ); + + // Integer/Float compare across kinds, so this is not reported. + assert_eq!(validate(&guarded("amount", "price"), &catalog), vec![]); + } + + #[test] + fn a_literal_against_a_column_of_unrecognised_type_is_left_alone() { + // No declared kind means coercion is deliberately off for that column, so the literal is + // compared as authored and there is nothing to report. + let catalog = ExtractionCatalog::new().with_table( + "erp", + TableSchema::new( + "orders", + [("id", "INTEGER", false), ("payload", "GEOMETRY", false)], + ), + ); + let b = bp( + vec![source("o", "orders")], + vec![MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: Some(Predicate::In { + column: "payload".into(), + values: vec![Literal::Text("anything".into())], + }), + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + })], + ); + assert_eq!(validate(&b, &catalog), vec![]); + } + + #[test] + fn a_typo_in_a_join_key_is_reported() { + // Join.on column pairs were never checked against either input's schema. + let join = Node { + id: "joined".into(), + label: None, + op: NodeOp::Join { + left: "o".into(), + right: "o2".into(), + on: vec![("not_a_column".into(), "id".into())], + }, + }; + let b = bp( + vec![ + source("o", "orders"), + Node { + id: "o2".into(), + label: None, + op: NodeOp::Source { + source_id: "erp".into(), + table: "orders".into(), + }, + }, + join, + ], + vec![], + ); + assert!(validate(&b, &catalog()).iter().any( + |e| matches!(e, ValidationError::UnknownColumn { column, .. } if column == "not_a_column") + )); + } + + #[test] + fn an_unlabelled_mapping_error_names_the_authored_path_not_the_flattened_index() { + // With mappings = [Ordered{3 members}, Single], the Single is at flattened index 3 but + // its authored JSON path is mappings[1]. Comparing the complete set of (from, id) pairs + // rules out a flattened-index naming that would satisfy a bare `from == "mappings[1]"` + // check for the wrong reason. + let ordered = MappingEntry::Ordered { + mappings: vec![ + object_mapping_value("a", Some(Predicate::And { conditions: vec![] })), + object_mapping_value("b", Some(Predicate::And { conditions: vec![] })), + object_mapping_value("c", None), + ], + }; + let single = MappingEntry::Single(Mapping { + node: "nope".into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![source("o", "orders")], vec![ordered, single]); + let errs = validate(&b, &catalog()); + let mut got: Vec<(&str, &str)> = errs + .iter() + .filter_map(|e| match e { + ValidationError::UnknownNodeRef { from, id } => Some((from.as_str(), id.as_str())), + _ => None, + }) + .collect(); + got.sort_unstable(); + let mut expected = vec![ + ("mappings[0].mappings[0]", "a"), + ("mappings[0].mappings[1]", "b"), + ("mappings[0].mappings[2]", "c"), + ("mappings[1]", "nope"), + ]; + expected.sort_unstable(); + assert_eq!(got, expected, "got {errs:?}"); + } + + fn object_mapping_value(node: &str, when: Option) -> Mapping { + Mapping { + node: node.into(), + label: None, + when, + target: Target::Object { + object_type: ValueExpression::Constant { + value: "Order".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + } + } + + #[test] + fn an_unterminated_template_placeholder_is_reported() { + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Template { + template: "{a}-{b".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![source("o", "orders")], vec![m]); + assert!(validate(&b, &catalog()) + .iter() + .any(|e| matches!(e, ValidationError::InvalidTemplate { .. }))); + } + + #[test] + fn an_empty_template_placeholder_is_reported() { + // Regression: an empty placeholder used to also collect "" as a referenced column, + // so validation additionally reported a spurious `UnknownColumn { column: "" }` + // alongside the `InvalidTemplate` that actually names the defect. Only the latter is + // the right diagnostic here, so the full error set is asserted, not just `.any()`. + let m = MappingEntry::Single(Mapping { + node: "o".into(), + label: None, + when: None, + target: Target::Object { + object_type: ValueExpression::Template { + template: "{}".into(), + }, + id: ValueExpression::Column { + column: "id".into(), + }, + timestamp: None, + attributes: vec![], + }, + }); + let b = bp(vec![source("o", "orders")], vec![m]); + assert_eq!( + validate(&b, &catalog()), + vec![ValidationError::InvalidTemplate { + template: "{}".into(), + reason: "empty placeholder".into(), + }] + ); + } + + #[test] + fn validation_errors_round_trip_through_json_externally_tagged_kebab_case() { + // a binding boundary sends `extraction_validate(..) -> Vec` through + // serde_json::to_vec, which needs Serialize; and needs to render like every sibling enum + // in this module rather than the default PascalCase. + let err = ValidationError::MissingTypeForPrefixing { + mapping: "mappings[0]".into(), + endpoint: "object".into(), + }; + let json = serde_json::to_value(&err).expect("serialize"); + assert_eq!(json["type"], "missing-type-for-prefixing"); + assert_eq!(json["endpoint"], "object"); + let back: ValidationError = serde_json::from_value(json).expect("deserialize"); + assert_eq!(back, err); + } +} diff --git a/process_mining/src/core/event_data/object_centric/extraction/value.rs b/process_mining/src/core/event_data/object_centric/extraction/value.rs new file mode 100644 index 00000000..cb6eaba8 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/extraction/value.rs @@ -0,0 +1,511 @@ +//! The value type rows carry, and its canonical rendering. +//! +//! Floats follow SQL's total order rather than IEEE's: `DuckDB` and `PostgreSQL` both sort `NaN` +//! above everything else and treat `NaN = NaN` and `-0.0 = 0.0` as true, where `partial_cmp` +//! answers `None`. [`Value::compare`] and [`Value::join_key_part`] follow the engines, so that +//! `Ne` does not go false for every operand and `-0.0`/`0.0` do not get separate join keys. +//! +//! Join keys carry a kind tag, so text never joins with numbers. The engines disagree here: +//! `DuckDB` casts and joins `'1'` with `1`, `PostgreSQL` rejects `text = integer`. Refusing the +//! join is the only total engine-independent option, so a compiler must not emit a bare +//! `l.k = r.k` across kinds. +//! +//! The tag is the value's runtime kind, which a compiler cannot see: it only has +//! [`ColumnSchema::declared_kind`](super::catalog::ColumnSchema::declared_kind). Reproducing the +//! rule in SQL is sound exactly when the catalog matches the values' actual kinds, which holds for +//! statically typed engines but not for `SQLite`. That is a catalog precondition rather than +//! something a blueprint can be checked for, so [`validate`](super::validate::validate) does not +//! reject such a join. + +use std::cmp::Ordering; +use std::fmt::Write; + +use chrono::{DateTime, FixedOffset}; + +/// A single cell of a row, normalised across data sources. +/// +/// Owned by this crate rather than taken from a driver, so the model and compiler need no +/// connector dependency. Not `Serialize`: a row's values never cross a bindings boundary. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + /// SQL `NULL` or a missing cell. + Null, + /// Text. + Text(String), + /// A 64-bit signed integer. + Integer(i64), + /// A double-precision float. + Float(f64), + /// A boolean. + Boolean(bool), + /// An instant with a fixed UTC offset. + Timestamp(DateTime), +} + +/// A [`Value`]'s kind, independent of any particular value. +/// +/// This is the vocabulary a source's declared column type ([`ColumnSchema::col_type`], mapped by +/// [`ColumnSchema::declared_kind`](super::catalog::ColumnSchema::declared_kind)) is translated +/// into, and the target of [`Value::coerce_to`]. +/// +/// [`ColumnSchema::col_type`]: super::catalog::ColumnSchema::col_type +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ValueKind { + /// [`Value::Text`]. + Text, + /// [`Value::Integer`]. + Integer, + /// [`Value::Float`]. + Float, + /// [`Value::Boolean`]. + Boolean, + /// [`Value::Timestamp`]. + Timestamp, +} + +/// `f` as an `i64` when it holds a whole number exactly representable as one, else `None`. +/// +/// `f64` can represent integers beyond `i64::MAX` only approximately, so the range check keeps a +/// value that would round on conversion out of an identity. The upper bound is strict because +/// `i64::MAX as f64` rounds up to 2^63, which `as i64` then saturates back to `i64::MAX`. +fn whole_number(f: f64) -> Option { + (f.fract() == 0.0 && f >= i64::MIN as f64 && f < i64::MAX as f64).then_some(f as i64) +} + +impl Value { + /// This value's [`ValueKind`], or `None` for [`Value::Null`], which has no kind of its own. + #[must_use] + pub fn kind(&self) -> Option { + match self { + Value::Null => None, + Value::Text(_) => Some(ValueKind::Text), + Value::Integer(_) => Some(ValueKind::Integer), + Value::Float(_) => Some(ValueKind::Float), + Value::Boolean(_) => Some(ValueKind::Boolean), + Value::Timestamp(_) => Some(ValueKind::Timestamp), + } + } + + /// Reinterpret this value as `kind`, when that parses cleanly. `None` otherwise. + /// + /// Used by `Predicate::prepare` to coerce a + /// `Compare` literal to its column's declared type. Part of the model's semantics, not an + /// implementation detail: [`compile`](super::compile()) reproduces exactly this rule in SQL. + /// + /// A value already of `kind` is returned unchanged, and one number converts to the other + /// numeric kind directly. Everything else reads [`Value::display_string`] and parses it as + /// `kind`: + /// + /// - `Integer`, `Float`: [`str::parse`], so `"1"` and `"1.5"` succeed and `"abc"` does not. + /// - `Boolean`: exactly `"true"` or `"false"`. + /// - `Timestamp`: strict RFC 3339 only (via [`DateTime::parse_from_rfc3339`]), not the + /// lenient cascade `TimestampSource::parse` uses. + /// RFC 3339 is what every mainstream engine's timestamp cast accepts, which is what keeps + /// the evaluator and a compiled view in agreement. + /// - `Text`: always succeeds, since every non-`Null` value has a `display_string`. + /// + /// `Null` never coerces to anything. + #[must_use] + pub fn coerce_to(&self, kind: ValueKind) -> Option { + if self.kind() == Some(kind) { + return Some(self.clone()); + } + // Same answer as the text round trip below, without an allocation per value. + match (self, kind) { + #[allow(clippy::cast_precision_loss)] + (Value::Integer(i), ValueKind::Float) => return Some(Value::Float(*i as f64)), + (Value::Float(f), ValueKind::Integer) => return whole_number(*f).map(Value::Integer), + _ => {} + } + let s = self.display_string()?; + match kind { + ValueKind::Text => Some(Value::Text(s)), + ValueKind::Integer => s.parse::().ok().map(Value::Integer), + ValueKind::Float => s.parse::().ok().map(Value::Float), + ValueKind::Boolean => match s.as_str() { + "true" => Some(Value::Boolean(true)), + "false" => Some(Value::Boolean(false)), + _ => None, + }, + ValueKind::Timestamp => DateTime::parse_from_rfc3339(&s).ok().map(Value::Timestamp), + } + } + + /// The canonical text form used wherever a value becomes an identity: entity ids, + /// relation endpoints, type names and template placeholders. + /// + /// Only `Text`, `Integer`, `Boolean` and a `Float` holding a whole number have one. + /// Fractional `Float` formatting is not stable across engines, `Timestamp` offset rendering + /// varies, and `Null` has no identity at all, so those return `None` and the caller drops the + /// row rather than inventing an id. + /// + /// A whole-number `Float` renders as that integer, because `numeric`/`decimal` columns decode + /// as `Float` and an integer key in one would otherwise have no identity at all. + /// + /// Not the rendering used to match a regex or reparse a value. See + /// [`Value::display_string`]. + #[must_use] + pub fn canonical_string(&self) -> Option { + match self { + Value::Text(s) => Some(s.clone()), + Value::Integer(i) => Some(i.to_string()), + Value::Boolean(b) => Some(if *b { "true" } else { "false" }.to_string()), + Value::Float(f) => whole_number(*f).map(|i| i.to_string()), + Value::Timestamp(_) | Value::Null => None, + } + } + + /// The key one cell contributes to a [`Join`](super::blueprint::NodeOp::Join)'s equality + /// test, or `None` when this value never joins. + /// + /// Not [`Value::canonical_string`], which is `None` for `Float` and `Timestamp` and would + /// make a join on such a column silently yield zero rows where SQL joins them. Join keys get + /// a total rendering instead: + /// + /// - `Null` is the one value with no key: SQL `NULL` never equals anything. + /// - `Integer` and `Float` share one key space (`1` joins `1.0`, as in SQL). + /// - Floats follow SQL's total order, not IEEE's: `NaN` joins `NaN` and `-0.0` joins `0.0`, + /// as `DuckDB` and `PostgreSQL` both define. Infinities are ordinary values. + /// - `Timestamp` is normalised to UTC first, so one instant written with two offsets joins. + /// - Every other kind is tagged with its kind, so a `Text` `"1"` does not join `Integer` `1`. + /// + /// The kind tag diverges from every engine, so a compiler has to reproduce it rather than + /// inherit it. See this module's docs. + #[must_use] + pub fn join_key_part(&self) -> Option { + let mut out = String::new(); + self.write_join_key_part(&mut out).then_some(out) + } + + /// [`Value::join_key_part`] appended to `out`, reporting `false` for `Null`, the one value + /// with no key. + /// + /// The allocation-free spelling, for a hash join that keys every row of both its inputs. + pub(crate) fn write_join_key_part(&self, out: &mut String) -> bool { + let written = match self { + Value::Null => return false, + Value::Text(s) => { + out.push_str("s:"); + out.push_str(s); + Ok(()) + } + Value::Integer(i) => write!(out, "n:{i}"), + // `-0.0 == 0.0` in Rust, so this maps the negative zero onto the positive one and + // leaves every other value (`NaN` included) alone. + Value::Float(f) => write!(out, "n:{}", if *f == 0.0 { 0.0 } else { *f }), + Value::Boolean(b) => write!(out, "b:{b}"), + Value::Timestamp(t) => write!(out, "t:{}", t.to_utc().to_rfc3339()), + }; + written.is_ok() + } + + /// The text form used wherever a value is being read rather than turned into an identity: + /// the input to a `Matches` regex, or the fallback input to + /// `TimestampSource::parse` when the column is not + /// already a `Timestamp`. + /// + /// Unlike [`Value::canonical_string`], `Float` and `Timestamp` render here: reading a value + /// needs some text, and cross-engine stability only matters for an identity. Only `Null` + /// has no rendering. + #[must_use] + pub fn display_string(&self) -> Option { + match self { + Value::Text(s) => Some(s.clone()), + Value::Integer(i) => Some(i.to_string()), + Value::Float(f) => Some(f.to_string()), + Value::Boolean(b) => Some(if *b { "true" } else { "false" }.to_string()), + Value::Timestamp(ts) => Some(ts.to_rfc3339()), + Value::Null => None, + } + } + + /// Whether this is [`Value::Null`]. + #[must_use] + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + /// Typed ordering, used by comparison predicates. + /// + /// Integers and floats are one ordering class, so `2 < 2.5` holds. Any other pair of + /// different kinds returns `None`, as does any comparison involving `Null`, `Null` against + /// `Null` included, since SQL `NULL` is not equal to itself. A `None` result makes the + /// comparing predicate false. + /// + /// Floats follow SQL's total order rather than IEEE's, see this module's docs. + #[must_use] + pub fn compare(&self, other: &Value) -> Option { + match (self, other) { + (Value::Null, _) | (_, Value::Null) => None, + (Value::Integer(a), Value::Integer(b)) => Some(a.cmp(b)), + (Value::Float(a), Value::Float(b)) => Some(sql_float_cmp(*a, *b)), + #[allow(clippy::cast_precision_loss)] + (Value::Integer(a), Value::Float(b)) => Some(sql_float_cmp(*a as f64, *b)), + #[allow(clippy::cast_precision_loss)] + (Value::Float(a), Value::Integer(b)) => Some(sql_float_cmp(*a, *b as f64)), + (Value::Text(a), Value::Text(b)) => Some(a.cmp(b)), + (Value::Boolean(a), Value::Boolean(b)) => Some(a.cmp(b)), + (Value::Timestamp(a), Value::Timestamp(b)) => Some(a.cmp(b)), + _ => None, + } + } +} + +/// Compare two floats the way `DuckDB` and `PostgreSQL` do: a total order in which `NaN` equals +/// itself and sorts above everything else, and `-0.0` equals `0.0`. +/// +/// Not [`f64::total_cmp`], which separates `-0.0` from `0.0` and distinguishes `NaN` payloads and +/// signs. SQL does neither. +fn sql_float_cmp(a: f64, b: f64) -> Ordering { + match (a.is_nan(), b.is_nan()) { + (true, true) => Ordering::Equal, + (true, false) => Ordering::Greater, + (false, true) => Ordering::Less, + (false, false) => a.partial_cmp(&b).unwrap_or(Ordering::Equal), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cmp::Ordering; + + #[test] + fn canonical_string_renders_only_stable_types() { + assert_eq!( + Value::Text("a".into()).canonical_string().as_deref(), + Some("a") + ); + assert_eq!(Value::Integer(-7).canonical_string().as_deref(), Some("-7")); + assert_eq!( + Value::Boolean(true).canonical_string().as_deref(), + Some("true") + ); + assert_eq!( + Value::Boolean(false).canonical_string().as_deref(), + Some("false") + ); + // A whole-number float is an identity: `numeric`/`decimal` columns decode as Float, and an + // integer key in one (Sakila's `actor_id numeric`) must not drop every row it keys. + assert_eq!(Value::Float(1.0).canonical_string().as_deref(), Some("1")); + assert_eq!( + Value::Float(200.0).canonical_string().as_deref(), + Some("200") + ); + // A fractional float has no stable rendering across engines, so it still has no identity. + assert!(Value::Float(1.5).canonical_string().is_none()); + assert!(Value::Float(f64::NAN).canonical_string().is_none()); + assert!(Value::Float(f64::INFINITY).canonical_string().is_none()); + // Beyond i64 an f64 only approximates whole numbers, so it would round into a wrong id. + assert!(Value::Float(1e30).canonical_string().is_none()); + // `i64::MAX as f64` rounds up to 2^63, which `as i64` saturates back to `i64::MAX`: an + // inclusive upper bound gave this float the identity of a different number. + assert!(Value::Float(9_223_372_036_854_775_808.0) + .canonical_string() + .is_none()); + assert!(Value::Null.canonical_string().is_none()); + } + + #[test] + fn display_string_renders_everything_but_null() { + assert_eq!(Value::Float(1.5).display_string().as_deref(), Some("1.5")); + assert!(Value::Timestamp( + chrono::DateTime::parse_from_rfc3339("2020-02-03T04:05:06+02:00").unwrap() + ) + .display_string() + .is_some()); + assert!(Value::Null.display_string().is_none()); + } + + #[test] + fn numbers_compare_across_integer_and_float() { + assert_eq!( + Value::Integer(2).compare(&Value::Float(2.5)), + Some(Ordering::Less) + ); + assert_eq!( + Value::Float(10.0).compare(&Value::Integer(9)), + Some(Ordering::Greater) + ); + assert_eq!( + Value::Integer(3).compare(&Value::Float(3.0)), + Some(Ordering::Equal) + ); + } + + #[test] + fn negative_floats_order_before_positive() { + assert_eq!( + Value::Float(-1.5).compare(&Value::Float(0.5)), + Some(Ordering::Less) + ); + } + + #[test] + fn mismatched_kinds_and_null_do_not_order() { + assert!(Value::Text("1".into()) + .compare(&Value::Integer(1)) + .is_none()); + assert!(Value::Null.compare(&Value::Null).is_none()); + assert!(Value::Null.compare(&Value::Integer(1)).is_none()); + } + + #[test] + fn coerce_to_parses_cleanly_or_returns_none() { + assert_eq!( + Value::Text("1".into()).coerce_to(ValueKind::Integer), + Some(Value::Integer(1)) + ); + assert_eq!( + Value::Text("2020-01-01T00:00:00Z".into()).coerce_to(ValueKind::Timestamp), + Some(Value::Timestamp( + chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap() + )) + ); + assert_eq!( + Value::Text("abc".into()).coerce_to(ValueKind::Integer), + None + ); + assert_eq!( + Value::Text("not-a-date".into()).coerce_to(ValueKind::Timestamp), + None + ); + } + + #[test] + fn join_keys_are_total_except_for_null() { + // Regression: Float and Timestamp keys used to render through `canonical_string`, which + // is `None` for both, so a join on such a column silently produced zero rows. + assert!(Value::Float(1.5).join_key_part().is_some()); + assert!(Value::Timestamp( + chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap() + ) + .join_key_part() + .is_some()); + assert!(Value::Null.join_key_part().is_none()); + } + + /// `DuckDB` and `PostgreSQL` both give floats a total order for comparison: `NaN = NaN` and + /// `-0.0 = 0.0` are both true. Treating `NaN` like `NULL`, or giving `-0.0` its own key, loses + /// rows a compiled view keeps. + #[test] + fn join_keys_follow_sql_s_total_float_order() { + assert_eq!( + Value::Float(f64::NAN).join_key_part(), + Value::Float(f64::NAN).join_key_part() + ); + assert!(Value::Float(f64::NAN).join_key_part().is_some()); + assert_eq!( + Value::Float(f64::INFINITY).join_key_part(), + Value::Float(f64::INFINITY).join_key_part() + ); + assert_ne!( + Value::Float(f64::INFINITY).join_key_part(), + Value::Float(f64::NEG_INFINITY).join_key_part() + ); + assert_ne!( + Value::Float(f64::NAN).join_key_part(), + Value::Float(f64::INFINITY).join_key_part() + ); + assert_eq!( + Value::Float(-0.0).join_key_part(), + Value::Float(0.0).join_key_part() + ); + assert_eq!( + Value::Float(-0.0).join_key_part(), + Value::Integer(0).join_key_part() + ); + } + + /// `Value::compare` decides `Compare` and `In`, whose emitted SQL is + /// `COALESCE(col lit, FALSE)`, run by an engine that gives floats the same total order + /// `join_key_part` commits to. `partial_cmp` answers `None` for any pair + /// involving `NaN`, which `PreparedPredicate::evaluate` turns into `false` for every + /// operator, while `DuckDB` makes `NaN = NaN` and `NaN > 1.0` both true. + #[test] + fn compare_follows_sql_s_total_float_order() { + assert_eq!( + Value::Float(f64::NAN).compare(&Value::Float(f64::NAN)), + Some(Ordering::Equal) + ); + assert_eq!( + Value::Float(f64::NAN).compare(&Value::Float(1.0)), + Some(Ordering::Greater) + ); + assert_eq!( + Value::Float(1.0).compare(&Value::Float(f64::NAN)), + Some(Ordering::Less) + ); + assert_eq!( + Value::Float(f64::NAN).compare(&Value::Float(f64::INFINITY)), + Some(Ordering::Greater) + ); + // Mixed integer/float: an integer is never `NaN`, so it always sorts below one. + assert_eq!( + Value::Integer(1).compare(&Value::Float(f64::NAN)), + Some(Ordering::Less) + ); + assert_eq!( + Value::Float(f64::NAN).compare(&Value::Integer(1)), + Some(Ordering::Greater) + ); + // `-0.0` and `0.0` are one value, exactly as in `join_key_part`. + assert_eq!( + Value::Float(-0.0).compare(&Value::Float(0.0)), + Some(Ordering::Equal) + ); + // `Null` stays incomparable: SQL `NULL` is not equal to itself. + assert!(Value::Null.compare(&Value::Float(f64::NAN)).is_none()); + } + + #[test] + fn join_keys_equate_integers_with_floats_but_not_text_with_numbers() { + assert_eq!( + Value::Integer(1).join_key_part(), + Value::Float(1.0).join_key_part() + ); + assert_ne!( + Value::Integer(1).join_key_part(), + Value::Text("1".into()).join_key_part() + ); + } + + #[test] + fn join_keys_normalise_a_timestamp_offset() { + let utc = + Value::Timestamp(chrono::DateTime::parse_from_rfc3339("2020-01-01T02:00:00Z").unwrap()); + let offset = Value::Timestamp( + chrono::DateTime::parse_from_rfc3339("2020-01-01T04:00:00+02:00").unwrap(), + ); + assert_eq!(utc.join_key_part(), offset.join_key_part()); + } + + #[test] + fn coerce_to_a_value_already_of_that_kind_is_unchanged() { + assert_eq!( + Value::Integer(5).coerce_to(ValueKind::Integer), + Some(Value::Integer(5)) + ); + } + + /// The direct numeric arms must answer exactly what the text round trip answered. + #[test] + fn a_number_coerces_to_the_other_numeric_kind_or_not_at_all() { + assert_eq!( + Value::Integer(5).coerce_to(ValueKind::Float), + Some(Value::Float(5.0)) + ); + assert_eq!( + Value::Float(5.0).coerce_to(ValueKind::Integer), + Some(Value::Integer(5)) + ); + assert_eq!(Value::Float(5.5).coerce_to(ValueKind::Integer), None); + assert_eq!(Value::Float(1e30).coerce_to(ValueKind::Integer), None); + assert_eq!(Value::Float(f64::NAN).coerce_to(ValueKind::Integer), None); + } + + #[test] + fn null_never_coerces() { + assert_eq!(Value::Null.coerce_to(ValueKind::Text), None); + } +} diff --git a/process_mining/src/core/event_data/object_centric/graph_db/mod.rs b/process_mining/src/core/event_data/object_centric/graph_db/mod.rs deleted file mode 100644 index 577205cd..00000000 --- a/process_mining/src/core/event_data/object_centric/graph_db/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -/// `KuzuDB` OCEL Functionality -/// -/// Requires the `kuzudb` feature to be enabled. -pub mod ocel_kuzudb; diff --git a/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs b/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs deleted file mode 100644 index 912979c2..00000000 --- a/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs +++ /dev/null @@ -1,442 +0,0 @@ -#[cfg(feature = "dataframes")] -use macros_process_mining::register_binding; -use polars::{error::PolarsResult, frame::DataFrame, io::SerWriter, prelude::CsvWriter}; -use std::{fs::File, path::Path}; - -use crate::core::event_data::object_centric::ocel_struct::OCELAttributeType; -#[cfg(feature = "dataframes")] -use crate::core::event_data::object_centric::{linked_ocel::LinkedOCELAccess, ocel_struct::OCEL}; - -/// -/// Error encountered while parsing XES -/// -#[derive(Debug)] -pub enum KuzuDBExportError { - /// Error orignating in kuzu - KuzuDBError(kuzu::Error), - /// General IO Error (e.g., when creating the database file) - IOError(std::io::Error), - #[cfg(feature = "dataframes")] - /// Error originiating in Polars (for `DataFrame` conversion used as an intermediate step) - PolarsError(polars::prelude::PolarsError), -} - -impl std::fmt::Display for KuzuDBExportError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Failed to export to kuzudb: {self:?}") - } -} - -impl std::error::Error for KuzuDBExportError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - KuzuDBExportError::KuzuDBError(e) => Some(e), - KuzuDBExportError::IOError(e) => Some(e), - #[cfg(feature = "dataframes")] - KuzuDBExportError::PolarsError(e) => Some(e), - } - } - - fn description(&self) -> &str { - "description() is deprecated; use Display" - } - - fn cause(&self) -> Option<&dyn std::error::Error> { - self.source() - } -} - -impl From for KuzuDBExportError { - fn from(e: std::io::Error) -> Self { - Self::IOError(e) - } -} - -impl From for KuzuDBExportError { - fn from(e: kuzu::Error) -> Self { - Self::KuzuDBError(e) - } -} - -#[cfg(feature = "dataframes")] -impl From for KuzuDBExportError { - fn from(e: polars::prelude::PolarsError) -> Self { - Self::PolarsError(e) - } -} - -/// Export an [`OCEL`] as a [kuzu](https://github.com/kuzudb/kuzu) database -/// -/// This export function does not create different node types for different event/object types -/// -/// Instead `Event` and `Object` nodes are added, and they both have an attribute `type`. -/// -/// For E2O relationships, the `E2O` relation is used, pointing from events to objects, with an additional relationship qualifier. -/// -/// **Limitations**: This function is work-in-progress, currently some aspects (O2O relationships, object attribute changes) are not recorded. -#[cfg(feature = "dataframes")] -#[register_binding(stringify_error)] -pub fn export_ocel_to_kuzudb_generic( - db_path: impl AsRef, - ocel: &OCEL, -) -> Result<(), KuzuDBExportError> { - use kuzu::{Connection, Database, SystemConfig}; - - use dataframe::{ - ocel_to_dataframes, OCEL_EVENT_ID_KEY, OCEL_EVENT_TIMESTAMP_KEY, OCEL_EVENT_TYPE_KEY, - OCEL_OBJECT_ID_2_KEY, OCEL_OBJECT_ID_KEY, OCEL_OBJECT_TYPE_KEY, OCEL_QUALIFIER_KEY, - }; - - use crate::core::event_data::object_centric::dataframe; - - let db = Database::new(db_path, SystemConfig::default())?; - let conn = Connection::new(&db)?; - let tmp = tempfile::tempdir()?; - let path = tmp.path(); - let mut df = ocel_to_dataframes(ocel); - df.export_events_csv( - path.join("events.csv"), - &[ - OCEL_EVENT_ID_KEY, - OCEL_EVENT_TYPE_KEY, - OCEL_EVENT_TIMESTAMP_KEY, - ], - )?; - df.export_objects_csv( - path.join("objects.csv"), - &[OCEL_OBJECT_ID_KEY, OCEL_OBJECT_TYPE_KEY], - )?; - df.export_e2o_csv( - path.join("e2o.csv"), - &[OCEL_EVENT_ID_KEY, OCEL_OBJECT_ID_KEY, OCEL_QUALIFIER_KEY], - )?; - df.export_o2o_csv( - path.join("o2o.csv"), - &[OCEL_OBJECT_ID_KEY, OCEL_OBJECT_ID_2_KEY, OCEL_QUALIFIER_KEY], - )?; - conn.query("CREATE NODE TABLE Event(id STRING PRIMARY KEY, type STRING, time TIMESTAMP);")?; - conn.query("CREATE NODE TABLE Object(id STRING PRIMARY KEY, type STRING);")?; - conn.query("CREATE REL TABLE E2O(FROM Event to Object, qualifier STRING);")?; - conn.query("CREATE REL TABLE O2O(FROM Object to Object, qualifier STRING);")?; - conn.query(&format!( - "COPY Event FROM '{}' (header=true);", - path.join("events.csv").to_string_lossy() - ))?; - conn.query(&format!( - "COPY Object FROM '{}' (header=true);", - path.join("objects.csv").to_string_lossy() - ))?; - conn.query(&format!( - "COPY E2O FROM '{}' (header=true)", - path.join("e2o.csv").to_string_lossy() - ))?; - conn.query(&format!( - "COPY O2O FROM '{}' (header=true)", - path.join("o2o.csv").to_string_lossy() - ))?; - Ok(()) -} - -fn export_df_to_csv>(df: &mut DataFrame, export_path: P) -> PolarsResult<()> { - let f = File::create(export_path)?; - // Force microsecond fractional seconds + Z suffix so the timestamp - // column round-trips through CSV without losing the µs precision - // that downstream consumers (Kuzu's TIMESTAMP column type, OCEL - // canonicalization invariants) rely on. Polars' default datetime - // format is "%Y-%m-%dT%H:%M:%S" — no fractional seconds — which - // silently rounded everything to whole seconds in the typed Kuzu - // export. - let mut csvw = - CsvWriter::new(f).with_datetime_format(Some("%Y-%m-%dT%H:%M:%S%.6fZ".to_string())); - csvw.finish(df)?; - Ok(()) -} - -fn clean_type_name(name: &str) -> String { - name.replace(" ", "") - .chars() - .map(|c| if c.is_alphanumeric() { c } else { '_' }) - .collect() -} -fn ocel_attribute_type_to_kuzu_dtype(attr_type: &str) -> &'static str { - match OCELAttributeType::from_type_str(attr_type) { - OCELAttributeType::String => "STRING", - OCELAttributeType::Time => "TIMESTAMP", - OCELAttributeType::Integer => "INT64", - OCELAttributeType::Float => "DOUBLE", - OCELAttributeType::Boolean => "BOOLEAN", - OCELAttributeType::Null => "NULL", - } -} -/// Export an OCEL to a (strictly typed) kuzu database -#[cfg(feature = "dataframes")] -#[register_binding(stringify_error)] -pub fn export_ocel_to_kuzudb_typed<'a>( - db_path: impl AsRef, - locel: &'a impl LinkedOCELAccess<'a>, -) -> Result<(), KuzuDBExportError> { - use std::fs::remove_file; - - use itertools::Itertools; - use kuzu::{Connection, Database, SystemConfig}; - - use dataframe::{ - e2o_to_df_for_types, event_type_to_df, o2o_to_df_for_types, object_attribute_changes_to_df, - object_type_to_df, ATTRIBUTE_CHANGE_DF_FROM_TIME, ATTRIBUTE_CHANGE_DF_ID, - ATTRIBUTE_CHANGE_DF_OBJ_ID, ATTRIBUTE_CHANGE_DF_TO_TIME, - }; - - use crate::core::event_data::object_centric::dataframe; - - let db = Database::new(db_path, SystemConfig::default())?; - let conn = Connection::new(&db)?; - let tmp = tempfile::tempdir()?; - let path = tmp.path(); - let mut all_ev_table_names = Vec::new(); - for ev_type in locel.get_ev_types() { - let mut ev_df = event_type_to_df(locel, ev_type)?; - if let Some(etype) = locel.get_ev_type(ev_type) { - export_df_to_csv(&mut ev_df, path.join("tmp.csv"))?; - let clean_name = clean_type_name(ev_type); - - let attribute_fields_str = etype - .attributes - .iter() - .map(|a| { - format!( - "`{}` {}", - a.name, - ocel_attribute_type_to_kuzu_dtype(&a.value_type) - ) - }) - .join(", "); - let q = format!( - "CREATE NODE TABLE `{}`(id STRING PRIMARY KEY, time TIMESTAMP {} {});", - clean_name, - if attribute_fields_str.is_empty() { - "" - } else { - ", " - }, - attribute_fields_str - ); - println!("Query for event type {ev_type}: {q}"); - conn.query(&q)?; - - conn.query(&format!( - "COPY {} FROM '{}' (header=true);", - clean_name, - path.join("tmp.csv").to_string_lossy() - ))?; - all_ev_table_names.push(clean_name); - remove_file(path.join("tmp.csv"))?; - } - } - for ob_type in locel.get_ob_types() { - let clean_name = clean_type_name(ob_type); - let q = format!("CREATE NODE TABLE `{clean_name}`(id STRING PRIMARY KEY);",); - conn.query(&q)?; - - let attribute_fields_str = locel - .get_ob_type(ob_type) - .map(|ot| &ot.attributes) - .into_iter() - .flatten() - .map(|a| { - format!( - "`{}` {}", - a.name, - ocel_attribute_type_to_kuzu_dtype(&a.value_type) - ) - }) - .join(", "); - let q = format!( - "CREATE NODE TABLE `{clean_name}Attributes`(id STRING PRIMARY KEY {} {});", - if attribute_fields_str.is_empty() { - "" - } else { - ", " - }, - attribute_fields_str - ); - conn.query(&q)?; - } - - conn.query(&format!( - "CREATE REL TABLE Attrs ({}, from_time TIMESTAMP, to_time TIMESTAMP)", - locel - .get_ob_types() - .map(clean_type_name) - .map(|ob_type| format!("FROM `{ob_type}` TO `{ob_type}Attributes`")) - .join(", "), - ))?; - let mut all_ob_table_names = Vec::new(); - for ob_type in locel.get_ob_types() { - let mut ob_df = object_type_to_df(locel, ob_type)?; - export_df_to_csv(&mut ob_df, path.join("tmp.csv"))?; - let clean_name = clean_type_name(ob_type); - // let q = format!("CREATE NODE TABLE `{clean_name}`(id STRING PRIMARY KEY);",); - // conn.query(&q)?; - - conn.query(&format!( - "COPY `{}` FROM '{}' (header=true);", - clean_name, - path.join("tmp.csv").to_string_lossy() - ))?; - remove_file(path.join("tmp.csv"))?; - - let ob_changes_df = object_attribute_changes_to_df(locel, ob_type)?; - - let mut rel_df = ob_changes_df.select([ - ATTRIBUTE_CHANGE_DF_OBJ_ID, - ATTRIBUTE_CHANGE_DF_ID, - ATTRIBUTE_CHANGE_DF_FROM_TIME, - ATTRIBUTE_CHANGE_DF_TO_TIME, - ])?; - let mut changes_df = ob_changes_df.drop_many([ - ATTRIBUTE_CHANGE_DF_OBJ_ID, - ATTRIBUTE_CHANGE_DF_FROM_TIME, - ATTRIBUTE_CHANGE_DF_TO_TIME, - ]); - export_df_to_csv(&mut changes_df, path.join("tmp.csv"))?; - conn.query(&format!( - "COPY `{clean_name}Attributes` FROM '{}' (header=true);", - path.join("tmp.csv").to_string_lossy() - ))?; - remove_file(path.join("tmp.csv"))?; - - // // Populate relation table object -> objectAttribute - export_df_to_csv(&mut rel_df, path.join("tmp.csv"))?; - let q = format!( - "COPY Attrs FROM '{}' (header=true, from='{}', to='{}Attributes');", - path.join("tmp.csv").to_string_lossy(), - clean_name, - clean_name, - ); - println!(":: {q}"); - conn.query(&q)?; - remove_file(path.join("tmp.csv"))?; - - all_ob_table_names.push(clean_name); - } - conn.query(&format!( - "CREATE REL TABLE E2O ({}, qualifier STRING)", - all_ev_table_names - .iter() - .cartesian_product(all_ob_table_names.iter()) - .map(|(ev_type, ob_type)| format!("FROM `{ev_type}` TO `{ob_type}`")) - .join(", "), - ))?; - for ev_type in locel.get_ev_types() { - for ob_type in locel.get_ob_types() { - let mut e2o_df = e2o_to_df_for_types(locel, ev_type, ob_type)?; - export_df_to_csv(&mut e2o_df, path.join("tmp.csv"))?; - conn.query(&format!( - "COPY E2O FROM '{}' (header=true, from='{}', to='{}');", - path.join("tmp.csv").to_string_lossy(), - clean_type_name(ev_type), - clean_type_name(ob_type) - ))?; - remove_file(path.join("tmp.csv"))?; - } - } - - conn.query(&format!( - "CREATE REL TABLE O2O ({}, qualifier STRING)", - all_ob_table_names - .iter() - .cartesian_product(all_ob_table_names.iter()) - .map(|(ev_type, ob_type)| format!("FROM `{ev_type}` TO `{ob_type}`")) - .join(", "), - ))?; - for from_ob_type in locel.get_ob_types() { - for to_ob_type in locel.get_ob_types() { - let mut o2o_df = o2o_to_df_for_types(locel, from_ob_type, to_ob_type)?; - export_df_to_csv(&mut o2o_df, path.join("tmp.csv"))?; - conn.query(&format!( - "COPY O2O FROM '{}' (header=true, from='{}', to='{}');", - path.join("tmp.csv").to_string_lossy(), - clean_type_name(from_ob_type), - clean_type_name(to_ob_type) - ))?; - remove_file(path.join("tmp.csv"))?; - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::{fs::remove_file, time::Instant}; - - use chrono::DateTime; - use kuzu::{Connection, Database}; - - use crate::{ - core::event_data::object_centric::{ - graph_db::ocel_kuzudb::export_ocel_to_kuzudb_generic, linked_ocel::IndexLinkedOCEL, - ocel_xml::xml_ocel_import::import_ocel_xml_path, - }, - test_utils::get_test_data_path, - }; - - use super::{export_ocel_to_kuzudb_typed, KuzuDBExportError}; - - #[test] - fn test_kuzudb_export() { - let export_path = get_test_data_path() - .join("export") - .join("order-management-ocel.kuzu"); - let _er = remove_file(&export_path); - let ocel = import_ocel_xml_path( - get_test_data_path() - .join("ocel") - .join("order-management.xml"), - ) - .unwrap(); - export_ocel_to_kuzudb_generic(export_path, &ocel).unwrap(); - } - - #[test] - fn test_typed_kuzudb_export() { - let export_path = get_test_data_path() - .join("export") - .join("order-management-typed-ocel.kuzu"); - let _er = remove_file(&export_path); - let ocel = import_ocel_xml_path( - get_test_data_path() - .join("ocel") - .join("order-management.xml"), - ) - .unwrap(); - - let locel = IndexLinkedOCEL::from(ocel); - let now = Instant::now(); - export_ocel_to_kuzudb_typed(export_path, &locel).unwrap(); - println!("Export took {:?}", now.elapsed()); - } - - #[test] - fn perf_test_kuzu() -> Result<(), KuzuDBExportError> { - let export_path = get_test_data_path().join("export").join("stress-ocel.kuzu"); - let _er = remove_file(&export_path); - let db = Database::new(export_path, kuzu::SystemConfig::default()).unwrap(); - let conn = Connection::new(&db)?; - conn.query("CREATE NODE TABLE Event(id STRING PRIMARY KEY, type STRING, time TIMESTAMP);")?; - let now = Instant::now(); - for i in 0..10_000 { - // println!("{i}"); - let query = format!( - "CREATE (e:Event {{id: {}, type: 'Pay Order', time: timestamp('{}')}});", - i, - DateTime::UNIX_EPOCH.to_rfc3339(), - ); - // conn.query(&query)?; - println!("{query}"); - } - println!("{:?}", now.elapsed()); - Ok(()) - // let ocel = import_ocel_xml_path(get_test_data_path().join("ocel").join("ocel2-p2p.xml")); - // export_ocel_to_kuzudb_typed(export_path, &ocel).unwrap(); - } -} diff --git a/process_mining/src/core/event_data/object_centric/io.rs b/process_mining/src/core/event_data/object_centric/io.rs index d445853d..3aae8006 100644 --- a/process_mining/src/core/event_data/object_centric/io.rs +++ b/process_mining/src/core/event_data/object_centric/io.rs @@ -29,6 +29,12 @@ pub enum OCELIOError { /// `DuckDB` Error #[cfg(feature = "ocel-duckdb")] DuckDB(duckdb::Error), + /// Reading a bundled CSV/Parquet container failed + #[cfg(feature = "ocel-bundle")] + BundleImport(crate::core::event_data::object_centric::ocel_bundle::BundleImportError), + /// Writing a bundled CSV/Parquet container failed + #[cfg(feature = "ocel-bundle")] + BundleExport(crate::core::event_data::object_centric::ocel_bundle::BundleExportError), /// Unsupported Format UnsupportedFormat(String), /// Other Error @@ -46,6 +52,10 @@ impl std::fmt::Display for OCELIOError { OCELIOError::Sqlite(e) => write!(f, "SQLite Error: {}", e), #[cfg(feature = "ocel-duckdb")] OCELIOError::DuckDB(e) => write!(f, "DuckDB Error: {}", e), + #[cfg(feature = "ocel-bundle")] + OCELIOError::BundleImport(e) => write!(f, "Bundle Import Error: {}", e), + #[cfg(feature = "ocel-bundle")] + OCELIOError::BundleExport(e) => write!(f, "Bundle Export Error: {}", e), OCELIOError::UnsupportedFormat(s) => write!(f, "Unsupported Format: {}", s), OCELIOError::Other(s) => write!(f, "Error: {}", s), } @@ -63,6 +73,10 @@ impl std::error::Error for OCELIOError { OCELIOError::Sqlite(e) => Some(e), #[cfg(feature = "ocel-duckdb")] OCELIOError::DuckDB(e) => Some(e), + #[cfg(feature = "ocel-bundle")] + OCELIOError::BundleImport(e) => Some(e), + #[cfg(feature = "ocel-bundle")] + OCELIOError::BundleExport(e) => Some(e), OCELIOError::UnsupportedFormat(_) => None, OCELIOError::Other(_) => None, } @@ -107,6 +121,20 @@ impl From for OCELIOError { } } +#[cfg(feature = "ocel-bundle")] +impl From for OCELIOError { + fn from(e: crate::core::event_data::object_centric::ocel_bundle::BundleImportError) -> Self { + OCELIOError::BundleImport(e) + } +} + +#[cfg(feature = "ocel-bundle")] +impl From for OCELIOError { + fn from(e: crate::core::event_data::object_centric::ocel_bundle::BundleExportError) -> Self { + OCELIOError::BundleExport(e) + } +} + #[cfg(any(feature = "ocel-duckdb", feature = "ocel-sqlite"))] impl From for OCELIOError { fn from(e: DatabaseError) -> Self { @@ -125,6 +153,16 @@ impl Importable for OCEL { fn infer_format(path: &Path) -> Option { let p = path.to_string_lossy().to_lowercase(); + // Checked before `.json`, which the manifest name would otherwise match: pointing at a + // container's manifest means "read the directory it is in", not "parse this file". + #[cfg(feature = "ocel-bundle")] + if path.file_name().is_some_and(|n| { + n.eq_ignore_ascii_case( + crate::core::event_data::object_centric::ocel_bundle::META_FILE_NAME, + ) + }) { + return Some("ocel.zip".to_string()); + } if p.ends_with(".csv.gz") { Some("ocel.csv.gz".to_string()) } else if p.ends_with(".csv") { @@ -137,14 +175,18 @@ impl Importable for OCEL { Some("sqlite".to_string()) } else if p.ends_with(".duckdb") { Some("duckdb".to_string()) + } else if p.ends_with(".zip") || path.is_dir() { + // A directory is the bundled format's uncompressed form, with no extension to match. + Some("ocel.zip".to_string()) } else { infer_format_from_path(path) } } fn import_from_reader_with_options( - #[cfg(feature = "ocel-sqlite")] mut reader: R, - #[cfg(not(feature = "ocel-sqlite"))] reader: R, + // A SQLite file and a ZIP are both read from their end, so they consume the whole reader. + #[cfg(any(feature = "ocel-sqlite", feature = "ocel-bundle"))] mut reader: R, + #[cfg(not(any(feature = "ocel-sqlite", feature = "ocel-bundle")))] reader: R, format: &str, options: Self::ImportOptions, ) -> Result { @@ -190,6 +232,20 @@ impl Importable for OCEL { Err(OCELIOError::UnsupportedFormat( "DuckDB import from reader not supported".to_string(), )) + } else if format.ends_with("zip") { + #[cfg(feature = "ocel-bundle")] + { + let mut b = Vec::new(); + reader.read_to_end(&mut b)?; + crate::core::event_data::object_centric::ocel_bundle::import_ocel_bundle_from_bytes( + &b, + ) + .map_err(OCELIOError::from) + } + #[cfg(not(feature = "ocel-bundle"))] + Err(OCELIOError::UnsupportedFormat( + "bundled CSV/Parquet support not enabled".to_string(), + )) } else { Err(OCELIOError::UnsupportedFormat(format.to_string())) } @@ -223,6 +279,16 @@ impl Importable for OCEL { return Err(OCELIOError::UnsupportedFormat( "DuckDB support not enabled".to_string(), )); + } else if format.ends_with("zip") { + // A path can be a directory (the uncompressed form), and lets an archive be expanded + // a table at a time rather than held whole. + #[cfg(feature = "ocel-bundle")] + return crate::core::event_data::object_centric::ocel_bundle::import_ocel_bundle(path) + .map_err(OCELIOError::from); + #[cfg(not(feature = "ocel-bundle"))] + return Err(OCELIOError::UnsupportedFormat( + "bundled CSV/Parquet support not enabled".to_string(), + )); } else { let file = std::fs::File::open(path)?; let reader = std::io::BufReader::new(file); @@ -238,6 +304,12 @@ impl Importable for OCEL { ExtensionWithMime::new("xml.gz", "application/gzip"), ExtensionWithMime::new("ocel.csv", "text/csv"), ExtensionWithMime::new("ocel.csv.gz", "application/gzip"), + #[cfg(feature = "ocel-bundle")] + // Both names, even though the reader ignores the difference (storage is declared in + // `ocel-meta.json`): export uses the `-parquet` name to pick which storage to write. + ExtensionWithMime::new("ocel.zip", "application/zip"), + #[cfg(feature = "ocel-bundle-parquet")] + ExtensionWithMime::new("ocel-parquet.zip", "application/zip"), #[cfg(feature = "ocel-sqlite")] ExtensionWithMime::new("sqlite", "application/x-sqlite3"), #[cfg(feature = "ocel-duckdb")] @@ -264,6 +336,17 @@ impl Exportable for OCEL { Some("sqlite".to_string()) } else if p.ends_with(".duckdb") { Some("duckdb".to_string()) + } else if p.ends_with("-parquet.zip") || p.ends_with("-parquet") { + // Storage is not part of the format's own naming, so a distinct format string is how + // a caller asks for Parquet. + Some("ocel-parquet.zip".to_string()) + } else if p.ends_with(".zip") { + Some("ocel.zip".to_string()) + } else if path.is_dir() { + // Only an existing directory: a path that does not yet exist cannot be told apart + // from a misspelled filename, so writing a new directory container has to go through + // `export_ocel_bundle` explicitly. + Some("ocel.zip".to_string()) } else { infer_format_from_path(path) } @@ -272,7 +355,7 @@ impl Exportable for OCEL { fn export_to_path_with_options>( &self, path: P, - _: Self::ExportOptions, + options: Self::ExportOptions, ) -> Result<(), Self::Error> { let path = path.as_ref(); let format = ::infer_format(path).ok_or_else(|| { @@ -281,7 +364,18 @@ impl Exportable for OCEL { "Could not infer format from path", ) })?; + ::export_to_path_as(self, path, &format, options) + } + /// Handles the formats that need a real path: a database driver opens its target by name, and + /// the bundled format's uncompressed form is a directory. + fn export_to_path_as>( + &self, + path: P, + format: &str, + _: Self::ExportOptions, + ) -> Result<(), Self::Error> { + let path = path.as_ref(); if format.ends_with("sqlite") || (format.ends_with("db") && !format.ends_with("duckdb")) { #[cfg(feature = "ocel-sqlite")] return crate::core::event_data::object_centric::ocel_sql::export_ocel_sqlite_to_path( @@ -304,10 +398,43 @@ impl Exportable for OCEL { return Err(OCELIOError::UnsupportedFormat( "DuckDB support not enabled".to_string(), )); + } else if format.ends_with("zip") { + #[cfg(feature = "ocel-bundle")] + { + use crate::core::event_data::object_centric::ocel_bundle::{ + export_ocel_bundle, BundleExportOptions, ContainerLayout, StorageFormat, + }; + // A directory, or a name with no extension, means the uncompressed form. + let layout = if path.is_dir() || path.extension().is_none() { + ContainerLayout::Directory + } else { + ContainerLayout::Archive + }; + if layout == ContainerLayout::Directory { + std::fs::create_dir_all(path)?; + } + export_ocel_bundle( + self, + path, + BundleExportOptions { + layout, + storage: if format.starts_with("ocel-parquet") { + StorageFormat::Parquet + } else { + StorageFormat::Csv + }, + }, + ) + .map_err(OCELIOError::from) + } + #[cfg(not(feature = "ocel-bundle"))] + return Err(OCELIOError::UnsupportedFormat( + "bundled CSV/Parquet support not enabled".to_string(), + )); } else { let file = std::fs::File::create(path)?; let writer = std::io::BufWriter::new(file); - Self::export_to_writer(self, writer, &format) + Self::export_to_writer(self, writer, format) } } @@ -351,6 +478,33 @@ impl Exportable for OCEL { return Err(OCELIOError::UnsupportedFormat( "SQLite support not enabled".to_string(), )); + } else if format.ends_with("zip") { + #[cfg(feature = "ocel-bundle")] + { + use crate::core::event_data::object_centric::ocel_bundle::{ + write_ocel_bundle_archive, StorageFormat, + }; + // A ZIP is assembled by seeking back to fix up each entry's header, which a bare + // `Write` cannot do, so it is built in memory and handed over whole. + let mut buffer = std::io::Cursor::new(Vec::new()); + write_ocel_bundle_archive( + self, + &mut buffer, + if format.starts_with("ocel-parquet") { + StorageFormat::Parquet + } else { + StorageFormat::Csv + }, + ) + .map_err(OCELIOError::from)?; + let mut writer = writer; + writer.write_all(&buffer.into_inner())?; + Ok(()) + } + #[cfg(not(feature = "ocel-bundle"))] + return Err(OCELIOError::UnsupportedFormat( + "bundled CSV/Parquet support not enabled".to_string(), + )); } else if format.ends_with("duckdb") { Err(OCELIOError::UnsupportedFormat( "DuckDB export to writer not supported".to_string(), @@ -368,6 +522,10 @@ impl Exportable for OCEL { ExtensionWithMime::new("xml.gz", "application/gzip"), ExtensionWithMime::new("ocel.csv", "text/csv"), ExtensionWithMime::new("ocel.csv.gz", "application/gzip"), + #[cfg(feature = "ocel-bundle")] + ExtensionWithMime::new("ocel.zip", "application/zip"), + #[cfg(feature = "ocel-bundle-parquet")] + ExtensionWithMime::new("ocel-parquet.zip", "application/zip"), #[cfg(feature = "ocel-sqlite")] ExtensionWithMime::new("sqlite", "application/x-sqlite3"), #[cfg(feature = "ocel-duckdb")] diff --git a/process_mining/src/core/event_data/object_centric/linked_ocel/index_linked_ocel.rs b/process_mining/src/core/event_data/object_centric/linked_ocel/index_linked_ocel.rs index b02c3690..20b88c83 100644 --- a/process_mining/src/core/event_data/object_centric/linked_ocel/index_linked_ocel.rs +++ b/process_mining/src/core/event_data/object_centric/linked_ocel/index_linked_ocel.rs @@ -116,6 +116,16 @@ pub struct IndexLinkedOCEL { o2o_rel: Vec>, e2o_rel_rev: Vec>, o2o_rel_rev: Vec>, + /// Event type names, indexed by the `usize` ids in [`Self::event_type_idx`]. Superset of + /// `ocel.event_types`: an event whose type isn't declared there still gets an id here + /// (assigned on first sight), matching `get_ev_type_of`'s tolerance of undeclared types. + event_type_names: Vec, + /// See [`Self::event_type_names`]. + object_type_names: Vec, + /// `ocel.events[i]`'s type id into [`Self::event_type_names`]. + event_type_idx: Vec, + /// `ocel.objects[i]`'s type id into [`Self::object_type_names`]. + object_type_idx: Vec, } impl IndexLinkedOCEL { @@ -215,6 +225,35 @@ impl Index<&ObjectIndex> for &IndexLinkedOCEL { } } +/// Assign each item's type name a stable `usize` id: `declared`'s order first, then any +/// name seen in `items` but missing from `declared` gets the next free id (an event/object +/// whose type isn't pre-declared is tolerated elsewhere in this file, e.g. `get_ev_type_of`, +/// so this must not panic on one). +fn build_type_index<'a>( + declared: &[OCELType], + items: impl Iterator, +) -> (Vec, Vec) { + let mut names: Vec = declared.iter().map(|t| t.name.clone()).collect(); + let mut index: HashMap = names + .iter() + .cloned() + .enumerate() + .map(|(i, n)| (n, i)) + .collect(); + let idx = items + .map(|name| match index.get(name) { + Some(&i) => i, + None => { + let i = names.len(); + names.push(name.to_string()); + index.insert(name.to_string(), i); + i + } + }) + .collect(); + (names, idx) +} + impl From for IndexLinkedOCEL { fn from(mut ocel: OCEL) -> Self { // Sort events so that the index order corresponds to the timstamp order @@ -335,6 +374,15 @@ impl From for IndexLinkedOCEL { }) .collect(); + let (event_type_names, event_type_idx) = build_type_index( + &ocel.event_types, + ocel.events.iter().map(|e| e.event_type.as_str()), + ); + let (object_type_names, object_type_idx) = build_type_index( + &ocel.object_types, + ocel.objects.iter().map(|o| o.object_type.as_str()), + ); + Self { ocel, event_ids_to_index, @@ -347,10 +395,28 @@ impl From for IndexLinkedOCEL { o2o_rel, e2o_rel_rev, o2o_rel_rev, + event_type_names, + object_type_names, + event_type_idx, + object_type_idx, } } } +#[cfg(feature = "ocel-duckdb")] +impl IndexLinkedOCEL { + /// Build an in-memory index by fully materializing a `DuckDB` connection (see + /// [`stream_ocel_file_to_duckdb`](crate::core::event_data::object_centric::ocel_sql::stream_ocel_file_to_duckdb)). + /// + /// Convenience eager-load for logs that fit in memory; for out-of-core access use + /// `DuckDbLinkedOCEL` + /// instead. + pub fn from_duckdb(con: &duckdb::Connection) -> Result { + let ocel = crate::core::event_data::object_centric::ocel_sql::read_ocel_from_duckdb(con)?; + Ok(Self::from_ocel(ocel)) + } +} + impl<'a> LinkedOCELAccess<'a> for IndexLinkedOCEL { type EventRepr = EventIndex; type ObjectRepr = ObjectIndex; diff --git a/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs b/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs index 5dc805cc..74eb87aa 100644 --- a/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs +++ b/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs @@ -19,11 +19,10 @@ use uuid::Uuid; use crate::{ core::{ event_data::object_centric::{ - appendable::AppendableOCEL, + appendable::{is_streaming_format, AppendableOCEL, StreamImportOCEL}, io::OCELIOError, linked_ocel::LinkedOCELAccess, - ocel_json::import_ocel_json_into, - ocel_xml::xml_ocel_import::{import_ocel_xml_into, OCELImportOptions}, + ocel_xml::xml_ocel_import::OCELImportOptions, readable::{OCELLookup, ReadableOCEL}, OCELAttributeType, OCELAttributeValue, OCELEvent, OCELEventAttribute, OCELObject, OCELObjectAttribute, OCELRelationship, OCELType, OCELTypeAttribute, @@ -243,7 +242,13 @@ impl EventIndex { } /// Get a mutable reference to the attribute value of this event, specified by the attribute name /// - /// Returns [`None`] if there is no such attribute. + /// Returns [`None`] if the event's type does not declare `attr_name`. + /// + /// An event's attribute vector is sized when the event is added, so a type that grows a new + /// attribute afterwards leaves earlier events of that type short of it. This grows the vector + /// to `index + 1`, padding with [`OCELAttributeValue::Null`], rather than reporting a declared + /// attribute as absent. Attributes past this one stay absent until they are themselves asked + /// for, and [`Self::get_attribute_value`] still answers [`None`] for them. pub fn get_attribute_value_mut<'a>( &self, attr_name: &str, @@ -255,8 +260,10 @@ impl EventIndex { .iter() .enumerate() .find(|(_i, a)| a.name == attr_name)?; - let attr_val = ev.attributes.get_mut(index)?; - Some(attr_val) + if ev.attributes.len() <= index { + ev.attributes.resize(index + 1, OCELAttributeValue::Null); + } + ev.attributes.get_mut(index) } /// Get 'fat' version of Event (i.e., with all fields expanded, with a structure similar to the OCEL 2.0 specification) pub fn fat_ev(&self, locel: &SlimLinkedOCEL) -> OCELEvent { @@ -503,8 +510,12 @@ impl ObjectIndex { .iter() .enumerate() .find(|(_i, a)| a.name == attr_name)?; - let attr_val = ob.attributes.get_mut(index)?; - Some(attr_val) + // See `EventIndex::get_attribute_value_mut`: a type that grew an attribute after this + // object was added leaves the object's vector short of it. + if ob.attributes.len() <= index { + ob.attributes.resize_with(index + 1, Vec::new); + } + ob.attributes.get_mut(index) } fn fat_ob(&self, locel: &SlimLinkedOCEL) -> OCELObject { @@ -606,6 +617,19 @@ impl SlimLinkedOCEL { pub fn new() -> Self { Self::default() } + + /// Build a `SlimLinkedOCEL` from a `DuckDB` schema database (as written by + /// [`stream_ocel_file_to_duckdb`](crate::core::event_data::object_centric::ocel_sql::stream_ocel_file_to_duckdb)). + /// Eager, so the whole log is loaded into memory. For out-of-core access use + /// `DuckDbLinkedOCEL`. + #[cfg(feature = "ocel-duckdb")] + pub fn from_duckdb(con: &duckdb::Connection) -> Result { + use crate::core::event_data::object_centric::ocel_sql::duckdb::schema::reader::DuckDbReadInto; + let mut slim = SlimLinkedOCEL::new(); + slim.read_from_duckdb(con)?; + Ok(slim) + } + /// Convert an unlinked [`OCEL`] to a [`SlimLinkedOCEL`]. /// /// Events are sorted by time before insertion so that `events_per_type` lists are @@ -677,7 +701,10 @@ impl SlimLinkedOCEL { .flat_map(|et| &self.events_per_type[*et]) } /// Get all objects of the specified object type - fn get_obs_of_type<'a>(&'a self, object_type: &str) -> impl Iterator { + pub(crate) fn get_obs_of_type<'a>( + &'a self, + object_type: &str, + ) -> impl Iterator { self.obtype_to_index .get(object_type) .into_iter() @@ -1743,27 +1770,45 @@ impl Importable for SlimLinkedOCEL { format: &str, _: Self::ImportOptions, ) -> Result { - if let Some(inner) = format.strip_suffix(".gz") { - // Buffer the compressed bytes; `GzDecoder` reads from its inner in chunks. - let gz: Box = Box::new(flate2::read::GzDecoder::new( - std::io::BufReader::new(reader), - )); - return Self::import_from_reader_with_options(gz, inner, ()); - } - if format.ends_with("xml") || format.ends_with("xmlocel") { - let mut xml_reader = quick_xml::Reader::from_reader(std::io::BufReader::new(reader)); + if is_streaming_format(format) { let mut slim = SlimLinkedOCEL::new(); - import_ocel_xml_into(&mut xml_reader, &mut slim, OCELImportOptions::default())?; + slim.stream_ocel_from_reader(reader, format, OCELImportOptions::default())?; slim.finalize()?; Ok(slim) - } else if format.ends_with("json") || format.ends_with("jsonocel") { + } else { + Ok(SlimLinkedOCEL::from_ocel(OCEL::import_from_reader( + reader, format, + )?)) + } + } + + fn import_from_path_with_options>( + path: P, + _: Self::ImportOptions, + ) -> Result { + let path = path.as_ref(); + let format = ::infer_format(path).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Could not infer format from path", + ) + })?; + if is_streaming_format(&format) { let mut slim = SlimLinkedOCEL::new(); - import_ocel_json_into(std::io::BufReader::new(reader), &mut slim)?; + slim.stream_ocel_from_reader( + std::fs::File::open(path)?, + &format, + OCELImportOptions::default(), + )?; slim.finalize()?; Ok(slim) } else { - let ocel = OCEL::import_from_reader(reader, format)?; - Ok(SlimLinkedOCEL::from_ocel(ocel)) + // Formats needing true path access (a directory/`.ocel.zip` bundle, SQLite, DuckDB) + // only work through `OCEL`'s path-aware importer. The default `Importable` impl + // would instead `File::open` the path and hand it to `import_from_reader_with_options` + // as a flat byte stream, which fails outright for a directory and silently + // misinterprets a bundle's manifest file as the archive itself. + Ok(SlimLinkedOCEL::from_ocel(OCEL::import_from_path(path)?)) } } @@ -1787,7 +1832,7 @@ impl Exportable for SlimLinkedOCEL { fn export_to_path_with_options>( &self, path: P, - _: Self::ExportOptions, + options: Self::ExportOptions, ) -> Result<(), Self::Error> { let path = path.as_ref(); let format = ::infer_format(path).ok_or_else(|| { @@ -1796,6 +1841,18 @@ impl Exportable for SlimLinkedOCEL { "Could not infer format from path", ) })?; + ::export_to_path_as(self, path, &format, options) + } + + /// See [`Exportable::export_to_path_as`]. Formats that write a file or a directory rather + /// than a byte stream are handled here, everything else streams. + fn export_to_path_as>( + &self, + path: P, + format: &str, + _: Self::ExportOptions, + ) -> Result<(), Self::Error> { + let path = path.as_ref(); if format.ends_with("sqlite") || (format.ends_with("db") && !format.ends_with("duckdb")) { #[cfg(feature = "ocel-sqlite")] return crate::core::event_data::object_centric::ocel_sql::export_ocel_sqlite_to_path( @@ -1818,9 +1875,37 @@ impl Exportable for SlimLinkedOCEL { "DuckDB support not enabled".to_string(), )); } + #[cfg(feature = "ocel-bundle")] + if format.ends_with("zip") { + use crate::core::event_data::object_centric::ocel_bundle::{ + export_ocel_bundle, BundleExportOptions, ContainerLayout, StorageFormat, + }; + // An existing directory, or a name with no extension, is the uncompressed form. + let layout = if path.is_dir() || path.extension().is_none() { + ContainerLayout::Directory + } else { + ContainerLayout::Archive + }; + if layout == ContainerLayout::Directory { + std::fs::create_dir_all(path)?; + } + return export_ocel_bundle( + self, + path, + BundleExportOptions { + layout, + storage: if format.starts_with("ocel-parquet") { + StorageFormat::Parquet + } else { + StorageFormat::Csv + }, + }, + ) + .map_err(|e| OCELIOError::Other(e.to_string())); + } let file = std::fs::File::create(path)?; let writer = std::io::BufWriter::new(file); - Self::export_to_writer(self, writer, &format) + Self::export_to_writer(self, writer, format) } fn export_to_writer_with_options( diff --git a/process_mining/src/core/event_data/object_centric/mod.rs b/process_mining/src/core/event_data/object_centric/mod.rs index f83f4da6..820a1512 100644 --- a/process_mining/src/core/event_data/object_centric/mod.rs +++ b/process_mining/src/core/event_data/object_centric/mod.rs @@ -8,13 +8,19 @@ pub mod appendable; /// #[cfg(feature = "dataframes")] pub mod dataframe; -/// Graph Database OCEL Features (e.g., Export/Import) -/// -#[cfg(feature = "kuzudb")] -pub mod graph_db; +/// Build an OCEL from relational data using a declarative blueprint. +// `ExtractionError` is deliberately descriptive (it carries the `MappingRef` a diagnostic points +// at), so it is well over clippy's `Err`-size threshold. Boxing it would shrink the per-row +// `Result`, at the cost of a `Box` in every construction site and match arm. +#[allow(clippy::result_large_err)] +#[cfg(feature = "extraction-blueprint")] +pub mod extraction; pub mod io; pub mod linked_ocel; pub mod macros; +/// The OCEL 2.0 bundled CSV/Parquet format. +#[cfg(any(feature = "extraction-blueprint", feature = "ocel-bundle"))] +pub mod ocel_bundle; pub mod ocel_csv; pub mod ocel_json; pub mod ocel_sql; diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/blueprint.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/blueprint.rs new file mode 100644 index 00000000..88a2fb76 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/blueprint.rs @@ -0,0 +1,193 @@ +//! The [`Blueprint`] a container's manifest denotes. +//! +//! The bundled format is relational, so importing it is an extraction rather than a parser: one +//! `Source` node per declared table, one mapping per thing that table produces. That reuses the +//! rest of the extraction subsystem, including validation against the discovered schema, a +//! per-mapping [`ExtractionReport`](super::super::extraction::ExtractionReport), and the SQL +//! compiler, which turns a Parquet container into `DuckDB` views that never materialise the log. +//! A CSV container cannot be compiled, since every column is text and the timestamps and numeric +//! attributes need the row executor's parsing. + +use super::meta::{ + columns, event_table, object_changes_table, object_table, AttributeDecl, BundleMeta, E2O_TABLE, + O2O_TABLE, +}; +use crate::core::event_data::object_centric::extraction::{ + AttributeMapping, Blueprint, CompareOp, DuplicateObjectPolicy, EventEndpoint, IdRendering, + Literal, Mapping, MappingEntry, MissingEndpointPolicy, Node, NodeOp, ObjectEndpoint, Operand, + Predicate, Target, TimestampSource, ValueExpression, MODEL_VERSION, +}; + +/// The source id a generated blueprint reads from. +pub const SOURCE_ID: &str = "bundle"; + +/// The blueprint that turns `meta`'s tables into the OCEL they encode. +/// +/// # Why the policies are what they are +/// +/// - [`IdRendering::Raw`]: OCEL ids are globally unique already, and `e2o`/`o2o` name an id with +/// no type column beside it, so there is nothing to prefix with. +/// - [`DuplicateObjectPolicy::FirstWins`]: an object appears once in its own table and again in +/// every change row, which is not a collision but the format working as designed. First-wins +/// creates it once and appends the later rows' attribute values at their own timestamps. +/// - [`MissingEndpointPolicy::Error`]: a relation naming an id no table declares is a broken +/// container, and should be reported rather than papered over by synthesising the object. +#[must_use] +pub fn blueprint_for(meta: &BundleMeta) -> Blueprint { + let mut nodes = Vec::new(); + let mut mappings = Vec::new(); + + for (ty, decl) in &meta.event_types { + let table = event_table(ty); + nodes.push(source(&table)); + mappings.push(single( + &table, + Target::Event { + event_type: constant(ty), + id: Some(column(columns::ID)), + timestamp: TimestampSource::column(columns::TIME), + attributes: attributes(&decl.attributes), + objects: Vec::new(), + }, + )); + } + + for (ty, decl) in &meta.object_types { + let table = object_table(ty); + nodes.push(source(&table)); + // No timestamp: the format reads an object table's values as initial ones, held from the + // Unix epoch, which is exactly what a `None` timestamp records. + mappings.push(single( + &table, + Target::Object { + object_type: constant(ty), + id: column(columns::ID), + timestamp: None, + attributes: attributes(&decl.attributes), + }, + )); + + if decl.changes_file.is_none() { + continue; + } + let changes = object_changes_table(ty); + nodes.push(source(&changes)); + // One filtered node per attribute, rather than one mapping carrying every attribute + // column. A change row fills only the column `ocel_changed_field` names and leaves the + // rest empty, so a single mapping would record those empty cells as an attribute being + // set to null at that instant, a change the container never declared. + for attr in &decl.attributes { + let filtered = format!("{changes}#{}", attr.name); + nodes.push(Node { + id: filtered.clone(), + label: None, + op: NodeOp::Filter { + input: changes.clone(), + condition: Predicate::Compare { + left: Operand::Column { + column: columns::CHANGED_FIELD.to_string(), + }, + op: CompareOp::Eq, + right: Operand::Literal { + value: Literal::Text(attr.name.clone()), + }, + }, + }, + }); + mappings.push(single( + &filtered, + Target::Object { + object_type: constant(ty), + id: column(columns::ID), + timestamp: Some(TimestampSource::column(columns::TIME)), + attributes: attributes(std::slice::from_ref(attr)), + }, + )); + } + } + + nodes.push(source(E2O_TABLE)); + mappings.push(single( + E2O_TABLE, + Target::E2O { + event: EventEndpoint { + id: column(columns::EVENT_ID), + event_type: None, + }, + object: endpoint(columns::OBJECT_ID), + qualifier: Some(column(columns::QUALIFIER)), + }, + )); + + nodes.push(source(O2O_TABLE)); + mappings.push(single( + O2O_TABLE, + Target::O2O { + source: endpoint(columns::SOURCE_ID), + target: endpoint(columns::TARGET_ID), + qualifier: Some(column(columns::QUALIFIER)), + }, + )); + + Blueprint { + version: MODEL_VERSION, + id_rendering: IdRendering::Raw, + nodes, + mappings, + on_missing_endpoint: MissingEndpointPolicy::Error, + on_duplicate_object: DuplicateObjectPolicy::FirstWins, + } +} + +fn source(table: &str) -> Node { + Node { + id: table.to_string(), + label: None, + op: NodeOp::Source { + source_id: SOURCE_ID.to_string(), + table: table.to_string(), + }, + } +} + +fn single(node: &str, target: Target) -> MappingEntry { + MappingEntry::Single(Mapping { + node: node.to_string(), + label: None, + when: None, + target, + }) +} + +fn column(name: &str) -> ValueExpression { + ValueExpression::Column { + column: name.to_string(), + } +} + +fn constant(value: &str) -> ValueExpression { + ValueExpression::Constant { + value: value.to_string(), + } +} + +/// A relation endpoint. No `object_type`: the relation tables carry ids only, which is legal +/// exactly because ids are rendered raw. +fn endpoint(id_column: &str) -> ObjectEndpoint { + ObjectEndpoint { + id: column(id_column), + object_type: None, + split: None, + } +} + +fn attributes(decls: &[AttributeDecl]) -> Vec { + decls + .iter() + .map(|a| AttributeMapping { + source_column: a.name.clone(), + name: a.name.clone(), + value_type: Some(a.value_type.into()), + }) + .collect() +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/export.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/export.rs new file mode 100644 index 00000000..720d091c --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/export.rs @@ -0,0 +1,814 @@ +//! Writing an OCEL as a bundled container. +//! +//! An object's attributes are time-versioned, and the format splits them across two tables: the +//! object table holds the values in force from the Unix epoch, and the object-change table holds +//! every later observation, one row per changed attribute. The split is therefore by timestamp, +//! and lossless in both directions. +//! +//! An object whose earliest observation is after the epoch has no initial value, and its cell is +//! left empty rather than back-dating the first observation. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; +use std::path::Path; + +use super::meta::{ + columns, encode_type_name, epoch, AttributeDecl, AttributeType, BundleMeta, EventTypeDecl, + ObjectTypeDecl, RelationFiles, StorageFormat, Value, BUNDLE_FORMAT_VERSION, META_FILE_NAME, + OCEL_VERSION, +}; +use crate::core::event_data::object_centric::readable::ReadableOCEL; +use crate::core::event_data::object_centric::{ + OCELAttributeType, OCELAttributeValue, OCELTypeAttribute, +}; + +/// Whether a container is one file or a tree of them. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ContainerLayout { + /// A `.ocel.zip` archive. + #[default] + Archive, + /// A directory with the archive's internal layout. + Directory, +} + +/// How to write a container. +#[derive(Debug, Clone, Copy, Default)] +pub struct BundleExportOptions { + /// One file or a directory. + pub layout: ContainerLayout, + /// Which physical storage the tables use. + pub storage: StorageFormat, +} + +/// Why writing a container failed. +#[derive(Debug)] +pub enum BundleExportError { + /// Writing to the target failed. + Io(std::io::Error), + /// Assembling the archive failed. + Archive(String), + /// Encoding a table failed. + Encode(String), + /// A declared attribute name collides with a column the format fixes. + ReservedAttribute { + /// The type declaring it. + type_name: String, + /// The attribute name, which is also a `ocel_`-prefixed fixed column name. + attribute: String, + }, + /// Parquet storage was asked for in a build without the `ocel-bundle-parquet` feature. + ParquetUnavailable, +} + +impl std::fmt::Display for BundleExportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BundleExportError::Io(e) => write!(f, "writing the container failed: {e}"), + BundleExportError::Archive(m) => write!(f, "assembling the archive failed: {m}"), + BundleExportError::Encode(m) => write!(f, "encoding a table failed: {m}"), + BundleExportError::ReservedAttribute { + type_name, + attribute, + } => write!( + f, + "'{type_name}' declares an attribute named '{attribute}', which is a column name the format reserves" + ), + BundleExportError::ParquetUnavailable => write!( + f, + "Parquet storage needs the 'ocel-bundle-parquet' feature; this build has only CSV" + ), + } + } +} + +impl std::error::Error for BundleExportError {} + +impl From for BundleExportError { + fn from(e: std::io::Error) -> Self { + BundleExportError::Io(e) + } +} + +/// Write `ocel` to `path` as a bundled container. +/// +/// # Errors +/// See [`BundleExportError`]. +pub fn export_ocel_bundle( + ocel: &O, + path: P, + options: BundleExportOptions, +) -> Result<(), BundleExportError> +where + O: ReadableOCEL + ?Sized, + P: AsRef, +{ + #[cfg(not(feature = "ocel-bundle-parquet"))] + if options.storage == StorageFormat::Parquet { + return Err(BundleExportError::ParquetUnavailable); + } + + let files = build(ocel, options.storage)?; + match options.layout { + ContainerLayout::Directory => { + for (rel, bytes) in files { + let out = path.as_ref().join(&rel); + if let Some(parent) = out.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(out, bytes)?; + } + Ok(()) + } + ContainerLayout::Archive => { + let file = std::fs::File::create(path)?; + write_archive(std::io::BufWriter::new(file), files, options.storage) + } + } +} + +/// Write the archive form to a writer, for a caller with no path (an HTTP response, say). +/// +/// # Errors +/// See [`BundleExportError`]. +pub fn write_ocel_bundle_archive( + ocel: &O, + writer: W, + storage: StorageFormat, +) -> Result<(), BundleExportError> +where + O: ReadableOCEL + ?Sized, + W: Write + std::io::Seek, +{ + #[cfg(not(feature = "ocel-bundle-parquet"))] + if storage == StorageFormat::Parquet { + return Err(BundleExportError::ParquetUnavailable); + } + write_archive(writer, build(ocel, storage)?, storage) +} + +fn write_archive( + writer: W, + files: Vec<(String, Vec)>, + storage: StorageFormat, +) -> Result<(), BundleExportError> { + let mut zw = zip::ZipWriter::new(writer); + // Parquet entries are stored rather than deflated so a reader can seek within them without + // expanding the archive: a Parquet file is read footer-first, and a deflated entry has to be + // fully inflated before that footer can be found. That costs nothing in size only because + // `writer_properties` compresses each column chunk with ZSTD, so deflating the entry on top + // would buy almost nothing and give up the seeking. CSV is read start to finish either way, + // so it is deflated. + let table_method = match storage { + StorageFormat::Csv => zip::CompressionMethod::Deflated, + StorageFormat::Parquet => zip::CompressionMethod::Stored, + }; + for (rel, bytes) in files { + let method = if rel == META_FILE_NAME { + zip::CompressionMethod::Deflated + } else { + table_method + }; + let opts: zip::write::FileOptions<'_, ()> = + zip::write::FileOptions::default().compression_method(method); + zw.start_file(&rel, opts) + .map_err(|e| BundleExportError::Archive(e.to_string()))?; + zw.write_all(&bytes)?; + } + zw.finish() + .map_err(|e| BundleExportError::Archive(e.to_string()))?; + Ok(()) +} + +/// Every file the container holds, as `(path inside the container, contents)`. +fn build( + ocel: &O, + storage: StorageFormat, +) -> Result)>, BundleExportError> { + let ext = storage.extension(); + let mut files: Vec<(String, Vec)> = Vec::new(); + let mut meta = BundleMeta { + ocel_version: OCEL_VERSION.to_string(), + bundle_format_version: BUNDLE_FORMAT_VERSION.to_string(), + storage_format: storage, + event_types: BTreeMap::new(), + object_types: BTreeMap::new(), + relations: RelationFiles { + e2o: format!("relations/e2o.{ext}"), + o2o: format!("relations/o2o.{ext}"), + }, + }; + + let mut e2o = Table::new( + &[columns::EVENT_ID, columns::OBJECT_ID, columns::QUALIFIER], + &[], + ); + for ev in ocel.iter_events() { + for rel in &ev.relationships { + e2o.push(vec![ + text(&ev.id), + text(&rel.object_id), + text(&rel.qualifier), + ]); + } + } + files.push((meta.relations.e2o.clone(), e2o.encode(storage)?)); + + let mut o2o = Table::new( + &[columns::SOURCE_ID, columns::TARGET_ID, columns::QUALIFIER], + &[], + ); + for ob in ocel.iter_objects() { + for rel in &ob.relationships { + o2o.push(vec![ + text(&ob.id), + text(&rel.object_id), + text(&rel.qualifier), + ]); + } + } + files.push((meta.relations.o2o.clone(), o2o.encode(storage)?)); + + // One pass per entity kind, not one per declared type. + // + // `ReadableOCEL::iter_events_of_type`/`iter_objects_of_type` default to filtering a full + // scan, and `OCEL` does not override them, so reading them once per type made the export + // O(types x entities), i.e. one complete pass per type. The tables are built up front and + // every entity is routed to its own in a single pass instead. + let mut event_tables: BTreeMap)> = BTreeMap::new(); + for ty in ocel.event_types() { + let attrs = declared(&ty.name, &ty.attributes)?; + event_tables.insert(ty.name.clone(), (event_table(&attrs), attrs)); + } + for ev in ocel.iter_events() { + // An entity of a type the log never declared still has to be written: the relation tables + // cover every entity, so skipping it would leave relation rows pointing at nothing. + if !event_tables.contains_key(ev.event_type.as_str()) { + event_tables.insert(ev.event_type.clone(), (event_table(&[]), Vec::new())); + } + let (table, attrs) = event_tables + .get_mut(ev.event_type.as_str()) + .expect("just inserted"); + let mut row = vec![text(&ev.id), Some(Value::Time(ev.time))]; + for a in attrs.iter() { + let value = ev + .attributes + .iter() + .find(|x| x.name == a.name) + .map(|x| &x.value); + row.push(cell(value, a.value_type)); + } + table.push(row); + } + for (ty, (table, attributes)) in event_tables { + let file = format!("events/event_{}.{ext}", encode_type_name(&ty)); + files.push((file.clone(), table.encode(storage)?)); + meta.event_types + .insert(ty, EventTypeDecl { file, attributes }); + } + + let mut object_tables: BTreeMap)> = BTreeMap::new(); + for ty in ocel.object_types() { + let attrs = declared(&ty.name, &ty.attributes)?; + let (objects, changes) = object_and_change_tables(&attrs); + object_tables.insert(ty.name.clone(), (objects, changes, attrs)); + } + for ob in ocel.iter_objects() { + if !object_tables.contains_key(ob.object_type.as_str()) { + let (objects, changes) = object_and_change_tables(&[]); + object_tables.insert(ob.object_type.clone(), (objects, changes, Vec::new())); + } + let (objects, changes, attrs) = object_tables + .get_mut(ob.object_type.as_str()) + .expect("just inserted"); + + // The object table holds one value per attribute, so only the first observation at the + // epoch goes there. A second one at the same instant is routed to the change table. + let mut in_object_row: BTreeSet<&str> = BTreeSet::new(); + let mut later = Vec::new(); + for obs in &ob.attributes { + if obs.time == epoch() && in_object_row.insert(obs.name.as_str()) { + continue; + } + later.push(obs); + } + + let mut row = vec![text(&ob.id)]; + for a in attrs.iter() { + let initial = ob + .attributes + .iter() + .find(|x| x.name == a.name && x.time == epoch()) + .map(|x| &x.value); + row.push(cell(initial, a.value_type)); + } + objects.push(row); + + // One row per remaining observation, naming the attribute it changed and leaving the + // other attribute columns empty. Reading a change row's other columns as values would + // record changes the log never had. + later.sort_by(|a, b| a.time.cmp(&b.time).then_with(|| a.name.cmp(&b.name))); + for obs in later { + let Some(pos) = attrs.iter().position(|a| a.name == obs.name) else { + continue; + }; + let mut row = vec![text(&ob.id), Some(Value::Time(obs.time)), text(&obs.name)]; + for (i, a) in attrs.iter().enumerate() { + row.push(if i == pos { + cell(Some(&obs.value), a.value_type) + } else { + None + }); + } + changes.push(row); + } + } + for (ty, (objects, changes, attributes)) in object_tables { + let enc = encode_type_name(&ty); + let file = format!("objects/object_{enc}.{ext}"); + let changes_file = format!("object_changes/object_changes_{enc}.{ext}"); + files.push((file.clone(), objects.encode(storage)?)); + files.push((changes_file.clone(), changes.encode(storage)?)); + meta.object_types.insert( + ty, + ObjectTypeDecl { + file, + changes_file: Some(changes_file), + attributes, + }, + ); + } + + let manifest = + serde_json::to_vec_pretty(&meta).map_err(|e| BundleExportError::Encode(e.to_string()))?; + files.push((META_FILE_NAME.to_string(), manifest)); + Ok(files) +} + +/// The manifest's attribute list for one type, deduplicated and ordered so two exports of the +/// same log are byte-identical. +/// +/// # Errors +/// An attribute named after a column the format fixes would give the table two columns of that +/// name, so it is refused rather than written. +fn declared( + type_name: &str, + attrs: &[OCELTypeAttribute], +) -> Result, BundleExportError> { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut out = Vec::new(); + for a in attrs { + if is_fixed_column(&a.name) { + return Err(BundleExportError::ReservedAttribute { + type_name: type_name.to_string(), + attribute: a.name.clone(), + }); + } + if !seen.insert(a.name.as_str()) { + continue; + } + out.push(AttributeDecl { + name: a.name.clone(), + // `from_type_str` maps anything it does not know onto `Null`, which is not an OCEL + // type; the format has no such variant, so it is written as `string`. + value_type: match OCELAttributeType::from_type_str(&a.value_type) { + OCELAttributeType::String | OCELAttributeType::Null => AttributeType::String, + OCELAttributeType::Time => AttributeType::Time, + OCELAttributeType::Integer => AttributeType::Integer, + OCELAttributeType::Float => AttributeType::Float, + OCELAttributeType::Boolean => AttributeType::Boolean, + }, + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) +} + +/// Whether `name` is a column name the format fixes. +fn is_fixed_column(name: &str) -> bool { + matches!( + name, + columns::ID + | columns::TIME + | columns::CHANGED_FIELD + | columns::EVENT_ID + | columns::OBJECT_ID + | columns::SOURCE_ID + | columns::TARGET_ID + | columns::QUALIFIER + ) +} + +fn event_table(attrs: &[AttributeDecl]) -> Table { + let mut table = Table::new(&[columns::ID, columns::TIME], attrs); + table.set_time_column(1); + table +} + +fn object_and_change_tables(attrs: &[AttributeDecl]) -> (Table, Table) { + let objects = Table::new(&[columns::ID], attrs); + let mut changes = Table::new(&[columns::ID, columns::TIME, columns::CHANGED_FIELD], attrs); + changes.set_time_column(1); + (objects, changes) +} + +fn text(value: &str) -> Option { + Some(Value::Text(value.to_string())) +} + +/// `value` rendered as `declared`, or `None` for a cell the log has no value for. +/// +/// A value that does not fit its declared type falls back to its own text, which CSV storage +/// carries fine. Parquet storage writes null rather than a value of the wrong physical type. +fn cell(value: Option<&OCELAttributeValue>, declared: AttributeType) -> Option { + let value = value?; + Some(match (value, declared) { + (OCELAttributeValue::Null, _) => return None, + (OCELAttributeValue::Integer(i), AttributeType::Integer) => Value::Integer(*i), + (OCELAttributeValue::Integer(i), AttributeType::Float) => Value::Float(*i as f64), + (OCELAttributeValue::Float(f), AttributeType::Float) => Value::Float(*f), + (OCELAttributeValue::Boolean(b), AttributeType::Boolean) => Value::Boolean(*b), + (OCELAttributeValue::Time(t), AttributeType::Time) => Value::Time(*t), + (v, _) => Value::Text(v.to_string()), + }) +} + +/// A table being assembled, held column-wise at encode time because Parquet is written that way. +struct Table { + header: Vec, + rows: Vec>>, + /// How many leading columns the format fixes, and so `required` in Parquet storage. Counted + /// when the header is built, so an attribute named like a fixed column stays optional. + #[cfg_attr(not(feature = "ocel-bundle-parquet"), allow(dead_code))] + fixed: usize, + /// Index of the `ocel_time` column, when the table has one. + time_column: Option, +} + +impl Table { + fn new(fixed: &[&str], attrs: &[AttributeDecl]) -> Self { + Self { + header: fixed + .iter() + .map(|s| (*s).to_string()) + .chain(attrs.iter().map(|a| a.name.clone())) + .collect(), + rows: Vec::new(), + fixed: fixed.len(), + time_column: None, + } + } + + fn set_time_column(&mut self, index: usize) { + self.time_column = Some(index); + } + + fn push(&mut self, row: Vec>) { + debug_assert_eq!(row.len(), self.header.len()); + self.rows.push(row); + } + + fn encode(&self, storage: StorageFormat) -> Result, BundleExportError> { + match storage { + StorageFormat::Csv => self.to_csv(), + #[cfg(feature = "ocel-bundle-parquet")] + StorageFormat::Parquet => self.to_parquet(), + #[cfg(not(feature = "ocel-bundle-parquet"))] + StorageFormat::Parquet => Err(BundleExportError::ParquetUnavailable), + } + } + + fn to_csv(&self) -> Result, BundleExportError> { + let mut w = csv::Writer::from_writer(Vec::new()); + w.write_record(&self.header) + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + for row in &self.rows { + w.write_record( + row.iter() + .map(|c| c.as_ref().map(Value::to_string).unwrap_or_default()), + ) + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + } + w.into_inner() + .map_err(|e| BundleExportError::Encode(e.to_string())) + } + + #[cfg(feature = "ocel-bundle-parquet")] + fn to_parquet(&self) -> Result, BundleExportError> { + use parquet::basic::{LogicalType, Repetition, TimeUnit, Type as PhysicalType}; + use parquet::data_type::{BoolType, ByteArray, ByteArrayType, DoubleType, Int64Type}; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type; + use std::sync::Arc; + + let fixed = self.fixed; + // Built through the schema API rather than by formatting a `message { ... }` string: + // an attribute name is arbitrary log data, and one containing a space (or a brace, or a + // semicolon) does not survive the text parser. + let mut fields = Vec::with_capacity(self.header.len()); + for (i, name) in self.header.iter().enumerate() { + let repetition = if i < fixed { + Repetition::REQUIRED + } else { + Repetition::OPTIONAL + }; + let (physical, logical) = match self.column_kind(i) { + ColumnKind::Time => ( + PhysicalType::INT64, + Some(LogicalType::Timestamp { + is_adjusted_to_u_t_c: true, + unit: self.time_unit(i), + }), + ), + ColumnKind::Integer => (PhysicalType::INT64, None), + ColumnKind::Float => (PhysicalType::DOUBLE, None), + ColumnKind::Boolean => (PhysicalType::BOOLEAN, None), + ColumnKind::Text => (PhysicalType::BYTE_ARRAY, Some(LogicalType::String)), + }; + let field = Type::primitive_type_builder(name, physical) + .with_repetition(repetition) + .with_logical_type(logical) + .build() + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + fields.push(Arc::new(field)); + } + let schema = Arc::new( + Type::group_type_builder("row") + .with_fields(fields) + .build() + .map_err(|e| BundleExportError::Encode(e.to_string()))?, + ); + + let mut out = Vec::new(); + let mut writer = SerializedFileWriter::new(&mut out, schema, Arc::new(writer_properties())) + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + // A table with no rows is a row group with no rows; the header alone is the schema, so + // an empty change table still declares its columns. + { + let mut group = writer + .next_row_group() + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + for i in 0..self.header.len() { + let mut col = group + .next_column() + .map_err(|e| BundleExportError::Encode(e.to_string()))? + .ok_or_else(|| { + BundleExportError::Encode("fewer columns than the schema".to_string()) + })?; + let optional = i >= fixed; + let name = self.header[i].as_str(); + match self.column_kind(i) { + ColumnKind::Time => { + let unit = self.time_unit(i); + write_column::( + &mut col, + &self.rows, + i, + optional, + name, + |c| match c { + Some(Value::Time(t)) => match unit { + TimeUnit::NANOS => t.timestamp_nanos_opt(), + _ => Some(t.timestamp_micros()), + }, + _ => None, + }, + )?; + } + ColumnKind::Integer => { + write_column::( + &mut col, + &self.rows, + i, + optional, + name, + |c| match c { + Some(Value::Integer(v)) => Some(*v), + _ => None, + }, + )?; + } + ColumnKind::Float => { + write_column::( + &mut col, + &self.rows, + i, + optional, + name, + |c| match c { + Some(Value::Float(v)) => Some(*v), + Some(Value::Integer(v)) => Some(*v as f64), + _ => None, + }, + )?; + } + ColumnKind::Boolean => { + write_column::( + &mut col, + &self.rows, + i, + optional, + name, + |c| match c { + Some(Value::Boolean(v)) => Some(*v), + _ => None, + }, + )?; + } + // Text is the one kind that holds every value: a cell of another type is + // written as the text it renders to. + ColumnKind::Text => { + write_column::( + &mut col, + &self.rows, + i, + optional, + name, + |c| c.as_ref().map(|v| ByteArray::from(v.to_string().as_str())), + )?; + } + } + col.close() + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + } + group + .close() + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + } + writer + .close() + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + Ok(out) + } + + /// The physical type column `i` is written as. Taken from the cells rather than from the + /// manifest so a fixed column (always text, except `ocel_time`) needs no special case, and + /// an all-null attribute column still gets a type. + #[cfg(feature = "ocel-bundle-parquet")] + fn column_kind(&self, i: usize) -> ColumnKind { + if self.time_column == Some(i) { + return ColumnKind::Time; + } + for row in &self.rows { + match row[i] { + None => continue, + Some(Value::Integer(_)) => return ColumnKind::Integer, + Some(Value::Float(_)) => return ColumnKind::Float, + Some(Value::Boolean(_)) => return ColumnKind::Boolean, + Some(Value::Time(_)) => return ColumnKind::Time, + Some(Value::Text(_)) => return ColumnKind::Text, + } + } + ColumnKind::Text + } + + /// The unit a timestamp column is written in. + /// + /// Nanoseconds, so a sub-microsecond instant survives. `timestamp_nanos_opt` covers only + /// 1677..=2262, and the unit belongs to the column rather than the cell, so one instant + /// outside that range puts the whole column back on microseconds. + #[cfg(feature = "ocel-bundle-parquet")] + fn time_unit(&self, i: usize) -> parquet::basic::TimeUnit { + use parquet::basic::TimeUnit; + let representable = self.rows.iter().all(|r| match &r[i] { + Some(Value::Time(t)) => t.timestamp_nanos_opt().is_some(), + _ => true, + }); + if representable { + TimeUnit::NANOS + } else { + TimeUnit::MICROS + } + } +} + +/// How every table in a Parquet container is written. +/// +/// `parquet`'s writers default to no block compression, on the reasoning that a reader +/// streaming from object storage would rather have decode throughput. A container is a file +/// someone keeps and sends, so the tradeoff runs the other way: without this an exported +/// container came out several times larger than the same log as CSV, which is the opposite of +/// what choosing Parquet is for. ZSTD is what `parquet`'s own docs recommend for ratio, +/// speed and ecosystem support together. +#[cfg(feature = "ocel-bundle-parquet")] +fn writer_properties() -> parquet::file::properties::WriterProperties { + use parquet::basic::{Compression, ZstdLevel}; + parquet::file::properties::WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .build() +} + +#[cfg(feature = "ocel-bundle-parquet")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ColumnKind { + Text, + Integer, + Float, + Boolean, + Time, +} + +/// Write column `i` of `rows`, taking its values and its definition levels from the same `of`. +/// +/// A cell whose type is not the column's has no representation in it. `of` returning `None` +/// writes it as a null, rather than as a zero of the column's type standing in for it. +#[cfg(feature = "ocel-bundle-parquet")] +fn write_column( + col: &mut parquet::file::writer::SerializedColumnWriter<'_>, + rows: &[Vec>], + i: usize, + optional: bool, + name: &str, + of: F, +) -> Result<(), BundleExportError> +where + T: parquet::data_type::DataType, + F: Fn(&Option) -> Option, +{ + let mut values = Vec::with_capacity(rows.len()); + let mut defs = Vec::with_capacity(rows.len()); + for row in rows { + match of(&row[i]) { + Some(v) => { + values.push(v); + defs.push(1); + } + None => defs.push(0), + } + } + if !optional && values.len() != rows.len() { + return Err(BundleExportError::Encode(format!( + "'{name}' is a column the format fixes and cannot hold a null" + ))); + } + col.typed::() + .write_batch(&values, optional.then_some(&defs[..]), None) + .map_err(|e| BundleExportError::Encode(e.to_string()))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn decl(name: &str, value_type: AttributeType) -> AttributeDecl { + AttributeDecl { + name: name.to_string(), + value_type, + } + } + + #[test] + fn a_declared_attribute_list_is_deduplicated_and_ordered() { + let attrs = vec![ + OCELTypeAttribute::new("b", &OCELAttributeType::Integer), + OCELTypeAttribute::new("a", &OCELAttributeType::String), + OCELTypeAttribute::new("b", &OCELAttributeType::Float), + ]; + let out = declared("order", &attrs).expect("no reserved name"); + assert_eq!(out.len(), 2, "the repeat is dropped"); + assert_eq!(out[0].name, "a"); + assert_eq!(out[1].name, "b"); + assert_eq!(out[1].value_type, AttributeType::Integer, "first wins"); + } + + #[test] + fn an_attribute_named_after_a_fixed_column_is_refused() { + let attrs = vec![OCELTypeAttribute::new( + columns::ID, + &OCELAttributeType::String, + )]; + let err = declared("order", &attrs).expect_err("reserved"); + assert!(err.to_string().contains(columns::ID), "{err}"); + } + + #[test] + fn a_value_that_does_not_fit_its_declared_type_falls_back_to_text() { + let c = cell( + Some(&OCELAttributeValue::String("not a number".to_string())), + AttributeType::Integer, + ); + assert!(matches!(c, Some(Value::Text(_))), "{c:?}"); + assert_eq!(c.expect("some").to_string(), "not a number"); + } + + #[test] + fn an_absent_value_is_an_empty_cell() { + assert!(cell(None, AttributeType::String).is_none()); + assert!(cell(Some(&OCELAttributeValue::Null), AttributeType::Integer).is_none()); + } + + /// Fixed-ness is a position, not a name. + #[test] + fn fixed_columns_are_counted_when_the_header_is_built() { + let t = Table::new( + &[columns::ID, columns::TIME], + &[ + decl("resource", AttributeType::String), + decl("ocel_like_name", AttributeType::String), + ], + ); + assert_eq!(t.fixed, 2); + assert_eq!(t.header.len(), 4); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/import.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/import.rs new file mode 100644 index 00000000..360006f3 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/import.rs @@ -0,0 +1,765 @@ +//! Reading a bundled container into an [`OCEL`]. +//! +//! This reads the layout directly instead of going through the extraction engine, keeping +//! `.ocel.zip` an ordinary [`Importable`](crate::core::io::Importable) format alongside +//! `.ocel.csv` and `.jsonocel`, readable with no connector and on `wasm32`. The +//! [`Blueprint`](super::blueprint::blueprint_for) route reads the same layout through the +//! extraction engine, for treating a container as a data source in the blueprint editor. Both +//! share this module's [`meta`](super::meta). +//! +//! A directory is read a table at a time straight from disk. An archive is first expanded into a +//! temporary directory, one entry at a time, since a ZIP entry is only seekable when stored +//! uncompressed. Peak memory stays flat either way; an archive additionally costs disk for its +//! expanded size, freed when the [`Container`] drops. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, FixedOffset}; + +use super::meta::{ + columns, epoch, type_attributes, BundleMeta, StorageFormat, Value, BUNDLE_FORMAT_VERSION, + META_FILE_NAME, OCEL_VERSION, +}; +use crate::core::event_data::object_centric::{ + OCELAttributeType, OCELAttributeValue, OCELEvent, OCELEventAttribute, OCELObject, + OCELObjectAttribute, OCELRelationship, OCELType, OCEL, +}; +use crate::core::event_data::timestamp_utils::parse_timestamp; + +/// Why reading a container failed. +#[derive(Debug)] +pub enum BundleImportError { + /// The container could not be read: a missing path, an unreadable archive, or one with no + /// [`META_FILE_NAME`]. + Container(String), + /// [`META_FILE_NAME`] is not a manifest this build understands. + Manifest(serde_json::Error), + /// The manifest declares a major version this build does not implement. + Version { + /// Which manifest field. + field: String, + /// What the container declares. + found: String, + /// What this build implements. + implemented: String, + }, + /// A table the manifest declares is missing, unreadable, or lacks a column the format fixes. + Table { + /// The declared path inside the container. + file: String, + /// What went wrong. + detail: String, + }, + /// Parquet storage in a build without the `ocel-bundle-parquet` feature. + ParquetUnavailable, +} + +impl std::fmt::Display for BundleImportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BundleImportError::Container(m) => write!(f, "reading the container failed: {m}"), + BundleImportError::Manifest(e) => write!(f, "reading {META_FILE_NAME} failed: {e}"), + BundleImportError::Version { + field, + found, + implemented, + } => write!( + f, + "the container declares {field} '{found}', which this build does not implement: it reads '{implemented}'" + ), + BundleImportError::Table { file, detail } => { + write!(f, "reading '{file}' failed: {detail}") + } + BundleImportError::ParquetUnavailable => write!( + f, + "this container uses Parquet storage, which needs the 'ocel-bundle-parquet' feature" + ), + } + } +} + +impl std::error::Error for BundleImportError {} + +impl From for BundleImportError { + fn from(e: std::io::Error) -> Self { + BundleImportError::Container(e.to_string()) + } +} + +fn container_err(e: impl std::fmt::Display) -> BundleImportError { + BundleImportError::Container(e.to_string()) +} + +/// What one entry of an archive may expand to, and what a whole archive may. +/// +/// A compressed entry can expand by orders of magnitude, so neither its declared size nor the +/// archive's own bounds what expanding it costs. Both limits are far above any real container. +const MAX_ENTRY_BYTES: u64 = 4 << 30; +const MAX_TOTAL_BYTES: u64 = 16 << 30; + +/// Copy one archive entry, refusing to write past the limits above. `total` carries the running +/// count across the entries of one archive. +fn copy_entry( + entry: &mut R, + sink: &mut W, + name: &str, + total: &mut u64, +) -> Result<(), BundleImportError> { + let allowed = MAX_ENTRY_BYTES.min(MAX_TOTAL_BYTES - *total); + // One byte past the limit, so a copy that fills it exactly is told apart from an overrun. + let written = io::copy(&mut io::Read::take(entry, allowed + 1), sink)?; + if written > allowed { + return Err(BundleImportError::Container(format!( + "'{name}' expands past what a container may hold ({MAX_ENTRY_BYTES} bytes per entry, {MAX_TOTAL_BYTES} in total)" + ))); + } + *total += written; + Ok(()) +} + +/// An opened container: its manifest, and where each declared file's bytes are. +#[derive(Debug)] +pub struct Container { + meta: BundleMeta, + files: Files, +} + +#[derive(Debug)] +enum Files { + /// Files on disk under this root. `_temp` is present only for an expanded archive, and + /// removes it when the container drops. + Dir { + root: PathBuf, + _temp: Option, + }, + /// Entry contents, for an archive with no path to expand beside. + Memory(HashMap>), +} + +impl Container { + /// Open a directory or a `.ocel.zip` archive. + /// + /// # Errors + /// See [`BundleImportError`]. + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + // The manifest is the only file in an uncompressed container a person can point at, so + // selecting it means the directory it sits in. A file dialog cannot pick a directory on + // every platform, which makes this the practical way to open one. + let path = if path.file_name().is_some_and(|n| n == META_FILE_NAME) { + // A bare `ocel-meta.json` has an empty parent rather than none. + match path.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + } + } else { + path + }; + if path.is_dir() { + let manifest = path.join(META_FILE_NAME); + let meta = read_manifest( + &std::fs::read(&manifest) + .map_err(|e| container_err(format!("{}: {e}", manifest.display())))?, + )?; + return Ok(Self { + meta, + files: Files::Dir { + root: path.to_path_buf(), + _temp: None, + }, + }); + } + + let file = std::fs::File::open(path) + .map_err(|e| container_err(format!("{}: {e}", path.display())))?; + let mut archive = zip::ZipArchive::new(io::BufReader::new(file)).map_err(container_err)?; + let temp = tempfile::tempdir().map_err(container_err)?; + let root = temp.path().to_path_buf(); + + let mut total = 0; + for i in 0..archive.len() { + let mut entry = archive.by_index(i).map_err(container_err)?; + // `enclosed_name` rejects an entry that would escape the directory. An archive is + // untrusted input, and `../` in an entry name is the standard way to abuse one. + let Some(rel) = entry.enclosed_name() else { + continue; + }; + if entry.is_dir() { + continue; + } + let out = root.join(&rel); + if let Some(parent) = out.parent() { + std::fs::create_dir_all(parent)?; + } + // Copied a block at a time, so the peak is a buffer rather than a whole entry. + let mut sink = io::BufWriter::new(std::fs::File::create(&out)?); + copy_entry(&mut entry, &mut sink, &rel.to_string_lossy(), &mut total)?; + } + + let meta = + read_manifest(&std::fs::read(root.join(META_FILE_NAME)).map_err(|e| { + container_err(format!("the archive has no {META_FILE_NAME}: {e}")) + })?)?; + Ok(Self { + meta, + files: Files::Dir { + root, + _temp: Some(temp), + }, + }) + } + + /// Open an archive already in memory, for contents with no path such as a browser upload. + /// Holds every table in memory. Prefer [`Container::open`] where there is a path. + /// + /// # Errors + /// See [`BundleImportError`]. + pub fn open_bytes(bytes: &[u8]) -> Result { + let mut archive = zip::ZipArchive::new(io::Cursor::new(bytes)).map_err(container_err)?; + let mut files: HashMap> = HashMap::new(); + let mut total = 0; + for i in 0..archive.len() { + let mut entry = archive.by_index(i).map_err(container_err)?; + let Some(rel) = entry.enclosed_name() else { + continue; + }; + if entry.is_dir() { + continue; + } + let name = rel.to_string_lossy().replace('\\', "/"); + // No capacity hint: `size()` is a header field the archive supplies, so reserving it + // up front allocates whatever an attacker wrote there before a byte is read. + let mut buf = Vec::new(); + copy_entry(&mut entry, &mut buf, &name, &mut total)?; + files.insert(name, buf); + } + let meta = read_manifest( + files + .get(META_FILE_NAME) + .ok_or_else(|| container_err(format!("the archive has no {META_FILE_NAME}")))?, + )?; + Ok(Self { + meta, + files: Files::Memory(files), + }) + } + + /// What the container declares about itself. + #[must_use] + pub fn meta(&self) -> &BundleMeta { + &self.meta + } + + /// Read the whole container into an [`OCEL`]. + /// + /// # Errors + /// See [`BundleImportError`]. + pub fn read(&self) -> Result { + let mut ocel = OCEL { + event_types: Vec::new(), + object_types: Vec::new(), + events: Vec::new(), + objects: Vec::new(), + }; + + for (ty, decl) in &self.meta.event_types { + ocel.event_types.push(OCELType { + name: ty.clone(), + attributes: type_attributes(&decl.attributes), + }); + for row in self.table(&decl.file, &[columns::ID, columns::TIME])? { + let id = row.text(columns::ID, &decl.file)?; + let time = row.time(columns::TIME, &decl.file)?; + ocel.events.push(OCELEvent { + id, + event_type: ty.clone(), + time, + attributes: decl + .attributes + .iter() + .filter_map(|a| { + row.value(&a.name, a.value_type.into()).map(|value| { + OCELEventAttribute { + name: a.name.clone(), + value, + } + }) + }) + .collect(), + relationships: Vec::new(), + }); + } + } + + // Object rows first, so a change row always has an object to append to. + let mut object_at: HashMap = HashMap::new(); + for (ty, decl) in &self.meta.object_types { + ocel.object_types.push(OCELType { + name: ty.clone(), + attributes: type_attributes(&decl.attributes), + }); + for row in self.table(&decl.file, &[columns::ID])? { + let id = row.text(columns::ID, &decl.file)?; + object_at.insert(id.clone(), ocel.objects.len()); + ocel.objects.push(OCELObject { + id, + object_type: ty.clone(), + // An object table's values are the ones in force from the epoch. + attributes: decl + .attributes + .iter() + .filter_map(|a| { + row.value(&a.name, a.value_type.into()).map(|value| { + OCELObjectAttribute { + name: a.name.clone(), + value, + time: epoch(), + } + }) + }) + .collect(), + relationships: Vec::new(), + }); + } + } + + for decl in self.meta.object_types.values() { + let Some(file) = &decl.changes_file else { + continue; + }; + let mut unknown_object = Dangling::default(); + let mut unknown_field = Dangling::default(); + for row in self.table(file, &[columns::ID, columns::TIME, columns::CHANGED_FIELD])? { + let id = row.text(columns::ID, file)?; + let time = row.time(columns::TIME, file)?; + let name = row.text(columns::CHANGED_FIELD, file)?; + let Some(&at) = object_at.get(&id) else { + unknown_object.note(&id); + continue; + }; + // Only the column `ocel_changed_field` names. The rest of a change row is empty + // by construction, and reading them would record changes the container never + // declared. + let Some(a) = decl.attributes.iter().find(|a| a.name == name) else { + unknown_field.note(&name); + continue; + }; + if let Some(value) = row.value(&name, a.value_type.into()) { + ocel.objects[at] + .attributes + .push(OCELObjectAttribute { name, value, time }); + } + } + unknown_object.into_error(file, "no object table declares the object")?; + unknown_field.into_error(file, "the manifest declares no attribute")?; + } + + // Only the endpoint a relation is stored on is checked. The other is kept as the id it + // is, which is what an OCEL relationship holds anyway. + let e2o = &self.meta.relations.e2o; + let event_at: HashMap = ocel + .events + .iter() + .enumerate() + .map(|(i, ev)| (ev.id.clone(), i)) + .collect(); + let mut unknown_event = Dangling::default(); + for row in self.table( + e2o, + &[columns::EVENT_ID, columns::OBJECT_ID, columns::QUALIFIER], + )? { + let event_id = row.text(columns::EVENT_ID, e2o)?; + let object_id = row.text(columns::OBJECT_ID, e2o)?; + let Some(&at) = event_at.get(&event_id) else { + unknown_event.note(&event_id); + continue; + }; + ocel.events[at].relationships.push(OCELRelationship { + object_id, + qualifier: row.text_or_empty(columns::QUALIFIER), + }); + } + unknown_event.into_error(e2o, "no event table declares the event")?; + + let o2o = &self.meta.relations.o2o; + let mut unknown_source = Dangling::default(); + for row in self.table( + o2o, + &[columns::SOURCE_ID, columns::TARGET_ID, columns::QUALIFIER], + )? { + let source_id = row.text(columns::SOURCE_ID, o2o)?; + let object_id = row.text(columns::TARGET_ID, o2o)?; + let Some(&at) = object_at.get(&source_id) else { + unknown_source.note(&source_id); + continue; + }; + ocel.objects[at].relationships.push(OCELRelationship { + object_id, + qualifier: row.text_or_empty(columns::QUALIFIER), + }); + } + unknown_source.into_error(o2o, "no object table declares the object")?; + + Ok(ocel) + } + + /// The rows of one declared table, once it is known to carry every column in `fixed`. + fn table(&self, file: &str, fixed: &[&str]) -> Result, BundleImportError> { + let missing = || BundleImportError::Table { + file: file.to_string(), + detail: "no such file in the container".to_string(), + }; + if !is_contained(file) { + // A manifest is untrusted input just as an archive's entry names are, and joining a + // declared path onto the container root is exactly as exploitable. + return Err(BundleImportError::Table { + file: file.to_string(), + detail: "a declared path must stay inside the container".to_string(), + }); + } + match (&self.files, self.meta.storage_format) { + (Files::Dir { root, .. }, StorageFormat::Csv) => { + let path = root.join(file); + if !path.is_file() { + return Err(missing()); + } + read_csv(std::fs::File::open(path)?, file, fixed) + } + (Files::Memory(files), StorageFormat::Csv) => read_csv( + io::Cursor::new(files.get(file).ok_or_else(missing)?), + file, + fixed, + ), + #[cfg(feature = "ocel-bundle-parquet")] + (Files::Dir { root, .. }, StorageFormat::Parquet) => { + let path = root.join(file); + if !path.is_file() { + return Err(missing()); + } + read_parquet(std::fs::read(path)?, file, fixed) + } + #[cfg(feature = "ocel-bundle-parquet")] + (Files::Memory(files), StorageFormat::Parquet) => { + read_parquet(files.get(file).ok_or_else(missing)?.clone(), file, fixed) + } + #[cfg(not(feature = "ocel-bundle-parquet"))] + (_, StorageFormat::Parquet) => Err(BundleImportError::ParquetUnavailable), + } + } +} + +/// Read a container at `path`. +/// +/// # Errors +/// See [`BundleImportError`]. +pub fn import_ocel_bundle(path: impl AsRef) -> Result { + Container::open(path)?.read() +} + +/// [`import_ocel_bundle`] for an archive already in memory. +/// +/// # Errors +/// See [`BundleImportError`]. +pub fn import_ocel_bundle_from_bytes(bytes: &[u8]) -> Result { + Container::open_bytes(bytes)?.read() +} + +fn read_manifest(bytes: &[u8]) -> Result { + let meta: BundleMeta = serde_json::from_slice(bytes).map_err(BundleImportError::Manifest)?; + check_version("ocelVersion", &meta.ocel_version, OCEL_VERSION)?; + check_version( + "bundleFormatVersion", + &meta.bundle_format_version, + BUNDLE_FORMAT_VERSION, + )?; + Ok(meta) +} + +/// Whether `found` is a version this build implements. Only the major must match: a later minor +/// revision may only add what a reader can ignore, which is also why the manifest is not +/// `deny_unknown_fields`. +fn check_version(field: &str, found: &str, implemented: &str) -> Result<(), BundleImportError> { + let major = |v: &str| v.split('.').next().unwrap_or_default().to_string(); + if major(found) == major(implemented) { + return Ok(()); + } + Err(BundleImportError::Version { + field: field.to_string(), + found: found.to_string(), + implemented: implemented.to_string(), + }) +} + +/// Whether a manifest's declared path stays inside the container. Checked per component rather +/// than by canonicalising, so it holds for an in-memory container with no filesystem to resolve +/// against. +fn is_contained(file: &str) -> bool { + let path = Path::new(file); + !path.is_absolute() + && path.components().all(|c| { + matches!( + c, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) +} + +/// Ids a table names that nothing declares, kept as the first offender and a count so one broken +/// container yields one error rather than a flood. +#[derive(Debug, Default)] +struct Dangling { + first: Option, + count: usize, +} + +impl Dangling { + fn note(&mut self, name: &str) { + self.count += 1; + if self.first.is_none() { + self.first = Some(name.to_string()); + } + } + + fn into_error(self, file: &str, what: &str) -> Result<(), BundleImportError> { + let Some(first) = self.first else { + return Ok(()); + }; + let more = match self.count { + 1 => String::new(), + n => format!(", and {} further rows like it", n - 1), + }; + Err(BundleImportError::Table { + file: file.to_string(), + detail: format!("{what} '{first}'{more}"), + }) + } +} + +/// One row, as a map from column name to whatever the storage held. CSV yields only +/// [`Value::Text`], Parquet the file's own types. +#[derive(Debug, Default)] +struct Row { + cells: HashMap, +} + +impl Row { + /// A fixed column's text. Absent or empty is an error: the format makes these `required`. + fn text(&self, column: &str, file: &str) -> Result { + let text = match self.cells.get(column) { + Some(Value::Text(s)) => s.clone(), + Some(other) => other.to_string(), + None => String::new(), + }; + if text.is_empty() { + return Err(BundleImportError::Table { + file: file.to_string(), + detail: format!("a row has no '{column}'"), + }); + } + Ok(text) + } + + fn time(&self, column: &str, file: &str) -> Result, BundleImportError> { + let err = |detail: String| BundleImportError::Table { + file: file.to_string(), + detail, + }; + match self.cells.get(column) { + Some(Value::Time(t)) => Ok(*t), + Some(Value::Text(s)) if !s.is_empty() => parse_timestamp(s, None, false) + .map_err(|_| err(format!("'{s}' in '{column}' is not a timestamp"))), + _ => Err(err(format!("a row has no '{column}'"))), + } + } + + /// A fixed column's text where the format allows it to be empty. The column's presence is + /// checked when the table is read, so an absent cell here is an empty one. + fn text_or_empty(&self, column: &str) -> String { + self.cells + .get(column) + .map(Value::to_string) + .unwrap_or_default() + } + + /// An attribute value, or `None` when the cell is absent. + /// + /// Absence is decided when the table is read, not here: a CSV reader drops empty cells + /// because the format says an empty cell is a missing value, while Parquet has a real null + /// and so keeps an empty string as the value it is. + fn value(&self, column: &str, declared: OCELAttributeType) -> Option { + Some(coerce(self.cells.get(column)?, declared)) + } +} + +/// Read a cell as its declared type. A value that does not parse keeps its text, rather than +/// becoming null: losing it silently would be worse than carrying it in the wrong type, and the +/// manifest is the only thing claiming the type in CSV storage. +fn coerce(raw: &Value, declared: OCELAttributeType) -> OCELAttributeValue { + match raw { + Value::Integer(i) => match declared { + OCELAttributeType::Float => OCELAttributeValue::Float(*i as f64), + OCELAttributeType::String => OCELAttributeValue::String(i.to_string()), + _ => OCELAttributeValue::Integer(*i), + }, + Value::Float(f) => match declared { + OCELAttributeType::String => OCELAttributeValue::String(f.to_string()), + _ => OCELAttributeValue::Float(*f), + }, + Value::Boolean(b) => OCELAttributeValue::Boolean(*b), + Value::Time(t) => OCELAttributeValue::Time(*t), + Value::Text(s) => match declared { + OCELAttributeType::Integer => s.parse::().map_or_else( + |_| OCELAttributeValue::String(s.clone()), + OCELAttributeValue::Integer, + ), + OCELAttributeType::Float => s.parse::().map_or_else( + |_| OCELAttributeValue::String(s.clone()), + OCELAttributeValue::Float, + ), + OCELAttributeType::Boolean => match s.as_str() { + "true" => OCELAttributeValue::Boolean(true), + "false" => OCELAttributeValue::Boolean(false), + _ => OCELAttributeValue::String(s.clone()), + }, + OCELAttributeType::Time => parse_timestamp(s, None, false).map_or_else( + |_| OCELAttributeValue::String(s.clone()), + OCELAttributeValue::Time, + ), + _ => OCELAttributeValue::String(s.clone()), + }, + } +} + +/// Reject a table missing a column the format fixes, so a reader never has to decide between a +/// column that is absent and a cell that is empty. +fn require_columns( + present: impl Fn(&str) -> bool, + fixed: &[&str], + file: &str, +) -> Result<(), BundleImportError> { + for column in fixed { + if !present(column) { + return Err(BundleImportError::Table { + file: file.to_string(), + detail: format!("the table has no '{column}' column"), + }); + } + } + Ok(()) +} + +fn read_csv( + reader: R, + file: &str, + fixed: &[&str], +) -> Result, BundleImportError> { + let table_err = |e: csv::Error| BundleImportError::Table { + file: file.to_string(), + detail: e.to_string(), + }; + let mut rdr = csv::Reader::from_reader(reader); + let header: Vec = rdr + .headers() + .map_err(table_err)? + .iter() + .map(str::to_string) + .collect(); + require_columns(|c| header.iter().any(|h| h == c), fixed, file)?; + let mut rows = Vec::new(); + for record in rdr.records() { + let record = record.map_err(table_err)?; + let mut cells = HashMap::with_capacity(header.len()); + for (name, cell) in header.iter().zip(record.iter()) { + // "missing values are represented by empty cells". CSV has no null, so this is the + // only reading available, and why CSV storage cannot carry an attribute whose value + // is the empty string. + if cell.is_empty() { + continue; + } + cells.insert(name.clone(), Value::Text(cell.to_string())); + } + rows.push(Row { cells }); + } + Ok(rows) +} + +#[cfg(feature = "ocel-bundle-parquet")] +fn read_parquet(bytes: Vec, file: &str, fixed: &[&str]) -> Result, BundleImportError> { + use parquet::basic::{LogicalType, TimeUnit}; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::Field; + + let table_err = |e: parquet::errors::ParquetError| BundleImportError::Table { + file: file.to_string(), + detail: e.to_string(), + }; + let reader = SerializedFileReader::new(bytes::Bytes::from(bytes)).map_err(table_err)?; + // Timestamps are physically INT64. The logical type says which unit, so it has to be read + // from the schema rather than guessed from the value. + let units: HashMap> = reader + .metadata() + .file_metadata() + .schema() + .get_fields() + .iter() + .map(|f| { + let unit = match f.get_basic_info().logical_type_ref() { + Some(LogicalType::Timestamp { unit, .. }) => Some(*unit), + _ => None, + }; + (f.name().to_string(), unit) + }) + .collect(); + require_columns(|c| units.contains_key(c), fixed, file)?; + + let mut rows = Vec::new(); + for row in reader.get_row_iter(None).map_err(table_err)? { + let row = row.map_err(table_err)?; + let mut cells = HashMap::new(); + for (name, field) in row.get_column_iter() { + let value = match *field { + Field::Null => continue, + Field::Bool(b) => Value::Boolean(b), + Field::Byte(v) => Value::Integer(i64::from(v)), + Field::Short(v) => Value::Integer(i64::from(v)), + Field::Int(v) => Value::Integer(i64::from(v)), + // Physically an INT64; only the schema's logical type says whether it is an + // instant, and in which unit. + Field::Long(v) => match units.get(name).copied().flatten() { + Some(unit) => micros_of(v, unit).map_or(Value::Integer(v), Value::Time), + None => Value::Integer(v), + }, + Field::UByte(v) => Value::Integer(i64::from(v)), + Field::UShort(v) => Value::Integer(i64::from(v)), + Field::UInt(v) => Value::Integer(i64::from(v)), + Field::ULong(v) => Value::Integer(i64::try_from(v).unwrap_or(i64::MAX)), + Field::Float(v) => Value::Float(f64::from(v)), + Field::Double(v) => Value::Float(v), + Field::TimestampMillis(v) => { + micros_of(v, TimeUnit::MILLIS).map_or(Value::Integer(v), Value::Time) + } + Field::TimestampMicros(v) => { + micros_of(v, TimeUnit::MICROS).map_or(Value::Integer(v), Value::Time) + } + Field::Str(ref s) => Value::Text(s.clone()), + ref other => Value::Text(other.to_string()), + }; + cells.insert(name.clone(), value); + } + rows.push(Row { cells }); + } + Ok(rows) +} + +#[cfg(feature = "ocel-bundle-parquet")] +fn micros_of(value: i64, unit: parquet::basic::TimeUnit) -> Option> { + use parquet::basic::TimeUnit; + let utc = match unit { + TimeUnit::MILLIS => DateTime::from_timestamp_millis(value)?, + TimeUnit::MICROS => DateTime::from_timestamp_micros(value)?, + TimeUnit::NANOS => DateTime::from_timestamp_nanos(value), + }; + Some(utc.into()) +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/meta.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/meta.rs new file mode 100644 index 00000000..9659023e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/meta.rs @@ -0,0 +1,345 @@ +//! `ocel-meta.json`: what a bundled container declares about itself. + +use std::collections::BTreeMap; + +use chrono::{DateTime, FixedOffset}; +use serde::{Deserialize, Serialize}; + +use crate::core::event_data::object_centric::{OCELAttributeType, OCELTypeAttribute}; + +/// The manifest's name inside a container. Every other filename is declared by this file, +/// never inferred from a path. +pub const META_FILE_NAME: &str = "ocel-meta.json"; + +/// The bundled-format revision this build reads and writes. +pub const BUNDLE_FORMAT_VERSION: &str = "1.0"; + +/// The OCEL revision the bundled format carries. +pub const OCEL_VERSION: &str = "2.0"; + +/// Fixed column names, identical in both storage formats. +pub mod columns { + /// An event's or object's own id. + pub const ID: &str = "ocel_id"; + /// When an event happened, or when an object attribute took its value. + pub const TIME: &str = "ocel_time"; + /// Which attribute an object-change row changes. + pub const CHANGED_FIELD: &str = "ocel_changed_field"; + /// `e2o`: the event. + pub const EVENT_ID: &str = "ocel_event_id"; + /// `e2o`: the object. + pub const OBJECT_ID: &str = "ocel_object_id"; + /// `o2o`: the referring object. + pub const SOURCE_ID: &str = "ocel_source_id"; + /// `o2o`: the referred-to object. + pub const TARGET_ID: &str = "ocel_target_id"; + /// A relation's qualifier. + pub const QUALIFIER: &str = "ocel_qualifier"; +} + +/// Which physical storage a container uses. One per container: a `csv` container holds no +/// Parquet files and vice versa. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum StorageFormat { + /// Every cell is text, read according to the declared attribute type. + #[default] + Csv, + /// Attribute types are carried by the file's own schema. + Parquet, +} + +impl StorageFormat { + /// The filename extension for this storage. + #[must_use] + pub fn extension(self) -> &'static str { + match self { + StorageFormat::Csv => "csv", + StorageFormat::Parquet => "parquet", + } + } +} + +/// One of OCEL 2.0's primitive attribute types, as `ocel-meta.json` spells it. +/// +/// A separate enum rather than [`OCELAttributeType`] directly: that type carries a `Null` +/// variant which is not an OCEL type, and its `from_type_str` maps anything unrecognised +/// onto it, so a typo in a manifest would be read as a valid declaration. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AttributeType { + /// Text. + String, + /// A timezone-aware instant. + Time, + /// A 64-bit integer. + Integer, + /// A double. + Float, + /// `true` or `false`. + Boolean, +} + +impl From for OCELAttributeType { + fn from(t: AttributeType) -> Self { + match t { + AttributeType::String => OCELAttributeType::String, + AttributeType::Time => OCELAttributeType::Time, + AttributeType::Integer => OCELAttributeType::Integer, + AttributeType::Float => OCELAttributeType::Float, + AttributeType::Boolean => OCELAttributeType::Boolean, + } + } +} + +/// One cell's value, as one of the format's primitive types. +#[derive(Debug, Clone)] +pub enum Value { + /// Text. + Text(String), + /// A 64-bit integer. + Integer(i64), + /// A double. + Float(f64), + /// `true` or `false`. + Boolean(bool), + /// A timezone-aware instant. + Time(DateTime), +} + +impl std::fmt::Display for Value { + /// The cell's text, as CSV storage holds it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Value::Text(s) => f.write_str(s), + Value::Integer(i) => write!(f, "{i}"), + Value::Float(v) => write!(f, "{v}"), + Value::Boolean(b) => write!(f, "{b}"), + Value::Time(t) => write!(f, "{}", t.to_rfc3339()), + } + } +} + +/// The instant an object table's values are in force from. +#[must_use] +pub fn epoch() -> DateTime { + DateTime::from_timestamp_nanos(0).into() +} + +/// One declared attribute: the column that holds it, and how to read it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AttributeDecl { + /// Attribute name, which is also the column name. + pub name: String, + /// Primitive type. + #[serde(rename = "type")] + pub value_type: AttributeType, +} + +/// An event type's table and attributes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EventTypeDecl { + /// Path inside the container. + pub file: String, + /// Attribute columns beyond `ocel_id` and `ocel_time`. + #[serde(default)] + pub attributes: Vec, +} + +/// A declared attribute list as OCEL type attributes. +#[must_use] +pub fn type_attributes(attrs: &[AttributeDecl]) -> Vec { + attrs + .iter() + .map(|a| OCELTypeAttribute::new(&a.name, &OCELAttributeType::from(a.value_type))) + .collect() +} + +/// An object type's tables and attributes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ObjectTypeDecl { + /// Path inside the container. + pub file: String, + /// Path to this type's object-change table. + /// + /// The format requires one per object type even when it has no changes, but this stays + /// optional so a container that omits an empty one still imports. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub changes_file: Option, + /// Attribute columns beyond `ocel_id`. + #[serde(default)] + pub attributes: Vec, +} + +/// The two relation tables. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RelationFiles { + /// Event-to-object relations. + pub e2o: String, + /// Object-to-object relations. + pub o2o: String, +} + +/// A container's `ocel-meta.json`. +/// +/// Deliberately not `deny_unknown_fields`: the bundled format is young, and a container written +/// against a later revision should still import for everything this build understands rather +/// than be rejected wholesale. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BundleMeta { + /// OCEL version, `"2.0"`. + pub ocel_version: String, + /// Bundled-format revision. + pub bundle_format_version: String, + /// Which physical storage the tables use. + pub storage_format: StorageFormat, + /// Event types, by exact type name. `BTreeMap` so a generated blueprint and an exported + /// manifest are byte-identical across runs. + #[serde(default)] + pub event_types: BTreeMap, + /// Object types, by exact type name. + #[serde(default)] + pub object_types: BTreeMap, + /// The relation tables. + pub relations: RelationFiles, +} + +impl BundleMeta { + /// Every declared file, paired with the logical table name that reads it. + /// + /// This is the single place where the two halves of an import agree: the container opens + /// exactly these paths under exactly these names, and the generated blueprint's `Source` + /// nodes name the same ones. + pub fn tables(&self) -> Vec<(String, &str)> { + let mut out = Vec::new(); + for (ty, d) in &self.event_types { + out.push((event_table(ty), d.file.as_str())); + } + for (ty, d) in &self.object_types { + out.push((object_table(ty), d.file.as_str())); + if let Some(f) = &d.changes_file { + out.push((object_changes_table(ty), f.as_str())); + } + } + out.push((E2O_TABLE.to_string(), self.relations.e2o.as_str())); + out.push((O2O_TABLE.to_string(), self.relations.o2o.as_str())); + out + } +} + +/// The `e2o` table's logical name. +pub const E2O_TABLE: &str = "e2o"; +/// The `o2o` table's logical name. +pub const O2O_TABLE: &str = "o2o"; + +// A colon after `event`/`object` and an underscore in `object_changes` means the three prefixes +// differ at a fixed position, so no type name can make two of these collide, nor equal the bare +// `e2o`/`o2o`. +/// Logical table name for an event type's table. +#[must_use] +pub fn event_table(event_type: &str) -> String { + format!("event:{event_type}") +} +/// Logical table name for an object type's table. +#[must_use] +pub fn object_table(object_type: &str) -> String { + format!("object:{object_type}") +} +/// Logical table name for an object type's change table. +#[must_use] +pub fn object_changes_table(object_type: &str) -> String { + format!("object_changes:{object_type}") +} + +/// Percent-encode a type name for use in a filename, per the format's naming rule: +/// `A-Z`, `a-z`, `0-9`, `.`, `_` and `-` stay, every other byte becomes `%HH` with uppercase hex. +/// +/// Import never needs this, since `ocel-meta.json` is authoritative and its declared paths are +/// used verbatim, but an exporter does, and keeping both here keeps the two consistent. +#[must_use] +pub fn encode_type_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for b in name.as_bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-') { + out.push(*b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn type_names_are_percent_encoded_as_the_format_specifies() { + assert_eq!(encode_type_name("place order"), "place%20order"); + assert_eq!(encode_type_name("pay/order"), "pay%2Forder"); + assert_eq!(encode_type_name("orders"), "orders"); + assert_eq!(encode_type_name("sales person"), "sales%20person"); + // Multi-byte UTF-8 is encoded byte by byte. + assert_eq!(encode_type_name("café"), "caf%C3%A9"); + } + + #[test] + fn no_type_name_can_make_two_logical_table_names_collide() { + let names = [ + event_table("changes:x"), + object_table("changes:x"), + object_changes_table("x"), + object_table("e2o"), + E2O_TABLE.to_string(), + ]; + let unique: std::collections::BTreeSet<_> = names.iter().collect(); + assert_eq!(unique.len(), names.len(), "{names:?}"); + } + + #[test] + fn the_specs_example_manifest_parses() { + let meta: BundleMeta = serde_json::from_str( + r#"{ + "ocelVersion": "2.0", + "bundleFormatVersion": "1.0", + "storageFormat": "csv", + "eventTypes": { + "place order": { "file": "events/event_place%20order.csv", + "attributes": [{ "name": "resource", "type": "string" }] }, + "pay order": { "file": "events/event_pay%20order.csv", "attributes": [] } + }, + "objectTypes": { + "orders": { "file": "objects/object_orders.csv", + "changesFile": "object_changes/object_changes_orders.csv", + "attributes": [{ "name": "price", "type": "float" }] }, + "sales person": { "file": "objects/object_sales%20person.csv", + "changesFile": "object_changes/object_changes_sales%20person.csv", + "attributes": [] } + }, + "relations": { "e2o": "relations/e2o.csv", "o2o": "relations/o2o.csv" } + }"#, + ) + .expect("parse"); + + assert_eq!(meta.storage_format, StorageFormat::Csv); + assert_eq!( + meta.event_types["place order"].attributes[0].name, + "resource" + ); + assert_eq!( + meta.object_types["orders"].attributes[0].value_type, + AttributeType::Float + ); + // 2 event tables + 2 object tables + 2 change tables + e2o + o2o. + assert_eq!(meta.tables().len(), 8); + } + + #[test] + fn an_unknown_attribute_type_is_rejected_rather_than_read_as_null() { + let err = serde_json::from_str::(r#"{"name":"x","type":"strng"}"#) + .expect_err("typo rejected"); + assert!(err.to_string().contains("strng"), "{err}"); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/mod.rs new file mode 100644 index 00000000..64169604 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/mod.rs @@ -0,0 +1,38 @@ +//! The OCEL 2.0 bundled CSV/Parquet format: a `.ocel.zip` archive, or a directory with the same +//! layout, holding one table per event type, object type, object-change set, and relation kind. +//! +//! Split by what each part needs: +//! +//! - `meta` holds the `ocel-meta.json` manifest and the naming rules. Pure, always available. +//! - `export`/`import` write and read a container (`ocel-bundle`, plus `ocel-bundle-parquet` for +//! Parquet storage). +//! - `blueprint` builds the extraction blueprint that turns a container's tables back into an OCEL +//! (`extraction-blueprint`). Running it needs a row provider over the container's files, which +//! `extraction-dbcon` supplies. + +#[cfg(feature = "extraction-blueprint")] +pub mod blueprint; +#[cfg(feature = "ocel-bundle")] +pub mod export; +#[cfg(feature = "ocel-bundle")] +pub mod import; +pub mod meta; + +#[cfg(feature = "extraction-blueprint")] +pub use blueprint::{blueprint_for, SOURCE_ID}; +#[cfg(feature = "ocel-bundle")] +pub use export::{ + export_ocel_bundle, write_ocel_bundle_archive, BundleExportError, BundleExportOptions, + ContainerLayout, +}; +#[cfg(feature = "ocel-bundle")] +pub use import::{import_ocel_bundle, import_ocel_bundle_from_bytes, BundleImportError, Container}; +pub use meta::{ + columns, encode_type_name, epoch, event_table, object_changes_table, object_table, + type_attributes, AttributeDecl, AttributeType, BundleMeta, EventTypeDecl, ObjectTypeDecl, + RelationFiles, StorageFormat, Value, BUNDLE_FORMAT_VERSION, E2O_TABLE, META_FILE_NAME, + O2O_TABLE, OCEL_VERSION, +}; + +#[cfg(all(test, feature = "extraction-blueprint"))] +mod tests; diff --git a/process_mining/src/core/event_data/object_centric/ocel_bundle/tests.rs b/process_mining/src/core/event_data/object_centric/ocel_bundle/tests.rs new file mode 100644 index 00000000..ddbf74ad --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_bundle/tests.rs @@ -0,0 +1,129 @@ +use super::*; +use crate::core::event_data::object_centric::extraction::{validate, ExtractionCatalog}; + +/// The manifest for the spec's running example, cut down to what these tests need: an object +/// type with two attributes of which only one ever changes, which is the case a naive mapping +/// gets wrong. +fn meta() -> BundleMeta { + serde_json::from_str( + r#"{ + "ocelVersion": "2.0", + "bundleFormatVersion": "1.0", + "storageFormat": "csv", + "eventTypes": { + "Create Purchase Order": { "file": "events/event_Create%20Purchase%20Order.csv", + "attributes": [{ "name": "po_creator", "type": "string" }] }, + "Change PO Quantity": { "file": "events/event_Change%20PO%20Quantity.csv", + "attributes": [{ "name": "po_editor", "type": "string" }] } + }, + "objectTypes": { + "Purchase Order": { "file": "objects/object_Purchase%20Order.csv", + "changesFile": "object_changes/object_changes_Purchase%20Order.csv", + "attributes": [{ "name": "po_product", "type": "string" }, + { "name": "po_quantity", "type": "integer" }] } + }, + "relations": { "e2o": "relations/e2o.csv", "o2o": "relations/o2o.csv" } + }"#, + ) + .expect("parse manifest") +} + +#[test] +fn every_declared_table_gets_a_source_node_reading_it() { + let meta = meta(); + let bp = blueprint_for(&meta); + let sources: Vec<&str> = bp + .nodes + .iter() + .filter_map(|n| match &n.op { + crate::core::event_data::object_centric::extraction::NodeOp::Source { + table, .. + } => Some(table.as_str()), + _ => None, + }) + .collect(); + let declared: Vec = meta.tables().into_iter().map(|(t, _)| t).collect(); + for t in &declared { + assert!(sources.contains(&t.as_str()), "{t} has no Source node"); + } + assert_eq!( + sources.len(), + declared.len(), + "no source node without a table" + ); +} + +/// One filtered node and one mapping per changeable attribute. See `blueprint.rs` for why a +/// single mapping over the whole change table is wrong. +#[test] +fn a_change_table_is_read_once_per_attribute_behind_its_own_filter() { + let bp = blueprint_for(&meta()); + let filters: Vec<&str> = bp + .nodes + .iter() + .filter(|n| { + matches!( + n.op, + crate::core::event_data::object_centric::extraction::NodeOp::Filter { .. } + ) + }) + .map(|n| n.id.as_str()) + .collect(); + assert_eq!( + filters, + [ + "object_changes:Purchase Order#po_product", + "object_changes:Purchase Order#po_quantity" + ] + ); +} + +#[test] +fn the_generated_blueprint_validates_against_the_schema_it_describes() { + let meta = meta(); + let bp = blueprint_for(&meta); + let errors = validate(&bp, &catalog(&meta)); + assert!(errors.is_empty(), "{errors:?}"); +} + +/// A catalog describing exactly the columns the format fixes, so validation sees the same shape +/// a real container would present. +fn catalog(meta: &BundleMeta) -> ExtractionCatalog { + use crate::core::event_data::object_centric::extraction::TableSchema; + use std::collections::BTreeMap; + + let mut tables: BTreeMap = BTreeMap::new(); + let mut add = |name: String, cols: Vec<&str>| { + let schema = TableSchema::new(&name, cols.into_iter().map(|c| (c, "TEXT", true))); + tables.insert(name, schema); + }; + + for (ty, d) in &meta.event_types { + let mut cols = vec![columns::ID, columns::TIME]; + cols.extend(d.attributes.iter().map(|a| a.name.as_str())); + add(event_table(ty), cols); + } + for (ty, d) in &meta.object_types { + let mut cols = vec![columns::ID]; + cols.extend(d.attributes.iter().map(|a| a.name.as_str())); + add(object_table(ty), cols); + if d.changes_file.is_some() { + let mut cols = vec![columns::ID, columns::TIME, columns::CHANGED_FIELD]; + cols.extend(d.attributes.iter().map(|a| a.name.as_str())); + add(object_changes_table(ty), cols); + } + } + add( + E2O_TABLE.to_string(), + vec![columns::EVENT_ID, columns::OBJECT_ID, columns::QUALIFIER], + ); + add( + O2O_TABLE.to_string(), + vec![columns::SOURCE_ID, columns::TARGET_ID, columns::QUALIFIER], + ); + + ExtractionCatalog { + tables: BTreeMap::from([(SOURCE_ID.to_string(), tables)]), + ..Default::default() + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs index d2e9d7f7..798c37df 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs @@ -1,5 +1,6 @@ //! CSV Export for OCEL 2.0 +use super::escaping::escape_reference_part; use crate::core::event_data::object_centric::{ ocel_struct::OCELAttributeValue, readable::{OCELLookup, ReadableOCEL}, @@ -107,7 +108,9 @@ where let mut headers: Vec = vec!["id".into(), "activity".into(), "timestamp".into()]; headers.extend(object_type_names.iter().map(|n| format!("ot:{n}"))); - headers.extend(event_attr_names.iter().map(|n| format!("ea:{n}"))); + // Event attributes are written under their plain name: any column that is not `id`, + // `activity`, `timestamp`, or `ot:` is an event attribute on import. + headers.extend(event_attr_names.iter().map(|n| (*n).to_string())); csv_writer.write_record(&headers)?; // (time, object_id) pairs covered by event rows, so the later object-attribute pass @@ -273,10 +276,10 @@ fn format_timestamp(dt: &DateTime, options: &OCELCSVExportOptions) fn format_object_refs(refs: &[ObjectRef<'_>]) -> String { refs.iter() .map(|r| { - let mut s = r.object_id.to_string(); + let mut s = escape_reference_part(r.object_id).into_owned(); if !r.qualifier.is_empty() { s.push('#'); - s.push_str(r.qualifier); + s.push_str(&escape_reference_part(r.qualifier)); } if let Some(attrs) = &r.attributes { if !attrs.is_empty() { @@ -383,6 +386,32 @@ e4,send order,2026-01-26T09:57:28+0000,o1,i1/i2,yes,"#; assert_eq!(ocel.objects.len(), reimported.objects.len()); } + /// Event attribute columns are written under their plain name; the legacy `ea:` prefix is + /// only understood on import, never produced on export. + #[test] + fn test_event_attributes_are_exported_without_prefix() { + let csv_input = r#"id,activity,timestamp,ot:item,ea:billable,area +e1,place order,2026-01-22T09:57:28+0000,i1,no,outdoor"#; + let ocel = import_ocel_csv(csv_input.as_bytes()).unwrap(); + let exported = export_ocel_csv_to_string(&ocel).unwrap(); + let header = exported.lines().next().unwrap(); + assert_eq!(header, "id,activity,timestamp,ot:item,area,billable"); + + let reimported = import_ocel_csv(exported.as_bytes()).unwrap(); + let attrs = &reimported.events[0].attributes; + assert_eq!( + attrs + .iter() + .find(|a| a.name == "billable") + .map(|a| &a.value), + Some(&OCELAttributeValue::String("no".into())) + ); + assert_eq!( + attrs.iter().find(|a| a.name == "area").map(|a| &a.value), + Some(&OCELAttributeValue::String("outdoor".into())) + ); + } + /// Regression: object attributes with an "initial value" timestamp (not matching any event /// time) must survive a default CSV roundtrip. #[test] diff --git a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_import.rs b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_import.rs index 28396ae9..e3012820 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_import.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_import.rs @@ -14,6 +14,8 @@ use crate::core::event_data::{ timestamp_utils::parse_timestamp, }; +use super::escaping::{find_unescaped, unescape_reference_part}; + /// Error type for CSV OCEL parsing #[derive(Debug, Clone, Serialize, Deserialize)] pub enum OCELCSVImportError { @@ -89,13 +91,15 @@ struct ObjectRef { /// Parse object reference /// Can of the following shapes: `id`, `id#qualifier`, `id{json}`, or `id#qualifier{json}` +/// +/// The `#` and `{` that divide the parts are the first unescaped ones. See [`super::escaping`]. fn parse_object_ref(input: &str) -> Result { let input = input.trim(); if input.is_empty() { return Err("Empty object reference".into()); } - let (before_brace, json_map) = match input.find('{') { + let (before_brace, json_map) = match find_unescaped(input, '{') { Some(pos) => { let json_str = &input[pos..]; let parsed: serde_json::Value = serde_json::from_str(json_str) @@ -105,19 +109,21 @@ fn parse_object_ref(input: &str) -> Result { None => (input, None), }; - let (id, qualifier) = match before_brace.find('#') { + let (id, qualifier) = match find_unescaped(before_brace, '#') { Some(pos) => (before_brace[..pos].trim(), before_brace[pos + 1..].trim()), None => (before_brace.trim(), ""), }; Ok(ObjectRef { - id: id.to_string(), - qualifier: qualifier.to_string(), + id: unescape_reference_part(id).into_owned(), + qualifier: unescape_reference_part(qualifier).into_owned(), attributes: json_map, }) } /// Parse cell with multiple object references separated by `/` (respecting JSON braces) +/// +/// A `/` inside JSON attributes, or written as `\/`, does not separate references. fn parse_object_cell(cell: &str) -> Result, String> { let cell = cell.trim(); if cell.is_empty() { @@ -127,9 +133,20 @@ fn parse_object_cell(cell: &str) -> Result, String> { let mut refs = Vec::new(); let mut current = String::new(); let mut brace_depth = 0; + let mut escaped = false; for c in cell.chars() { + if escaped { + // The backslash is kept: `parse_object_ref` resolves the escape and needs to see it. + escaped = false; + current.push(c); + continue; + } match c { + '\\' => { + escaped = true; + current.push(c); + } '{' => { brace_depth += 1; current.push(c); @@ -157,26 +174,94 @@ fn parse_object_cell(cell: &str) -> Result, String> { /// /// Tries to interpret the string in the following order: /// bool > int > float > time > string +/// +/// `s` is the cell exactly as written, padding included. A number is only recognised when the +/// text is already the canonical spelling of the value it would parse to, so nothing that could +/// not be written back out as it came in is parsed. fn parse_value(s: &str, date_fmt: Option<&str>) -> OCELAttributeValue { - let s = s.trim(); if s.eq_ignore_ascii_case("true") { return OCELAttributeValue::Boolean(true); } if s.eq_ignore_ascii_case("false") { return OCELAttributeValue::Boolean(false); } - if let Ok(i) = s.parse::() { - return OCELAttributeValue::Integer(i); - } - if let Ok(f) = s.parse::() { - return OCELAttributeValue::Float(f); + if is_canonical_integer(s) { + // No `f64` fallback, which would round a long identifier. + if let Ok(i) = s.parse::() { + return OCELAttributeValue::Integer(i); + } + } else if is_canonical_decimal(s) { + if let Ok(f) = s.parse::() { + return OCELAttributeValue::Float(f); + } } - if let Ok(ts) = parse_timestamp(s, date_fmt, false) { - return OCELAttributeValue::Time(ts); + if is_timestamp_like(s, date_fmt) { + if let Ok(ts) = parse_timestamp(s, date_fmt, false) { + return OCELAttributeValue::Time(ts); + } } OCELAttributeValue::String(s.to_string()) } +/// `0`, `-12`, `900`, but not `+7`, `007`, `-0`, `1_000` or `1e3`. +fn is_canonical_integer(s: &str) -> bool { + let digits = s.strip_prefix('-').unwrap_or(s); + match digits.as_bytes() { + [] => false, + // A single `0` is canonical, a leading one is not. The length check excludes `-0`. + [b'0'] => digits.len() == s.len(), + [first, ..] => { + first.is_ascii_digit() && *first != b'0' && digits.bytes().all(|b| b.is_ascii_digit()) + } + } +} + +/// `0.5`, `-0.5`, `-12.75`, but not `.5`, `5.`, `1e3` or `+1.0`. +/// +/// Trailing zeros in the fraction are accepted: `5.00` is the number five. A leading zero in the +/// whole part is not. +fn is_canonical_decimal(s: &str) -> bool { + let Some((whole, fraction)) = s.split_once('.') else { + return false; + }; + if fraction.is_empty() || !fraction.bytes().all(|b| b.is_ascii_digit()) { + return false; + } + let digits = whole.strip_prefix('-').unwrap_or(whole); + match digits.as_bytes() { + [] => false, + // Unlike a bare integer, `-0` is fine here: `-0.5` is a normal way to write the number. + [b'0'] => true, + [first, ..] => *first != b'0' && digits.bytes().all(|b| b.is_ascii_digit()), + } +} + +/// Whether `s` is worth handing to the timestamp parser. +/// +/// The format's timestamp type is ISO 8601 with a timezone, so a bare date or a local +/// wall-clock time is text: reading one as an instant would invent an offset the file never +/// gave. An explicit `date_format` overrides this. +fn is_timestamp_like(s: &str, date_fmt: Option<&str>) -> bool { + if date_fmt.is_some() { + return true; + } + let t = s.trim_end(); + if t.ends_with('Z') || t.ends_with('z') { + return true; + } + // A trailing `+HH:MM`, `-HH:MM`, `+HHMM` or `-HHMM`. `get` rather than a slice, since counting + // back a fixed number of bytes can land inside a character. + let tail = |n: usize| t.len().checked_sub(n).and_then(|i| t.get(i..)); + [6, 5].into_iter().any(|n| { + tail(n).is_some_and(|z| { + let bytes = z.as_bytes(); + matches!(bytes.first(), Some(b'+' | b'-')) + && bytes[1..].iter().all(|b| b.is_ascii_digit() || *b == b':') + && bytes[1..].iter().filter(|b| b.is_ascii_digit()).count() == 4 + }) + }) +} + /// Convert JSON value to OCEL attribute value fn json_to_value(v: &serde_json::Value) -> OCELAttributeValue { match v { @@ -194,19 +279,6 @@ fn json_to_value(v: &serde_json::Value) -> OCELAttributeValue { } } -/// Coalesce two types: same->same, null + x ->x, int + float -> float, other combination -> string -fn coalesce(t1: OCELAttributeType, t2: OCELAttributeType) -> OCELAttributeType { - use OCELAttributeType::*; - if t1 == t2 { - return t1; - } - match (t1, t2) { - (Null, other) | (other, Null) => other, - (Integer, Float) | (Float, Integer) => Float, - _ => String, - } -} - /// Convert value to target type (mainly int -> float or any -> string) fn convert_to_type(value: OCELAttributeValue, target: OCELAttributeType) -> OCELAttributeValue { let current: OCELAttributeType = value.get_type(); @@ -236,7 +308,7 @@ fn register_type( let value_type: OCELAttributeType = value.get_type(); if let Some(attrs) = registry.get_mut(type_name) { if let Some(current) = attrs.get_mut(attr_name) { - *current = coalesce(*current, value_type); + *current = current.coalesce(value_type); } else { attrs.insert(attr_name.to_string(), value_type); } @@ -265,6 +337,23 @@ enum Column { EventAttr(String), } +/// The header after `prefix`, matching the prefix without regard to case but leaving what +/// follows it byte for byte. +fn strip_prefix_ignore_case<'a>(header: &'a str, prefix: &str) -> Option<&'a str> { + let head = header.trim_start(); + // Compared as bytes, because a header may open with a character that a `str` slice at the + // prefix length would cut in half. Once the bytes match, the ASCII prefix's length is a + // character boundary. + let (head_bytes, prefix_bytes) = (head.as_bytes(), prefix.as_bytes()); + if head_bytes.len() >= prefix_bytes.len() + && head_bytes[..prefix_bytes.len()].eq_ignore_ascii_case(prefix_bytes) + { + Some(&head[prefix_bytes.len()..]) + } else { + None + } +} + /// Classify all columns of the CSV /// Returns a list with all columns, as well as the indices of the id column (1st), activity column (2nd), and timestamp column (3rd) fn classify_columns( @@ -274,8 +363,9 @@ fn classify_columns( let (mut id_col, mut act_col, mut ts_col) = (None, None, None); for (i, h) in headers.iter().enumerate() { - let h = h.trim(); - let h_lower = h.to_lowercase(); + // Trimmed only to recognise the fixed columns and the two prefixes. `ot:` carries the + // exact object type name, so the name itself is taken from the header as written. + let h_lower = h.trim().to_lowercase(); if h_lower == "id" { id_col = Some(i); columns.push(Column::Id); @@ -285,13 +375,10 @@ fn classify_columns( } else if h_lower == "timestamp" { ts_col = Some(i); columns.push(Column::Timestamp); - } else if let Some(name) = h_lower.strip_prefix("ot:") { - // Use original casing for the name part - let orig_name = h.get(3..).unwrap_or(name).trim(); - columns.push(Column::ObjectType(orig_name.to_string())); - } else if let Some(name) = h_lower.strip_prefix("ea:") { - let orig_name = h.get(3..).unwrap_or(name).trim(); - columns.push(Column::EventAttr(orig_name.to_string())); + } else if let Some(name) = strip_prefix_ignore_case(h, "ot:") { + columns.push(Column::ObjectType(name.to_string())); + } else if let Some(name) = strip_prefix_ignore_case(h, "ea:") { + columns.push(Column::EventAttr(name.to_string())); } else { columns.push(Column::EventAttr(h.to_string())); } @@ -438,7 +525,7 @@ pub fn import_ocel_csv_with_options( qualifier: obj_ref.qualifier.clone(), }); } - } else if options.verbose { + } else { // If object never appeared before, we can't know its type. if options.strict { return Err(OCELCSVImportError::InvalidObjectReference { @@ -448,7 +535,9 @@ pub fn import_ocel_csv_with_options( ), }); } - eprintln!("Warning: O2O source '{id}' not found at row {row_num}"); + if options.verbose { + eprintln!("Warning: O2O source '{id}' not found at row {row_num}"); + } } continue; } @@ -471,7 +560,9 @@ pub fn import_ocel_csv_with_options( let mut attrs: Vec = Vec::new(); for (col_idx, col) in columns.iter().enumerate() { if let Column::EventAttr(attr_name) = col { - let cell = record.get(col_idx).unwrap_or("").trim(); + // Taken exactly as written: the format preserves an event attribute value apart + // from RFC 4180 unquoting. Only an empty cell is a missing value. + let cell = record.get(col_idx).unwrap_or(""); if !cell.is_empty() { let value = parse_value(cell, date_fmt); register_type(&mut event_type_attrs, &event_type, attr_name, &value); @@ -635,7 +726,30 @@ e1,place order,2026-01-22T09:57:28+0000, o1 , i1 ,yes"#; let ot_names: HashSet<_> = ocel.object_types.iter().map(|t| t.name.as_str()).collect(); assert!(ot_names.contains("Order")); - assert!(ot_names.contains("Item")); + // ` ot:Item ` names the type `Item `: only whitespace before the prefix is skipped. + assert!(ot_names.contains("Item ")); + } + + #[test] + fn a_type_name_keeps_the_whitespace_the_header_gives_it() { + let csv = "id,activity,timestamp,ot:order ,ot:order\n\ + e1,place,2026-01-22T09:57:28+0000,o1,o2"; + let ocel = import_ocel_csv(csv.as_bytes()).unwrap(); + let names: HashSet<_> = ocel.object_types.iter().map(|t| t.name.as_str()).collect(); + assert!(names.contains("order "), "got {names:?}"); + assert!(names.contains("order"), "got {names:?}"); + assert_eq!(names.len(), 2); + } + + #[test] + fn an_event_attribute_value_keeps_its_padding() { + let csv = "id,activity,timestamp,note\n\ + e1,place,2026-01-22T09:57:28+0000,\" spaced \""; + let ocel = import_ocel_csv(csv.as_bytes()).unwrap(); + assert_eq!( + ocel.events[0].attributes[0].value, + OCELAttributeValue::String(" spaced ".to_string()) + ); } #[test] diff --git a/process_mining/src/core/event_data/object_centric/ocel_csv/escaping.rs b/process_mining/src/core/event_data/object_centric/ocel_csv/escaping.rs new file mode 100644 index 00000000..94176100 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_csv/escaping.rs @@ -0,0 +1,120 @@ +//! Escaping for the characters that structure an `ot:` cell. +//! +//! `/` separates references, `#` separates an object id from its qualifier, and `{` opens the +//! JSON attributes. A backslash escapes those three and itself. Anything else after a backslash +//! is a literal backslash, so a file written before this escape existed reads back unchanged. + +/// The characters a backslash may escape. +const ESCAPABLE: [char; 4] = ['/', '#', '{', '\\']; + +/// Writes an object id or qualifier so that [`unescape_reference_part`] gives it back. +pub fn escape_reference_part(part: &str) -> std::borrow::Cow<'_, str> { + if !part.contains(ESCAPABLE) { + return std::borrow::Cow::Borrowed(part); + } + let mut out = String::with_capacity(part.len() + 8); + for c in part.chars() { + if ESCAPABLE.contains(&c) { + out.push('\\'); + } + out.push(c); + } + std::borrow::Cow::Owned(out) +} + +/// Reads back what [`escape_reference_part`] wrote. +pub fn unescape_reference_part(part: &str) -> std::borrow::Cow<'_, str> { + if !part.contains('\\') { + return std::borrow::Cow::Borrowed(part); + } + let mut out = String::with_capacity(part.len()); + let mut chars = part.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.clone().next() { + Some(next) if ESCAPABLE.contains(&next) => { + out.push(next); + chars.next(); + } + // A backslash that escapes nothing is itself. + _ => out.push('\\'), + } + } + std::borrow::Cow::Owned(out) +} + +/// Byte index of the first unescaped `needle` in `haystack`. +pub(crate) fn find_unescaped(haystack: &str, needle: char) -> Option { + let mut escaped = false; + for (i, c) in haystack.char_indices() { + if escaped { + escaped = false; + continue; + } + if c == '\\' { + escaped = true; + } else if c == needle { + return Some(i); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_reserved_character_survives_a_round_trip() { + for part in [ + "plain", + "a/b", + "a#b", + "a{b", + "a\\b", + "a/b#c{d\\e", + "///", + "\\", + "ends with\\", + "Ünïcödé", + ] { + let escaped = escape_reference_part(part); + assert_eq!( + unescape_reference_part(&escaped), + part, + "round trip of {part:?} through {escaped:?}" + ); + } + } + + /// A file written before the escape existed still reads the same. + #[test] + fn a_backslash_that_escapes_nothing_stays_a_backslash() { + assert_eq!(unescape_reference_part("C:\\Users\\me"), "C:\\Users\\me"); + assert_eq!(unescape_reference_part("a\\nb"), "a\\nb"); + assert_eq!(unescape_reference_part("trailing\\"), "trailing\\"); + } + + #[test] + fn a_separator_is_found_only_where_it_is_not_escaped() { + assert_eq!(find_unescaped("a#b", '#'), Some(1)); + assert_eq!(find_unescaped("a\\#b", '#'), None); + assert_eq!(find_unescaped("a\\#b#c", '#'), Some(4)); + assert_eq!(find_unescaped("\\\\#c", '#'), Some(2)); + } + + #[test] + fn nothing_is_allocated_when_nothing_needs_escaping() { + assert!(matches!( + escape_reference_part("plain"), + std::borrow::Cow::Borrowed(_) + )); + assert!(matches!( + unescape_reference_part("plain"), + std::borrow::Cow::Borrowed(_) + )); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_csv/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_csv/mod.rs index e1c3b649..1f571573 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_csv/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_csv/mod.rs @@ -5,7 +5,7 @@ //! The CSV format for OCEL 2.0 has the following structure: //! //! ```text -//! id,activity,timestamp,ot:order,ot:item,ea:billable,ea:area +//! id,activity,timestamp,ot:order,ot:item,billable,area //! e1,place order,2026-01-22T09:57:28+0000,o1,i1#part-of{"price": "5€"}/i2#part-of{"price": "15€"},no, //! e2,pick item,2026-01-23T09:57:28+0000,,i1,no,outdoor //! e3,produce item,2026-01-24T09:57:28+0000,,i2#target,no,indoor @@ -21,7 +21,9 @@ //! - **`activity`**: Event type name, or "o2o" (case insensitive) for object relationships //! - **`timestamp`**: ISO 8601 formatted timestamp (empty for O2O relationships) //! - **Columns prefixed with `ot:`** (case-insensitive): Object type columns defining object involvements -//! - **Columns prefixed with `ea:`** (case-insensitive): Event attribute columns +//! - **Any other column**: An event attribute, named exactly as the header reads. For +//! compatibility with earlier versions of this format, a leading `ea:` prefix +//! (case-insensitive) is stripped on import; export never writes it. //! //! ## Object References //! @@ -73,6 +75,7 @@ //! Import → Export → Import may not be perfectly lossless: //! - Event attribute types are inferred as strings during CSV import //! - Column ordering may change (sorted alphabetically) +//! - Event attribute columns written with a legacy `ea:` prefix are re-exported without it //! - Object types are inferred from which `ot:` column an object first appears in //! //! # Import Options @@ -83,8 +86,11 @@ mod csv_ocel_export; mod csv_ocel_import; +mod escaping; #[doc(inline)] pub use csv_ocel_export::*; #[doc(inline)] pub use csv_ocel_import::*; +#[doc(inline)] +pub use escaping::{escape_reference_part, unescape_reference_part}; diff --git a/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs index 41771104..6964c4aa 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs @@ -290,6 +290,7 @@ where mod tests { use super::*; use crate::core::event_data::object_centric::linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}; + use crate::core::event_data::object_centric::ocel_struct::OCELAttributeValue; use crate::test_utils::{get_test_data_path, sort_ocel_for_equality_compare}; use std::collections::HashMap; use std::fs; @@ -391,6 +392,105 @@ mod tests { } } + /// Build a minimal OCEL JSON with a single event attribute of the given raw JSON value, + /// declared with `attr_type` in the event type. + fn json_with_event_attr(attr_type: &str, raw_value: &str) -> Vec { + format!( + r#"{{ + "eventTypes": [{{"name": "x", "attributes": [{{"name": "a", "type": "{attr_type}"}}]}}], + "objectTypes": [], + "events": [{{"id": "e1", "type": "x", "time": "2024-01-01T00:00:00Z", + "attributes": [{{"name": "a", "value": {raw_value}}}], "relationships": []}}], + "objects": [] + }}"# + ) + .into_bytes() + } + + fn attr_value_of(bytes: &[u8]) -> OCELAttributeValue { + let ocel = import_ocel_json_slice(bytes).unwrap(); + ocel.events[0].attributes[0].value.clone() + } + + /// Scalar inference is unchanged: strings stay strings unless they are RFC3339. + #[test] + fn import_scalar_attribute_value_inference() { + let cases: Vec<(&str, OCELAttributeValue)> = vec![ + ("42", OCELAttributeValue::Integer(42)), + ("-7", OCELAttributeValue::Integer(-7)), + ("1.5", OCELAttributeValue::Float(1.5)), + ("true", OCELAttributeValue::Boolean(true)), + ("null", OCELAttributeValue::Null), + ( + r#""hello""#, + OCELAttributeValue::String("hello".to_string()), + ), + // Not RFC3339 => stays a String at inference time. + ( + r#""2023-10-06 09:30:21.890421""#, + OCELAttributeValue::String("2023-10-06 09:30:21.890421".to_string()), + ), + ]; + for (raw, expected) in cases { + assert_eq!( + attr_value_of(&json_with_event_attr("string", raw)), + expected, + "raw value {raw:?}" + ); + } + // RFC3339 strings are recognized as timestamps (pre-existing untagged behavior). + assert!(matches!( + attr_value_of(&json_with_event_attr("string", r#""2024-01-01T00:00:00Z""#)), + OCELAttributeValue::Time(_) + )); + } + + /// Integers exceeding `i64` degrade to `Float` rather than failing the import. + #[test] + fn import_out_of_range_integer_becomes_float() { + assert!(matches!( + attr_value_of(&json_with_event_attr("integer", "18446744073709551615")), + OCELAttributeValue::Float(_) + )); + } + + /// Event timestamps accept the same non-RFC3339 formats as object attribute times. + #[test] + fn import_event_time_accepts_loose_formats() { + for time in [ + "2023-10-06 09:30:21.890421", + "2023-10-06T09:30:21", + "2023-10-06T09:30:21+0000", + "2023-10-06 09:30:21 UTC", + "Mon Apr 03 2023 12:08:18 GMT+0200 (Mitteleuropäische Sommerzeit)", + ] { + let json = format!( + r#"{{"eventTypes": [], "objectTypes": [], + "events": [{{"id": "e1", "type": "x", "time": "{time}", + "attributes": [], "relationships": []}}], + "objects": []}}"# + ); + assert!( + import_ocel_json_slice(json.as_bytes()).is_ok(), + "event time {time:?} should parse" + ); + } + } + + /// A string attribute declared as `time` but not in RFC3339 is recovered by the + /// type-directed coercion in `SlimLinkedOCEL`. + #[test] + fn declared_time_attribute_coerced_from_loose_format() { + let bytes = json_with_event_attr("time", r#""2023-10-06 09:30:21.890421""#); + let ocel = import_ocel_json_slice(&bytes).unwrap(); + let slim = SlimLinkedOCEL::from_ocel(ocel); + let back = slim.construct_ocel(); + assert!(matches!( + back.events[0].attributes[0].value, + OCELAttributeValue::Time(_) + )); + } + /// Streaming import directly into `SlimLinkedOCEL` matches the via-`from_ocel` baseline. #[test] fn import_into_slim_streaming() { diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_import.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_import.rs index 2f859ab6..6655d725 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_import.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_import.rs @@ -51,9 +51,22 @@ fn get_row_attribute_value( /// /// If you want to import from a filepath, see [`import_ocel_duckdb_from_path`] instead. /// +/// Rejects a file whose object-type table has no `ocel_changed_field` column. Use +/// [`import_ocel_duckdb_from_con_with_options`] to tolerate that instead. +/// /// Note: This function is only available if the `ocel-duckdb` feature is enabled. /// pub fn import_ocel_duckdb_from_con(con: Connection) -> Result { + import_ocel_duckdb_from_con_with_options(con, SqlOcelImportOptions::default()) +} + +/// Import [`OCEL`] log from `DuckDB` connection, with explicit [`SqlOcelImportOptions`]. +/// +/// Note: This function is only available if the `ocel-duckdb` feature is enabled. +pub fn import_ocel_duckdb_from_con_with_options( + con: Connection, + options: SqlOcelImportOptions, +) -> Result { let mut ocel = OCEL { event_types: Vec::default(), object_types: Vec::default(), @@ -85,25 +98,31 @@ pub fn import_ocel_duckdb_from_con(con: Connection) -> Result = HashMap::new(); for (ob_type, ob_type_ocel) in ob_type_map.iter() { - let mut s = con.prepare(format!("PRAGMA table_info('object_{ob_type}')").as_str())?; + let table_name = format!("object_{ob_type}"); + let mut s = + con.prepare(format!("PRAGMA table_info({})", quoted_str(&table_name)).as_str())?; let ob_attr_query = query_all::<_>(&mut s, [])?; - let ob_type_attrs: Vec = ob_attr_query + let raw_ob_columns: Vec<(String, String)> = ob_attr_query .and_then(|x| Ok::<(String, String), ::duckdb::Error>((x.get("name")?, x.get("type")?))) .flatten() - .filter(|(name, _)| !IGNORED_PRAGMA_COLUMNS.contains(&name.as_str())) - .map(|(name, atype)| OCELTypeAttribute { - name, - value_type: sql_type_to_ocel(&atype).to_type_string(), - }) .collect(); - let mut s = con.prepare( - format!("SELECT * FROM 'object_{ob_type}' WHERE {OCEL_CHANGED_FIELD} IS NULL").as_str(), - )?; + let ObjectTablePlan { + attributes: ob_type_attrs, + initial_query, + changed_query, + } = plan_object_table( + con.path().and_then(|p| p.to_str()), + ob_type, + &table_name, + raw_ob_columns, + options, + ) + .map_err(::duckdb::Error::InvalidColumnName)?; + let mut s = con.prepare(initial_query.as_str())?; let objs = query_all::<_>(&mut s, [])?; objs.and_then(|x| { - Ok::<(String, _, Vec<_>), ::duckdb::Error>(( + Ok::<(String, Vec<_>), ::duckdb::Error>(( x.get(OCEL_ID_COLUMN)?, - try_get_column_date_val(x, OCEL_TIME_COLUMN)?, ob_type_attrs .iter() .flat_map(|attr| { @@ -116,19 +135,16 @@ pub fn import_ocel_duckdb_from_con(con: Connection) -> Result Result(&mut s, [])?; - objs.and_then(|x| { - let changed_field: String = x.get(OCEL_CHANGED_FIELD)?; - let changed_val = ob_type_attrs - .iter() - .find(|at| at.name == changed_field) - .ok_or_else(|| { - println!( - "Could not get change field for {:?} in {:?}", - changed_field, ob_type_attrs - ); - ::duckdb::Error::InvalidQuery - }) - .and_then(|attr| get_row_attribute_value(attr, x)) - .unwrap(); - Ok::<(String, _, String, OCELAttributeValue), ::duckdb::Error>(( - x.get(OCEL_ID_COLUMN)?, - try_get_column_date_val(x, OCEL_TIME_COLUMN)?, - changed_field, - changed_val, - )) - }) - .flatten() - .for_each(|(ob_id, time, changed_field, changed_val)| { - object_map - .entry(ob_id.clone()) - .or_insert(OCELObject { - id: ob_id, - object_type: ob_type.clone(), - attributes: Vec::default(), - relationships: Vec::default(), - }) - .attributes - .push(OCELObjectAttribute { - name: changed_field, - value: changed_val, - time, - }); - }); + if let Some(changed_query) = &changed_query { + let mut s = con.prepare(changed_query.as_str())?; + let objs = query_all::<_>(&mut s, [])?; + objs.and_then(|x| { + let changed_field: String = x.get(OCEL_CHANGED_FIELD)?; + let changed_val = ob_type_attrs + .iter() + .find(|at| at.name == changed_field) + .ok_or(::duckdb::Error::InvalidQuery) + .and_then(|attr| get_row_attribute_value(attr, x))?; + Ok::<(String, _, String, OCELAttributeValue), ::duckdb::Error>(( + x.get(OCEL_ID_COLUMN)?, + try_get_column_date_val(x, OCEL_TIME_COLUMN)?, + changed_field, + changed_val, + )) + }) + .flatten() + .for_each(|(ob_id, time, changed_field, changed_val)| { + object_map + .entry(ob_id.clone()) + .or_insert(OCELObject { + id: ob_id, + object_type: ob_type.clone(), + attributes: Vec::default(), + relationships: Vec::default(), + }) + .attributes + .push(OCELObjectAttribute { + name: changed_field, + value: changed_val, + time, + }); + }); + } let t = OCELType { name: ob_type_ocel.clone(), @@ -196,7 +204,13 @@ pub fn import_ocel_duckdb_from_con(con: Connection) -> Result(&mut s, [])?; let ev_type_attrs: Vec = ev_attr_query .and_then(|x| Ok::<(String, String), ::duckdb::Error>((x.get("name")?, x.get("type")?))) @@ -208,7 +222,13 @@ pub fn import_ocel_duckdb_from_con(con: Connection) -> Result(&mut s, [])?; evs.and_then(|x| { Ok::<(String, _, Vec<_>), ::duckdb::Error>(( @@ -318,3 +338,162 @@ pub fn import_ocel_duckdb_from_path>( let con = Connection::open(path)?; import_ocel_duckdb_from_con(con) } + +#[cfg(test)] +mod missing_changed_field_tests { + use ::duckdb::Connection; + use chrono::DateTime; + + use crate::core::event_data::object_centric::{ + ocel_sql::{ + import_ocel_duckdb_from_con, import_ocel_duckdb_from_con_with_options, + SqlOcelImportOptions, + }, + ocel_struct::{OCELAttributeValue, OCELObjectAttribute}, + }; + + /// A non-conforming `DuckDB` file: object type `Truck` has no `ocel_changed_field` column, + /// but does carry a misspelled `ocel_change_field` one whose value names a real attribute. + fn build_fixture(path: &std::path::Path) { + let con = Connection::open(path).unwrap(); + con.execute_batch( + " + CREATE TABLE event_map_type (ocel_type_map TEXT, ocel_type TEXT); + CREATE TABLE object_map_type (ocel_type_map TEXT, ocel_type TEXT); + CREATE TABLE event_object (ocel_event_id TEXT, ocel_object_id TEXT, ocel_qualifier TEXT); + CREATE TABLE object_object (ocel_source_id TEXT, ocel_target_id TEXT, ocel_qualifier TEXT); + CREATE TABLE event_Ev (ocel_id TEXT PRIMARY KEY, ocel_time TIMESTAMP); + CREATE TABLE object_Truck ( + ocel_id TEXT PRIMARY KEY, + ocel_time TIMESTAMP, + driver TEXT, + ocel_change_field TEXT + ); + INSERT INTO object_map_type VALUES ('Truck', 'Truck'); + INSERT INTO event_map_type VALUES ('Ev', 'Ev'); + INSERT INTO event_Ev VALUES ('e1', '2020-01-01T00:00:00+00:00'); + INSERT INTO object_Truck VALUES ('t1', '2020-01-01T00:00:00+00:00', 'Alice', NULL); + INSERT INTO object_Truck VALUES ('t2', '2020-01-02T00:00:00+00:00', 'Bob', 'driver'); + ", + ) + .unwrap(); + } + + /// The column is only missing where there is no attribute change to record, so a file + /// without it is read rather than refused. + #[test] + fn default_import_reads_a_file_without_the_changed_field_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.duckdb"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let ocel = + import_ocel_duckdb_from_con(con).expect("a file without the column must still be read"); + assert_eq!(ocel.objects.len(), 2, "both Truck objects must be present"); + } + + /// Holding the file to the specification is opt-in, and names what is wrong with it. + #[test] + fn strict_import_rejects_missing_changed_field_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.duckdb"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let err = import_ocel_duckdb_from_con_with_options( + con, + SqlOcelImportOptions { + allow_missing_changed_field: false, + }, + ) + .unwrap_err(); + let msg = err.to_string(); + + assert!( + msg.contains("Truck"), + "message should name the object type: {msg}" + ); + assert!( + msg.contains("object_Truck"), + "message should name the table: {msg}" + ); + assert!( + msg.contains("ocel_changed_field"), + "message should name the missing column: {msg}" + ); + assert!( + msg.contains("does not conform"), + "message should say the file is non-conforming: {msg}" + ); + assert!( + msg.contains("ocel_change_field"), + "message should mention the misspelled column it found: {msg}" + ); + } + + #[test] + fn a_file_without_the_changed_field_column_reads_as_initial_state() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.duckdb"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let ocel = import_ocel_duckdb_from_con_with_options( + con, + SqlOcelImportOptions { + allow_missing_changed_field: true, + }, + ) + .unwrap(); + + assert_eq!(ocel.objects.len(), 2, "both Truck objects must be present"); + + let t1 = ocel.objects.iter().find(|o| o.id == "t1").unwrap(); + assert_eq!(t1.object_type, "Truck"); + assert_eq!( + t1.attributes, + vec![OCELObjectAttribute { + name: "driver".to_string(), + value: OCELAttributeValue::String("Alice".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }], + "t1's ocel_change_field cell is NULL, so it contributes no attribute; \ + it must not be invented as a change record" + ); + + let t2 = ocel.objects.iter().find(|o| o.id == "t2").unwrap(); + assert_eq!(t2.object_type, "Truck"); + let mut t2_attrs = t2.attributes.clone(); + t2_attrs.sort_by(|a, b| a.name.cmp(&b.name)); + assert_eq!( + t2_attrs, + vec![ + OCELObjectAttribute { + name: "driver".to_string(), + value: OCELAttributeValue::String("Bob".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }, + OCELObjectAttribute { + name: "ocel_change_field".to_string(), + value: OCELAttributeValue::String("driver".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }, + ], + "no attribute-change row must be invented from the misspelled column" + ); + + let truck_type = ocel + .object_types + .iter() + .find(|t| t.name == "Truck") + .unwrap(); + let mut attr_names: Vec<_> = truck_type + .attributes + .iter() + .map(|a| a.name.clone()) + .collect(); + attr_names.sort(); + assert_eq!(attr_names, vec!["driver", "ocel_change_field"]); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/mod.rs index 27996732..3acf453e 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/mod.rs @@ -1,5 +1,14 @@ pub(crate) mod duckdb_ocel_export; pub(crate) mod duckdb_ocel_import; +pub(crate) mod schema; + +#[cfg(feature = "ocel-duckdb")] +pub use schema::{ + generate_type_views, read_consolidated_ocel_from_duckdb_path, + read_consolidated_slim_ocel_from_duckdb_path, read_ocel_from_duckdb, + stream_ocel_file_to_duckdb, stream_ocel_file_to_duckdb_with, write_ocel_to_duckdb, + write_ocel_to_duckdb_with, DuckDbImportOptions, +}; #[cfg(test)] mod duckdb_tests { diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/mod.rs new file mode 100644 index 00000000..001c15d2 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/mod.rs @@ -0,0 +1,19 @@ +//! `DuckDB` OCEL schema (fixed tables; EAV object attributes): streaming import + on-demand wide views. +pub(crate) mod reader; +pub(crate) mod sink; +#[cfg(feature = "ocel-sqlite")] +pub(crate) mod sqlite_source; +pub(crate) mod stream; +pub(crate) mod tables; +pub(crate) mod value; +pub(crate) mod views; + +pub use reader::{ + read_consolidated_ocel_from_duckdb_path, read_consolidated_slim_ocel_from_duckdb_path, + read_ocel_from_duckdb, +}; +pub use stream::{ + stream_ocel_file_to_duckdb, stream_ocel_file_to_duckdb_with, write_ocel_to_duckdb, + write_ocel_to_duckdb_with, DuckDbImportOptions, +}; +pub use views::generate_type_views; diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/reader.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/reader.rs new file mode 100644 index 00000000..000d4d28 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/reader.rs @@ -0,0 +1,553 @@ +//! Read a `DuckDB` database back into an [`OCEL`] (consolidated schema). +use std::collections::HashMap; + +use chrono::{DateTime, FixedOffset}; +use duckdb::Connection; + +use crate::core::event_data::object_centric::appendable::AppendableOCEL; +use crate::core::event_data::object_centric::io::OCELIOError; +use crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL; +use crate::core::event_data::object_centric::ocel_struct::{ + OCELEventAttribute, OCELObjectAttribute, OCELRelationship, OCELType, OCELTypeAttribute, OCEL, +}; +use crate::core::event_data::timestamp_utils::parse_timestamp; +use macros_process_mining::register_binding; + +use super::value::{duck_timestamp_to_datetime, duck_value_to_ocel, from_sql_value}; + +/// Read an [`OCEL`] from a `DuckDB` database written by +/// [`stream_ocel_file_to_duckdb`](super::stream::stream_ocel_file_to_duckdb). Eager: the +/// whole log is materialized in memory. +/// +/// Reads the consolidated schema, not the per-type table layout that +/// [`import_ocel_duckdb_from_con`](crate::core::event_data::object_centric::ocel_sql::import_ocel_duckdb_from_con) +/// expects. +pub fn read_ocel_from_duckdb(con: &Connection) -> Result { + let mut ocel = OCEL { + event_types: Vec::new(), + object_types: Vec::new(), + events: Vec::new(), + objects: Vec::new(), + }; + ocel.read_from_duckdb(con)?; + Ok(ocel) +} + +/// Read an [`OCEL`] from a `DuckDB` database file in the consolidated schema, i.e. one written by +/// [`stream_ocel_file_to_duckdb`](super::stream::stream_ocel_file_to_duckdb). +/// +/// Databases in the per-type table layout of the OCEL 2.0 standard are read by +/// [`import_ocel_duckdb_from_path`](crate::core::event_data::object_centric::ocel_sql::import_ocel_duckdb_from_path) +/// instead. +#[register_binding(name = "read_consolidated_ocel_from_duckdb", stringify_error)] +pub fn read_consolidated_ocel_from_duckdb_path( + db_path: impl AsRef, +) -> Result { + let con = Connection::open(db_path)?; + read_ocel_from_duckdb(&con) +} + +/// Read a [`SlimLinkedOCEL`] from a `DuckDB` database file in the consolidated schema, i.e. one +/// written by [`stream_ocel_file_to_duckdb`](super::stream::stream_ocel_file_to_duckdb). +/// +/// Rows are read into the linked structure directly, without building an [`OCEL`] first. +#[register_binding(name = "read_consolidated_slim_ocel_from_duckdb", stringify_error)] +pub fn read_consolidated_slim_ocel_from_duckdb_path( + db_path: impl AsRef, +) -> Result { + let con = Connection::open(db_path)?; + SlimLinkedOCEL::from_duckdb(&con) +} + +/// Read a `DuckDB` schema database into any [`AppendableOCEL`] sink. +/// +/// Currently buffers `e2o`/`o2o` relationships and object attributes into maps. +/// TODO: Implement real row-streaming with SQL joins etc. +pub(crate) trait DuckDbReadInto: AppendableOCEL { + fn read_from_duckdb(&mut self, con: &Connection) -> Result<(), OCELIOError> + where + Self::Error: Into, + { + // Event-type attributes come from the persisted `event_attr_meta`. + for t in collect_types( + con, + "SELECT event_type, attr_name, attr_type FROM event_attr_meta", + "SELECT DISTINCT ocel_type FROM events", + )? { + self.declare_event_type(t).map_err(Into::into)?; + } + // Object-type attributes come from `object_attr_meta`, unioned with the types observed in + // the change rows: a declaration no row ever wrote exists only in the meta table. + for t in collect_types( + con, + r#"SELECT object_type, attr_name, attr_type FROM object_attr_meta + UNION + SELECT DISTINCT o.ocel_type, oa.name, oa.value_type + FROM objects o JOIN object_attribute_changes oa ON oa.id = o.id"#, + r#"SELECT DISTINCT ocel_type FROM objects + UNION + SELECT DISTINCT object_type FROM object_attr_meta"#, + )? { + self.declare_object_type(t).map_err(Into::into)?; + } + + // Buffer relationships + object attributes (append_* wants them up front). + let mut e2o: HashMap> = HashMap::new(); + { + let mut stmt = con.prepare("SELECT event_id, object_id, qualifier FROM e2o")?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + for row in rows { + let (event_id, object_id, qualifier) = row?; + e2o.entry(event_id).or_default().push(OCELRelationship { + object_id, + qualifier, + }); + } + } + let mut o2o: HashMap> = HashMap::new(); + { + let mut stmt = con.prepare("SELECT source_id, target_id, qualifier FROM o2o")?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + for row in rows { + let (source_id, target_id, qualifier) = row?; + o2o.entry(source_id).or_default().push(OCELRelationship { + object_id: target_id, + qualifier, + }); + } + } + let mut ob_attrs: HashMap> = HashMap::new(); + { + let mut stmt = con.prepare( + r#"SELECT id, name, "time", value, value_type FROM object_attribute_changes"#, + )?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, duckdb::types::Value>(2)?, + r.get::<_, String>(3)?, + r.get::<_, String>(4)?, + )) + })?; + for row in rows { + let (id, name, time_val, value, value_type) = row?; + let time = value_to_datetime(&time_val)?; + ob_attrs.entry(id).or_default().push(OCELObjectAttribute { + name, + value: from_sql_value(&value, &value_type), + time, + }); + } + } + + // Objects first so events' e2o object references already exist. + { + let mut stmt = con.prepare("SELECT id, ocel_type FROM objects")?; + let rows = + stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?; + for row in rows { + let (id, ocel_type) = row?; + let attributes = ob_attrs.remove(&id).unwrap_or_default(); + let relationships = o2o.remove(&id).unwrap_or_default(); + self.append_object(id, &ocel_type, attributes, relationships) + .map_err(Into::into)?; + } + } + + { + let col_names: Vec = { + let mut stmt = con.prepare( + "SELECT column_name FROM information_schema.columns \ + WHERE table_name = 'events' ORDER BY ordinal_position", + )?; + stmt.query_map([], |r| r.get::<_, String>(0))? + .collect::>()? + }; + let n = col_names.len(); + let mut stmt = con.prepare("SELECT * FROM events")?; + let rows = stmt.query_map([], |r| { + let mut vals: Vec = Vec::with_capacity(n); + for i in 0..n { + vals.push(r.get::<_, duckdb::types::Value>(i)?); + } + Ok(vals) + })?; + for row in rows { + let vals = row?; + let id = as_text(&vals[0], "id")?; + let ocel_type = as_text(&vals[1], "ocel_type")?; + let time = value_to_datetime(&vals[2])?; + let attributes = (3..n) + .filter(|&i| !matches!(vals[i], duckdb::types::Value::Null)) + .map(|i| OCELEventAttribute { + name: super::tables::event_attr_name(&col_names[i]).into_owned(), + value: duck_value_to_ocel(vals[i].clone()), + }) + .collect(); + let relationships = e2o.remove(&id).unwrap_or_default(); + self.append_event(id, &ocel_type, time, attributes, relationships) + .map_err(Into::into)?; + } + } + + self.finalize().map_err(Into::into)?; + Ok(()) + } +} + +impl DuckDbReadInto for A {} + +/// Read a `TEXT` cell, erroring (naming `column`) instead of silently collapsing distinct +/// non-text values to `""`, which would be fatal for an `id`/`ocel_type` cell. +fn as_text(v: &duckdb::types::Value, column: &str) -> Result { + match v { + duckdb::types::Value::Text(s) => Ok(s.clone()), + _ => Err(duckdb::Error::InvalidColumnType( + 0, + format!("expected TEXT in column {column:?}, got {v:?}"), + duckdb::types::Type::Text, + )), + } +} + +/// Read a timestamp column. Text is accepted too, for databases written by other tooling. +fn value_to_datetime(v: &duckdb::types::Value) -> Result, duckdb::Error> { + match v { + duckdb::types::Value::Timestamp(tu, t) => { + duck_timestamp_to_datetime(*tu, *t).ok_or(duckdb::Error::InvalidQuery) + } + duckdb::types::Value::Text(s) => { + parse_timestamp(s, None, false).map_err(|_| duckdb::Error::InvalidQuery) + } + _ => Err(duckdb::Error::InvalidQuery), + } +} + +/// Build `OCELType`s: every name from `all_types_sql`, `attributes` from `attr_sql`'s +/// `(type, name, value_type)` rows. +/// +/// One entry per distinct `(type, name, value_type)`: a name reported under several types (e.g. +/// `object_attr_meta`'s declaration and the types actually observed in +/// `object_attribute_changes` disagree) yields one `OCELTypeAttribute` per type observed for it, +/// not a single widened one, since there is no declaration-level type to prefer over another. +fn collect_types( + con: &Connection, + attr_sql: &str, + all_types_sql: &str, +) -> Result, duckdb::Error> { + let mut attrs_by_type: HashMap> = HashMap::new(); + { + let mut stmt = con.prepare(attr_sql)?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + for row in rows { + let (ocel_type, name, value_type) = row?; + let entry = attrs_by_type.entry(ocel_type).or_default(); + if !entry + .iter() + .any(|a| a.name == name && a.value_type == value_type) + { + entry.push(OCELTypeAttribute { name, value_type }); + } + } + } + + let mut types: Vec = Vec::new(); + let mut stmt = con.prepare(all_types_sql)?; + let rows = stmt.query_map([], |r| r.get::<_, String>(0))?; + for row in rows { + let name = row?; + let attributes = attrs_by_type.remove(&name).unwrap_or_default(); + types.push(OCELType { name, attributes }); + } + Ok(types) +} + +#[cfg(all(test, feature = "ocel-duckdb"))] +mod tests { + use std::collections::HashSet; + + use chrono::{DateTime, FixedOffset}; + + use super::super::stream::stream_ocel_file_to_duckdb; + use crate::core::event_data::object_centric::linked_ocel::{ + IndexLinkedOCEL, LinkedOCELAccess, SlimLinkedOCEL, + }; + use crate::core::event_data::object_centric::ocel_json::import_ocel_json_path; + use crate::test_utils::get_test_data_path; + + fn order_management_path() -> std::path::PathBuf { + get_test_data_path() + .join("ocel") + .join("order-management.json") + } + + // Object attributes live in the EAV change table, so an uninstantiated declaration survives + // only in `object_attr_meta`. + #[test] + fn an_object_type_with_no_instances_and_an_unwritten_attribute_survive_a_roundtrip() { + use crate::core::event_data::object_centric::appendable::AppendableOCEL; + use crate::core::event_data::object_centric::ocel_struct::{OCELType, OCELTypeAttribute}; + + let con = duckdb::Connection::open_in_memory().unwrap(); + super::super::tables::create_schema(&con).unwrap(); + { + let mut sink = super::super::sink::DuckDbOcelSink::new(&con).unwrap(); + sink.declare_object_type(OCELType { + name: "CreditNote".to_string(), + attributes: vec![OCELTypeAttribute { + name: "reason".to_string(), + value_type: "string".to_string(), + }], + }) + .unwrap(); + sink.finalize().unwrap(); + } + + let loaded = super::read_ocel_from_duckdb(&con).unwrap(); + let ty = loaded + .object_types + .iter() + .find(|t| t.name == "CreditNote") + .expect("a declared object type with no instances must survive"); + assert_eq!( + ty.attributes + .iter() + .map(|a| (a.name.as_str(), a.value_type.as_str())) + .collect::>(), + vec![("reason", "string")] + ); + } + + // ocel2-p2p is used because its event types declare attributes, unlike order-management. + #[test] + fn event_attr_declarations_and_values_roundtrip() { + let src = get_test_data_path().join("ocel").join("ocel2-p2p.json"); + let reference = IndexLinkedOCEL::from_ocel(import_ocel_json_path(&src).unwrap()); + + let out = get_test_data_path() + .join("export") + .join("p2p-attr-roundtrip.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = duckdb::Connection::open(&out).unwrap(); + let loaded = IndexLinkedOCEL::from_ocel(super::read_ocel_from_duckdb(&con).unwrap()); + + // (1) Declared `attributes` (name + value_type) round-trip via event_attr_meta. + let mut types_checked = 0; + for tname in reference.get_ev_types() { + let want: HashSet<(String, String)> = reference + .get_ev_type(tname) + .expect("reference type") + .attributes + .iter() + .map(|a| (a.name.clone(), a.value_type.clone())) + .collect(); + let got: HashSet<(String, String)> = loaded + .get_ev_type(tname) + .unwrap_or_else(|| panic!("loaded missing event type {tname}")) + .attributes + .iter() + .map(|a| (a.name.clone(), a.value_type.clone())) + .collect(); + assert_eq!(got, want, "declared attributes mismatch for type {tname}"); + if !want.is_empty() { + types_checked += 1; + } + } + assert!(types_checked > 1, "expected >1 type carrying attributes"); + + // (2) Attribute values round-trip: first valued event per type; >1 type, >1 attr. + let mut typed_events_checked = 0; + let mut attr_values_checked = 0; + for tname in reference.get_ev_types() { + let Some(ref_ev) = reference + .get_evs_of_type(tname) + .find(|e| reference.get_ev_attrs(*e).next().is_some()) + else { + continue; + }; + let id = reference.get_ev_id(ref_ev).to_string(); + let loaded_ev = loaded.get_ev_by_id(&id).expect("loaded has event"); + let mut names: Vec = + reference.get_ev_attrs(ref_ev).map(str::to_string).collect(); + names.sort(); + for name in &names { + let want = reference.get_ev_attr_val(ref_ev, name); + let got = loaded.get_ev_attr_val(&loaded_ev, name); + assert_eq!(got, want, "event {id} attr {name} value mismatch"); + attr_values_checked += 1; + } + typed_events_checked += 1; + } + assert!( + typed_events_checked > 1, + "expected >1 type with a valued event" + ); + assert!( + attr_values_checked > 1, + "expected >1 attribute value checked" + ); + } + + #[test] + fn from_duckdb_matches_json() { + let src = order_management_path(); + let reference = IndexLinkedOCEL::from_ocel(import_ocel_json_path(&src).unwrap()); + + let out = get_test_data_path() + .join("export") + .join("from-duckdb-parity.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = duckdb::Connection::open(&out).unwrap(); + let loaded = IndexLinkedOCEL::from_ocel(super::read_ocel_from_duckdb(&con).unwrap()); + + assert_eq!(loaded.get_num_evs(), reference.get_num_evs()); + assert_eq!(loaded.get_num_obs(), reference.get_num_obs()); + assert_eq!( + loaded.get_ev_types().count(), + reference.get_ev_types().count() + ); + assert_eq!( + loaded.get_ob_types().count(), + reference.get_ob_types().count() + ); + + // Spot-check a sample of event ids: type, time, and e2o set parity. + let sample_ev_ids: Vec = reference + .get_all_evs() + .take(25) + .map(|e| reference.get_ev_id(e).to_string()) + .collect(); + assert!(!sample_ev_ids.is_empty()); + for id in &sample_ev_ids { + let ref_ev = reference.get_ev_by_id(id).expect("reference has event"); + let loaded_ev = loaded.get_ev_by_id(id).expect("loaded should have event"); + + assert_eq!( + loaded.get_ev_type_of(loaded_ev), + reference.get_ev_type_of(ref_ev), + "event type mismatch for {id}" + ); + assert_eq!( + loaded.get_ev_time(loaded_ev), + reference.get_ev_time(ref_ev), + "event time mismatch for {id}" + ); + + let ref_e2o: HashSet<(String, String)> = reference + .get_e2o(ref_ev) + .map(|(q, o)| (q.to_string(), reference.get_ob_id(*o).to_string())) + .collect(); + let loaded_e2o: HashSet<(String, String)> = loaded + .get_e2o(loaded_ev) + .map(|(q, o)| (q.to_string(), loaded.get_ob_id(*o).to_string())) + .collect(); + assert_eq!(loaded_e2o, ref_e2o, "e2o mismatch for {id}"); + } + + // Spot-check an object with attributes (order-management carries them on objects, + // not on events). + let ref_ob_with_attr = reference + .get_all_obs() + .find_map(|ob| { + let name = reference + .get_ocel_ref() + .objects + .iter() + .find(|o| o.id == reference.get_ob_id(ob))? + .attributes + .first()? + .name + .clone(); + let vals: Vec<_> = reference.get_ob_attr_vals(ob, &name).collect(); + if vals.is_empty() { + None + } else { + Some((reference.get_ob_id(ob).to_string(), name)) + } + }) + .expect("order-management should have an object with an attribute"); + let (ob_id, attr_name) = ref_ob_with_attr; + + let ref_ob = reference.get_ob_by_id(&ob_id).unwrap(); + let loaded_ob = loaded + .get_ob_by_id(&ob_id) + .expect("loaded should have object"); + + let mut ref_vals: Vec<(DateTime, String)> = reference + .get_ob_attr_vals(ref_ob, &attr_name) + .map(|(t, v)| (*t, v.to_string())) + .collect(); + let mut loaded_vals: Vec<(DateTime, String)> = loaded + .get_ob_attr_vals(loaded_ob, &attr_name) + .map(|(t, v)| (*t, v.to_string())) + .collect(); + ref_vals.sort(); + loaded_vals.sort(); + assert_eq!( + loaded_vals, ref_vals, + "object attr value/time set mismatch for {ob_id}/{attr_name}" + ); + } + + // A non-text id/ocel_type cell would otherwise silently collapse to "", making distinct + // events collide instead of surfacing the schema mismatch. + #[test] + fn as_text_errors_on_non_text_cell_naming_the_column() { + let err = super::as_text(&duckdb::types::Value::Int(5), "id").unwrap_err(); + assert!( + format!("{err}").contains("id"), + "error should name the offending column: {err}" + ); + assert!(super::as_text(&duckdb::types::Value::Null, "ocel_type").is_err()); + assert_eq!( + super::as_text(&duckdb::types::Value::Text("e1".to_string()), "id").unwrap(), + "e1" + ); + } + + #[test] + fn slim_from_duckdb_matches_from_ocel() { + let src = order_management_path(); + let reference = SlimLinkedOCEL::from_ocel(import_ocel_json_path(&src).unwrap()); + + let out = get_test_data_path() + .join("export") + .join("slim-from-duckdb.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = duckdb::Connection::open(&out).unwrap(); + let loaded = SlimLinkedOCEL::from_duckdb(&con).unwrap(); + + assert_eq!(loaded.get_num_evs(), reference.get_num_evs()); + assert_eq!(loaded.get_num_obs(), reference.get_num_obs()); + assert_eq!( + loaded.get_ev_types().count(), + reference.get_ev_types().count() + ); + assert_eq!( + loaded.get_ob_types().count(), + reference.get_ob_types().count() + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sink.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sink.rs new file mode 100644 index 00000000..c74de3e9 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sink.rs @@ -0,0 +1,534 @@ +//! Streaming sink writing the schema. +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, FixedOffset}; +use duckdb::{Appender, Connection, ToSql}; + +use crate::core::event_data::object_centric::appendable::AppendableOCEL; +use crate::core::event_data::object_centric::ocel_struct::{ + OCELAttributeType, OCELEventAttribute, OCELObjectAttribute, OCELRelationship, OCELType, +}; + +use super::tables::{ + add_events_column, build_events_table, T_E2O, T_EVENTS, T_EVENT_ATTR_META, T_O2O, T_OBJECTS, + T_OBJECT_ATTR_CHANGES, T_OBJECT_ATTR_META, +}; +use super::value::{datetime_to_duck_timestamp, ocel_value_to_duck, to_sql_value}; + +/// Streaming sink for the schema. Holds one appender per table, each borrowing `&'con Connection`. +/// The `events` appender is created once the wide schema is known. +pub(crate) struct DuckDbOcelSink<'con> { + events: Option>, + event_attr_meta: Option>, + object_attr_meta: Option>, + e2o: Option>, + objects: Option>, + object_attribute_changes: Option>, + o2o: Option>, + con: &'con Connection, + /// Event-attr schema accumulated over all `declare_event_type` calls: name -> type, widened + /// via [`OCELAttributeType::coalesce`] when event types disagree (integer + float -> float). + ev_attr_types: HashMap, + /// `ev_attr_types` frozen into ordered columns on the first `append_event`, then extended by + /// any column added on the fly. The type is both the SQL column type and the conversion target. + ev_columns: Vec<(String, OCELAttributeType)>, + /// name -> index into `ev_columns` (for building a full-width row per event). + ev_col_index: HashMap, + events_created: bool, + /// Undeclared attribute name -> event types already written to `event_attr_meta`, + /// so a recurring name costs one meta row per event type, not one per event. + undeclared_meta_written: HashMap>, + /// Event type -> attribute names it declared, so an attribute another type declared still + /// gets this type's own `event_attr_meta` row when it first shows up on one of its events. + declared_ev_attrs: HashMap>, +} + +impl<'con> DuckDbOcelSink<'con> { + pub(crate) fn new(con: &'con Connection) -> Result { + Ok(Self { + events: None, + event_attr_meta: Some(con.appender(T_EVENT_ATTR_META)?), + object_attr_meta: Some(con.appender(T_OBJECT_ATTR_META)?), + e2o: Some(con.appender(T_E2O)?), + objects: Some(con.appender(T_OBJECTS)?), + object_attribute_changes: Some(con.appender(T_OBJECT_ATTR_CHANGES)?), + o2o: Some(con.appender(T_O2O)?), + con, + ev_attr_types: HashMap::new(), + ev_columns: Vec::new(), + ev_col_index: HashMap::new(), + events_created: false, + undeclared_meta_written: HashMap::new(), + declared_ev_attrs: HashMap::new(), + }) + } + + /// Add a column for an attribute no event type declared, so its values are not dropped. + /// `DuckDB` refuses `ALTER TABLE` while an appender is open, hence the flush/reopen, which + /// happens once per new name rather than per event. + fn add_undeclared_event_column( + &mut self, + name: &str, + event_type: &str, + ) -> Result<(), duckdb::Error> { + if let Some(mut a) = self.events.take() { + a.flush()?; + } + // Nothing declared this attribute, so nothing says what it holds. + let ty = OCELAttributeType::String; + self.con.execute_batch(&add_events_column(name, ty))?; + self.events = Some(self.con.appender(T_EVENTS)?); + // `ADD COLUMN` appends at the end, matching the push onto `ev_columns`. + self.ev_col_index + .insert(name.to_string(), self.ev_columns.len()); + self.ev_columns.push((name.to_string(), ty)); + self.record_undeclared_meta(name, event_type) + } + + /// Record an undeclared attribute in `event_attr_meta`, once per event type, so the + /// reader and `generate_type_views` both pick it up. + fn record_undeclared_meta( + &mut self, + name: &str, + event_type: &str, + ) -> Result<(), duckdb::Error> { + if self + .undeclared_meta_written + .get(name) + .is_some_and(|types| types.contains(event_type)) + { + return Ok(()); + } + self.event_attr_meta.as_mut().unwrap().append_row([ + event_type, + name, + OCELAttributeType::String.as_type_str(), + ])?; + self.undeclared_meta_written + .entry(name.to_string()) + .or_default() + .insert(event_type.to_string()); + Ok(()) + } + + /// Create the wide `events` table (attribute columns sorted for determinism) and its appender. + /// Idempotent: called on the first `append_event` and, for an event-less import, in `finalize`. + fn ensure_events_created(&mut self) -> Result<(), duckdb::Error> { + if self.events_created { + return Ok(()); + } + let mut names: Vec = self.ev_attr_types.keys().cloned().collect(); + names.sort(); + self.ev_columns = names + .into_iter() + .map(|n| { + let ty = self.ev_attr_types[&n]; + (n, ty) + }) + .collect(); + self.ev_col_index = self + .ev_columns + .iter() + .enumerate() + .map(|(i, (n, _))| (n.clone(), i)) + .collect(); + self.con + .execute_batch(&build_events_table(&self.ev_columns))?; + self.events = Some(self.con.appender(T_EVENTS)?); + self.events_created = true; + Ok(()) + } +} + +impl<'con> AppendableOCEL for DuckDbOcelSink<'con> { + type Error = duckdb::Error; + + // Accumulate the wide event-attr schema and record the declared attributes in + // `event_attr_meta` for round-trip export. A name declared with different types across + // event types widens to the type covering both (see `coalesce`). + fn declare_event_type(&mut self, t: OCELType) -> Result<(), Self::Error> { + let meta = self.event_attr_meta.as_mut().unwrap(); + let declared = self.declared_ev_attrs.entry(t.name.clone()).or_default(); + for a in &t.attributes { + let ty = OCELAttributeType::from_type_str(&a.value_type); + self.ev_attr_types + .entry(a.name.clone()) + .and_modify(|existing| *existing = existing.coalesce(ty)) + .or_insert(ty); + declared.insert(a.name.clone()); + meta.append_row([&t.name, &a.name, &a.value_type])?; + } + Ok(()) + } + // Record the declared object attributes in `object_attr_meta`. Object attributes live in the + // EAV change table, so a type with no instances, or an attribute no row ever wrote, has + // nowhere else to be observed from. + fn declare_object_type(&mut self, t: OCELType) -> Result<(), Self::Error> { + let meta = self.object_attr_meta.as_mut().unwrap(); + for a in &t.attributes { + meta.append_row([&t.name, &a.name, &a.value_type])?; + } + Ok(()) + } + + fn append_event( + &mut self, + id: String, + event_type: &str, + time: DateTime, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + self.ensure_events_created()?; + + // Give undeclared attributes a column before the row is sized, so none are dropped. + // Declarations are tracked per event type: a name only another type declared has a + // column already but still needs this type's own meta row. + for a in &attributes { + if !self.ev_col_index.contains_key(&a.name) { + self.add_undeclared_event_column(&a.name, event_type)?; + } else if !self + .declared_ev_attrs + .get(event_type) + .is_some_and(|names| names.contains(&a.name)) + { + self.record_undeclared_meta(&a.name, event_type)?; + } + } + + // Full-width row: [id, ocel_type, time, ], NULL where unfilled. + let mut row: Vec = Vec::with_capacity(3 + self.ev_columns.len()); + row.push(duckdb::types::Value::Text(id)); + row.push(duckdb::types::Value::Text(event_type.to_string())); + row.push(datetime_to_duck_timestamp(time)); + row.extend(std::iter::repeat_n( + duckdb::types::Value::Null, + self.ev_columns.len(), + )); + for a in &attributes { + let idx = self.ev_col_index[&a.name]; + row[3 + idx] = ocel_value_to_duck(&a.value, self.ev_columns[idx].1); + } + let params: Vec<&dyn ToSql> = row.iter().map(|v| v as &dyn ToSql).collect(); + self.events + .as_mut() + .unwrap() + .append_row(params.as_slice())?; + + let id_ref = match &row[0] { + duckdb::types::Value::Text(s) => s.as_str(), + _ => unreachable!(), + }; + let e2o = self.e2o.as_mut().unwrap(); + for r in &relationships { + e2o.append_row([id_ref, &r.object_id, &r.qualifier])?; + } + Ok(()) + } + + fn append_object( + &mut self, + id: String, + object_type: &str, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + self.objects + .as_mut() + .unwrap() + .append_row((&id, &object_type))?; + let oa = self.object_attribute_changes.as_mut().unwrap(); + for a in &attributes { + let (value, value_type) = to_sql_value(&a.value); + let value: &str = value.as_ref(); + let t = datetime_to_duck_timestamp(a.time); + oa.append_row((&id, &a.name, &t, value, value_type))?; + } + let o2o = self.o2o.as_mut().unwrap(); + for r in &relationships { + o2o.append_row([&id, &r.object_id, &r.qualifier])?; + } + Ok(()) + } + + fn finalize(&mut self) -> Result<(), Self::Error> { + // Ensure `events` exists even for an event-less import, so readers always find the table. + self.ensure_events_created()?; + // Flush explicitly (not just drop) so write errors such as PRIMARY KEY violations + // surface here instead of being swallowed by `Drop`. + if let Some(mut a) = self.events.take() { + a.flush()?; + } + if let Some(mut a) = self.event_attr_meta.take() { + a.flush()?; + } + if let Some(mut a) = self.object_attr_meta.take() { + a.flush()?; + } + if let Some(mut a) = self.e2o.take() { + a.flush()?; + } + if let Some(mut a) = self.objects.take() { + a.flush()?; + } + if let Some(mut a) = self.object_attribute_changes.take() { + a.flush()?; + } + if let Some(mut a) = self.o2o.take() { + a.flush()?; + } + // Indexes are created by `run_import` as the final step, after the optional file-size + // rewrite that would otherwise drop them. + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::tables::create_schema; + use super::*; + use crate::core::event_data::object_centric::ocel_struct::{ + OCELAttributeValue, OCELTypeAttribute, + }; + + #[test] + fn append_one_event_and_object() { + let con = Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + { + let mut sink = DuckDbOcelSink::new(&con).unwrap(); + sink.declare_event_type(OCELType { + name: "pay".into(), + attributes: vec![OCELTypeAttribute { + name: "amount".into(), + value_type: "float".into(), + }], + }) + .unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.append_event( + "e1".into(), + "pay", + t, + vec![OCELEventAttribute { + name: "amount".into(), + value: OCELAttributeValue::Float(9.5), + }], + vec![OCELRelationship { + object_id: "o1".into(), + qualifier: "reg".into(), + }], + ) + .unwrap(); + sink.append_object( + "o1".into(), + "order", + vec![OCELObjectAttribute { + name: "prio".into(), + value: OCELAttributeValue::Integer(1), + time: t, + }], + vec![], + ) + .unwrap(); + sink.finalize().unwrap(); + } + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + // The event attribute is stored as a typed wide column on `events`. + let amount: f64 = con + .query_row(r#"SELECT "amount" FROM events WHERE id = 'e1'"#, [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(amount, 9.5); + let n_meta: i64 = con + .query_row("SELECT count(*) FROM event_attr_meta", [], |r| r.get(0)) + .unwrap(); + let n_e2o: i64 = con + .query_row("SELECT count(*) FROM e2o", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + let n_oa: i64 = con + .query_row("SELECT count(*) FROM object_attribute_changes", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!((n_ev, n_meta, n_e2o, n_ob, n_oa), (1, 1, 1, 1, 1)); + } + + #[test] + fn conflicting_declared_types_widen() { + // integer + float must widen to DOUBLE, staying numeric rather than collapsing to VARCHAR. + let con = Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + { + let mut sink = DuckDbOcelSink::new(&con).unwrap(); + for (ty, vt) in [("a", "integer"), ("b", "float")] { + sink.declare_event_type(OCELType { + name: ty.into(), + attributes: vec![OCELTypeAttribute { + name: "n".into(), + value_type: vt.into(), + }], + }) + .unwrap(); + } + let t = chrono::Utc::now().fixed_offset(); + sink.append_event( + "e1".into(), + "a", + t, + vec![OCELEventAttribute { + name: "n".into(), + value: OCELAttributeValue::Integer(7), + }], + vec![], + ) + .unwrap(); + sink.finalize().unwrap(); + } + let col_type: String = con + .query_row( + "SELECT data_type FROM information_schema.columns \ + WHERE table_name = 'events' AND column_name = 'n'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(col_type, "DOUBLE"); + // The integer value survives as a number, not as text. + let n: f64 = con + .query_row(r#"SELECT "n" FROM events WHERE id = 'e1'"#, [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(n, 7.0); + } + + #[test] + fn undeclared_event_attrs_get_a_column_instead_of_being_dropped() { + let con = Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + { + let mut sink = DuckDbOcelSink::new(&con).unwrap(); + // "pay" declares only `amount`. The events below also carry `note` and `qty`, + // which no event type declares. + sink.declare_event_type(OCELType { + name: "pay".into(), + attributes: vec![OCELTypeAttribute { + name: "amount".into(), + value_type: "float".into(), + }], + }) + .unwrap(); + sink.append_event( + "e1".into(), + "pay", + t, + vec![ + OCELEventAttribute { + name: "amount".into(), + value: OCELAttributeValue::Float(9.5), + }, + OCELEventAttribute { + name: "note".into(), + value: OCELAttributeValue::String("hello".into()), + }, + ], + vec![], + ) + .unwrap(); + // A second event adds another new name and re-uses `note` with a non-string + // value: VARCHAR holds both. + sink.append_event( + "e2".into(), + "ship", + t, + vec![ + OCELEventAttribute { + name: "note".into(), + value: OCELAttributeValue::Integer(7), + }, + OCELEventAttribute { + name: "qty".into(), + value: OCELAttributeValue::Integer(3), + }, + ], + vec![], + ) + .unwrap(); + sink.finalize().unwrap(); + } + + let note1: String = con + .query_row(r#"SELECT "note" FROM events WHERE id = 'e1'"#, [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(note1, "hello"); + let (note2, qty2): (String, String) = con + .query_row( + r#"SELECT "note", "qty" FROM events WHERE id = 'e2'"#, + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!((note2.as_str(), qty2.as_str()), ("7", "3")); + // The declared column keeps its declared type, and e2 left it unset. + let amount1: f64 = con + .query_row(r#"SELECT "amount" FROM events WHERE id = 'e1'"#, [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(amount1, 9.5); + let qty1: Option = con + .query_row(r#"SELECT "qty" FROM events WHERE id = 'e1'"#, [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(qty1, None); + + // Each on-the-fly column is recorded in `event_attr_meta` once per event type, so + // the reader re-declares it and `generate_type_views` projects it. + let mut meta: Vec<(String, String)> = con + .prepare("SELECT event_type, attr_name FROM event_attr_meta ORDER BY 1, 2") + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + meta.sort(); + assert_eq!( + meta, + vec![ + ("pay".to_string(), "amount".to_string()), + ("pay".to_string(), "note".to_string()), + ("ship".to_string(), "note".to_string()), + ("ship".to_string(), "qty".to_string()), + ] + ); + } + + #[test] + fn duplicate_event_id_surfaces_as_error() { + let con = Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + let mut sink = DuckDbOcelSink::new(&con).unwrap(); + let t = chrono::Utc::now().fixed_offset(); + sink.append_event("dup".into(), "pay", t, vec![], vec![]) + .unwrap(); + // Whether the PRIMARY KEY violation surfaces at append or at finalize depends on + // duckdb's appender buffering, but it must not be silently dropped. + let append_result = sink.append_event("dup".into(), "pay", t, vec![], vec![]); + let result = append_result.and_then(|()| sink.finalize()); + assert!( + result.is_err(), + "duplicate id must surface as an error, not succeed silently" + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sqlite_source.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sqlite_source.rs new file mode 100644 index 00000000..31449ffc --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/sqlite_source.rs @@ -0,0 +1,54 @@ +//! `.sqlite` source -> `DuckDB`: materialize a full OCEL, then feed it to the streaming sink. +#![cfg(all(feature = "ocel-duckdb", feature = "ocel-sqlite"))] + +use std::path::Path; + +use crate::core::event_data::object_centric::io::OCELIOError; +use crate::core::event_data::object_centric::ocel_sql::import_ocel_sqlite_from_path; + +use super::stream::{write_ocel_to_duckdb_with, DuckDbImportOptions}; + +/// Load a `.sqlite` OCEL into `DuckDB`, reached via +/// [`stream_ocel_file_to_duckdb`](super::stream::stream_ocel_file_to_duckdb). +/// +/// TODO: Currently reuses `import_ocel_sqlite_from_path` (whole-file read). +/// Streaming from `SQLite` to `DuckDB` is future work. +pub(super) fn stream_ocel_sqlite_to_duckdb, Q: AsRef>( + sqlite_path: P, + db_path: Q, + options: &DuckDbImportOptions, +) -> Result<(), OCELIOError> { + let ocel = import_ocel_sqlite_from_path(sqlite_path)?; + write_ocel_to_duckdb_with(&ocel, db_path, options) +} + +#[cfg(test)] +mod tests { + use crate::core::event_data::object_centric::ocel_sql::{ + import_ocel_sqlite_from_path, stream_ocel_file_to_duckdb, + }; + use crate::test_utils::get_test_data_path; + use duckdb::Connection; + + #[test] + fn sqlite_stream_roundtrip_counts() { + let src = get_test_data_path() + .join("ocel") + .join("order-management.sqlite"); + let reference = import_ocel_sqlite_from_path(&src).unwrap(); + let out = get_test_data_path() + .join("export") + .join("stream-from-sqlite.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + assert_eq!(n_ob as usize, reference.objects.len()); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/stream.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/stream.rs new file mode 100644 index 00000000..20607f18 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/stream.rs @@ -0,0 +1,467 @@ +//! Public entry points: own the connection, create schema, import inside one transaction, finalize. +use std::fs::File; +use std::path::Path; + +use duckdb::Connection; + +use crate::core::event_data::object_centric::appendable::{AppendableOCEL, StreamImportOCEL}; +use crate::core::event_data::object_centric::io::OCELIOError; +use crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL; +use crate::core::event_data::object_centric::ocel_struct::OCEL; +use crate::core::event_data::object_centric::ocel_xml::OCELImportOptions; +use crate::core::event_data::object_centric::readable::ReadableOCEL; +use crate::core::io::infer_format_from_path; +use macros_process_mining::register_binding; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::sink::DuckDbOcelSink; +use super::tables::{create_indexes, create_schema}; + +fn open_fresh(db_path: &Path) -> Result { + if db_path.exists() { + let _ = std::fs::remove_file(db_path); + } + Ok(Connection::open(db_path)?) +} + +/// Options controlling how an OCEL is streamed into a `DuckDB` database. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct DuckDbImportOptions { + /// Whether `DuckDB` compresses columns. `true` (default) = smaller, faster-to-query + /// file; `false` = ~20% faster import, larger file. + pub compression: bool, + /// Whether to rewrite tables clustered by key after import. `true` (default) reorders + /// rows (events by type+time, relations by key) so similar values group, giving better + /// compression and range scans. Drops then rebuilds indexes/PKs. `false` skips it. + pub optimize_filesize: bool, +} + +impl Default for DuckDbImportOptions { + fn default() -> Self { + Self { + compression: true, + optimize_filesize: true, + } + } +} + +/// Run `import` (which appends into the sink) inside one transaction, then finalize. +pub(super) fn run_import( + db_path: &Path, + options: &DuckDbImportOptions, + import: F, +) -> Result<(), OCELIOError> +where + F: FnOnce(&mut DuckDbOcelSink<'_>) -> Result<(), OCELIOError>, +{ + let con = open_fresh(db_path)?; + if !options.compression { + con.execute_batch("PRAGMA force_compression='uncompressed';")?; + } + create_schema(&con)?; + con.execute_batch("BEGIN TRANSACTION")?; + { + let mut sink = DuckDbOcelSink::new(&con)?; + import(&mut sink)?; + sink.finalize()?; + } + if options.optimize_filesize { + // Rewrite tables clustered by key for better compression + scan locality. The + // `CREATE TABLE AS SELECT` rewrite drops indexes and primary keys; both are + // rebuilt below (PKs here, secondary indexes in `create_indexes`). + con.execute_batch(" + -- events: by type, then time + CREATE TABLE events_new AS SELECT * FROM events ORDER BY ocel_type, time; + DROP TABLE events; + ALTER TABLE events_new RENAME TO events; + ALTER TABLE events ADD PRIMARY KEY (id); + + -- e2o: by event, then qualifier + CREATE TABLE e2o_new AS SELECT * FROM e2o ORDER BY event_id, qualifier; + DROP TABLE e2o; + ALTER TABLE e2o_new RENAME TO e2o; + + -- objects: by type + CREATE TABLE objects_new AS SELECT * FROM objects ORDER BY ocel_type; + DROP TABLE objects; + ALTER TABLE objects_new RENAME TO objects; + ALTER TABLE objects ADD PRIMARY KEY (id); + + -- object attribute changes (EAV): by attribute name, then value type + CREATE TABLE oac_new AS SELECT * FROM object_attribute_changes ORDER BY name, value_type; + DROP TABLE object_attribute_changes; + ALTER TABLE oac_new RENAME TO object_attribute_changes; + + -- o2o: by qualifier + CREATE TABLE o2o_new AS SELECT * FROM o2o ORDER BY qualifier; + DROP TABLE o2o; + ALTER TABLE o2o_new RENAME TO o2o; + ")?; + } + // Build secondary indexes last, so the optimize rewrite above cannot drop them. + create_indexes(&con)?; + con.execute_batch("COMMIT")?; + con.execute_batch("CHECKPOINT")?; + con.execute_batch("VACUUM")?; + Ok(()) +} + +/// Write an in-memory OCEL into a fresh `DuckDB` database in the consolidated schema. +/// +/// For writing the OCEL 2.0 standard per-type layout instead, see [`export_ocel_duckdb_to_path`](crate::core::event_data::object_centric::ocel_sql::export_ocel_duckdb_to_path). +pub fn write_ocel_to_duckdb_with( + ocel: &O, + db_path: impl AsRef, + options: &DuckDbImportOptions, +) -> Result<(), OCELIOError> { + run_import(db_path.as_ref(), options, |sink| { + for et in ocel.event_types() { + sink.declare_event_type(et.clone())?; + } + for ot in ocel.object_types() { + sink.declare_object_type(ot.clone())?; + } + for e in ocel.iter_events() { + let e = e.into_owned(); + sink.append_event(e.id, &e.event_type, e.time, e.attributes, e.relationships)?; + } + for o in ocel.iter_objects() { + let o = o.into_owned(); + sink.append_object(o.id, &o.object_type, o.attributes, o.relationships)?; + } + Ok(()) + }) +} + +/// Write an in-memory OCEL into a fresh `DuckDB` database in the consolidated schema. +/// +/// Like [`write_ocel_to_duckdb_with`] with default [`DuckDbImportOptions`]. +pub fn write_ocel_to_duckdb( + ocel: &O, + db_path: impl AsRef, +) -> Result<(), OCELIOError> { + write_ocel_to_duckdb_with(ocel, db_path, &DuckDbImportOptions::default()) +} + +/// Write an in-memory OCEL into a fresh `DuckDB` database in the consolidated schema. +#[register_binding(name = "write_ocel_to_consolidated_duckdb", stringify_error)] +fn write_ocel_to_duckdb_binding( + ocel: &OCEL, + db_path: impl AsRef, + #[bind(default = Default::default())] options: &DuckDbImportOptions, +) -> Result<(), OCELIOError> { + write_ocel_to_duckdb_with(ocel, db_path, options) +} + +/// Write an in-memory Slim OCEL into a fresh `DuckDB` database in the consolidated schema. +#[register_binding(name = "write_slim_ocel_to_consolidated_duckdb", stringify_error)] +fn write_slim_ocel_to_duckdb_binding( + ocel: &SlimLinkedOCEL, + db_path: impl AsRef, + #[bind(default = Default::default())] options: &DuckDbImportOptions, +) -> Result<(), OCELIOError> { + write_ocel_to_duckdb_with(ocel, db_path, options) +} + +/// Stream an OCEL file into a fresh `DuckDB` database, dispatching by extension: `.json`, +/// `.xml`, `.sqlite`/`.db`/`.sqlite3` (and `.gz` variants). +/// +/// Read the result back with +/// [`read_ocel_from_duckdb`](super::reader::read_ocel_from_duckdb) or +/// [`SlimLinkedOCEL::from_duckdb`](crate::core::event_data::object_centric::linked_ocel::SlimLinkedOCEL::from_duckdb). +/// +/// # Timestamps +/// +/// Stored as `TIMESTAMPTZ` holding the UTC instant. The source's UTC offset is not +/// preserved: `2023-01-01T10:00:00+02:00` reads back as `2023-01-01T08:00:00+00:00`. +/// +/// # Event-attribute columns +/// +/// Columns on the wide `events` table come from the event-type declarations, and are typed +/// accordingly. An attribute that is never declared still gets a column, added on the fly +/// as `VARCHAR`; no attribute is dropped. +pub fn stream_ocel_file_to_duckdb, Q: AsRef>( + src_path: P, + db_path: Q, +) -> Result<(), OCELIOError> { + stream_ocel_file_to_duckdb_with(src_path, db_path, &DuckDbImportOptions::default()) +} + +/// Like [`stream_ocel_file_to_duckdb`] with explicit [`DuckDbImportOptions`]. +#[register_binding(stringify_error, name = "stream_ocel_to_duckdb")] +pub fn stream_ocel_file_to_duckdb_with( + src_path: impl AsRef, + db_path: impl AsRef, + #[bind(default = Default::default())] options: &DuckDbImportOptions, +) -> Result<(), OCELIOError> { + let src = src_path.as_ref(); + let db_path = db_path.as_ref(); + let format = infer_format_from_path(src).ok_or_else(|| { + OCELIOError::UnsupportedFormat(format!("cannot infer OCEL format from {src:?}")) + })?; + match format.as_str() { + // SQLite is materialized to a full OCEL then fed to the sink (see `sqlite_source`), + // not truly streamed. + "sqlite" | "db" | "sqlite3" => { + #[cfg(feature = "ocel-sqlite")] + { + super::sqlite_source::stream_ocel_sqlite_to_duckdb(src, db_path, options) + } + #[cfg(not(feature = "ocel-sqlite"))] + { + Err(OCELIOError::Other( + "ocel-sqlite feature required for .sqlite source".into(), + )) + } + } + // Everything else (`json`/`xml` and their `.gz` variants) streams into the sink; + // `stream_ocel_from_reader` validates the format and rejects unsupported ones. + _ => run_import(db_path, options, |sink| { + sink.stream_ocel_from_reader(File::open(src)?, &format, OCELImportOptions::default()) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::ocel_json::import_ocel_json_path; + use crate::test_utils::get_test_data_path; + + #[test] + fn json_stream_roundtrip_counts() { + let src = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let reference = import_ocel_json_path(&src).unwrap(); + + let out = get_test_data_path() + .join("export") + .join("stream-order-mgmt.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + let n_types: i64 = con + .query_row("SELECT count(DISTINCT ocel_type) FROM events", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + assert_eq!(n_ob as usize, reference.objects.len()); + assert_eq!(n_types as usize, reference.event_types.len()); + } + + #[test] + fn uncompressed_option_roundtrips() { + let src = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let reference = import_ocel_json_path(&src).unwrap(); + let out = get_test_data_path() + .join("export") + .join("stream-uncompressed.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb_with( + &src, + &out, + &DuckDbImportOptions { + compression: false, + ..Default::default() + }, + ) + .unwrap(); + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + } + + #[test] + fn gz_stream_roundtrip_counts() { + use std::io::Write; + // Proves the `.gz` path: gzip a JSON fixture, then stream the `.json.gz` in. Format + // (incl. the `.gz` layer) is resolved centrally via `StreamImportOCEL`. + let src = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let reference = import_ocel_json_path(&src).unwrap(); + let raw = std::fs::read(&src).unwrap(); + + let gz_path = get_test_data_path() + .join("export") + .join("order-management.json.gz"); + { + let f = std::fs::File::create(&gz_path).unwrap(); + let mut enc = flate2::write::GzEncoder::new(f, flate2::Compression::default()); + enc.write_all(&raw).unwrap(); + enc.finish().unwrap(); + } + + let out = get_test_data_path().join("export").join("stream-gz.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&gz_path, &out).unwrap(); + + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + assert_eq!(n_ob as usize, reference.objects.len()); + } + + #[test] + fn default_import_has_indexes_and_pks() { + // The default import runs the optimize_filesize rewrite, which drops indexes and + // PKs; run_import must rebuild all of them. Guards the D1 regression. + let src = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let out = get_test_data_path() + .join("export") + .join("stream-index-check.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = Connection::open(&out).unwrap(); + + let mut idx: Vec = con + .prepare("SELECT index_name FROM duckdb_indexes()") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + idx.sort(); + for expected in [ + "e2o_event", + "e2o_object", + "o2o_source", + "o2o_target", + "object_attribute_changes_id", + ] { + assert!( + idx.iter().any(|i| i == expected), + "missing index {expected}; have {idx:?}" + ); + } + + let n_pk: i64 = con + .query_row( + "SELECT count(*) FROM duckdb_constraints() \ + WHERE constraint_type = 'PRIMARY KEY' AND table_name IN ('events','objects')", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n_pk, 2, "expected PKs on events + objects"); + } + + #[test] + fn xml_stream_roundtrip_counts() { + use crate::core::event_data::object_centric::ocel_xml::xml_ocel_import::import_ocel_xml_path; + + let src = get_test_data_path() + .join("ocel") + .join("order-management.xml"); + let reference = import_ocel_xml_path(&src).unwrap(); + + let out = get_test_data_path() + .join("export") + .join("stream-order-mgmt-xml.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + assert_eq!(n_ob as usize, reference.objects.len()); + } + + #[test] + fn xml_stream_roundtrip_counts_ck() { + use crate::core::event_data::object_centric::ocel_xml::xml_ocel_import::import_ocel_xml_path; + + let src = get_test_data_path() + .join("ocel") + .join("ContainerLogistics.xml"); + let reference = import_ocel_xml_path(&src).unwrap(); + + let out = get_test_data_path() + .join("export") + .join("stream-container-logistics-xml.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + + let con = Connection::open(&out).unwrap(); + let n_ev: i64 = con + .query_row("SELECT count(*) FROM events", [], |r| r.get(0)) + .unwrap(); + let n_ob: i64 = con + .query_row("SELECT count(*) FROM objects", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n_ev as usize, reference.events.len()); + assert_eq!(n_ob as usize, reference.objects.len()); + } + + #[test] + fn json_stream_attribute_and_relationship_fidelity() { + use super::super::tables::quote_ident; + use super::super::value::duck_value_to_ocel; + + // order-management.json has zero events with any attributes (verified against + // the fixture); ocel2-p2p.json has events with both attributes and + // relationships, so it exercises the round-trip this test targets. + let src = get_test_data_path().join("ocel").join("ocel2-p2p.json"); + let reference = import_ocel_json_path(&src).unwrap(); + let out = get_test_data_path() + .join("export") + .join("stream-fidelity.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = Connection::open(&out).unwrap(); + + // Pick one event that has at least one attribute and one relationship. + let ev = reference + .events + .iter() + .find(|e| !e.attributes.is_empty() && !e.relationships.is_empty()) + .expect("an event with attrs + rels"); + + // Attribute value round-trips via the typed wide column. + let a = &ev.attributes[0]; + let value: duckdb::types::Value = con + .query_row( + &format!("SELECT {} FROM events WHERE id = ?", quote_ident(&a.name)), + duckdb::params![ev.id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(duck_value_to_ocel(value), a.value); + + // Relationship count matches. + let n_rel: i64 = con + .query_row( + "SELECT count(*) FROM e2o WHERE event_id = ?", + duckdb::params![ev.id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n_rel as usize, ev.relationships.len()); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/tables.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/tables.rs new file mode 100644 index 00000000..73adab8d --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/tables.rs @@ -0,0 +1,209 @@ +//! Table/index schema (`CREATE` statements) and shared table/column name constants. +use std::borrow::Cow; + +use duckdb::Connection; + +use crate::core::event_data::object_centric::ocel_struct::OCELAttributeType; + +pub(crate) const T_EVENTS: &str = "events"; +pub(crate) const T_OBJECTS: &str = "objects"; +pub(crate) const T_EVENT_ATTR_META: &str = "event_attr_meta"; +pub(crate) const T_OBJECT_ATTR_META: &str = "object_attr_meta"; +pub(crate) const T_OBJECT_ATTR_CHANGES: &str = "object_attribute_changes"; +pub(crate) const T_E2O: &str = "e2o"; +pub(crate) const T_O2O: &str = "o2o"; + +/// The fixed columns of the wide `events` table, which no attribute column may reuse. +pub(crate) const EVENT_BASE_COLS: [&str; 3] = ["id", "ocel_type", "time"]; + +// Everything except `events`, which `build_events_table` builds once its attribute columns are known. +const CREATE_TABLES: &str = r#" +CREATE TABLE objects (id TEXT PRIMARY KEY, ocel_type TEXT); +CREATE TABLE object_attribute_changes (id TEXT, name TEXT, "time" TIMESTAMPTZ, value VARCHAR, value_type TEXT); +CREATE TABLE event_attr_meta (event_type TEXT, attr_name TEXT, attr_type TEXT); +CREATE TABLE object_attr_meta (object_type TEXT, attr_name TEXT, attr_type TEXT); +CREATE TABLE e2o (event_id TEXT, object_id TEXT, qualifier TEXT); +CREATE TABLE o2o (source_id TEXT, target_id TEXT, qualifier TEXT); +"#; + +const INDEXES: &str = r#" +CREATE INDEX object_attribute_changes_id ON object_attribute_changes(id); +CREATE INDEX e2o_event ON e2o(event_id); +CREATE INDEX e2o_object ON e2o(object_id); +CREATE INDEX o2o_source ON o2o(source_id); +CREATE INDEX o2o_target ON o2o(target_id); +"#; + +pub(crate) fn create_schema(con: &Connection) -> Result<(), duckdb::Error> { + con.execute_batch(CREATE_TABLES) +} + +pub(crate) fn create_indexes(con: &Connection) -> Result<(), duckdb::Error> { + con.execute_batch(INDEXES) +} + +/// SQL-quote an identifier by doubling embedded double-quotes. +pub(crate) fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// `DuckDB` column type for an event-attribute [`OCELAttributeType`]. +pub(crate) fn event_attr_sql_type(t: OCELAttributeType) -> &'static str { + match t { + OCELAttributeType::Integer => "BIGINT", + OCELAttributeType::Float => "DOUBLE", + OCELAttributeType::Boolean => "BOOLEAN", + OCELAttributeType::Time => "TIMESTAMPTZ", + OCELAttributeType::String | OCELAttributeType::Null => "VARCHAR", + } +} + +/// Whether `name` clashes with a `base_cols` entry, directly or as an already-suffixed form of +/// one, which keeps the `_attr` suffixing reversible. +fn collides_with_base_col(name: &str, base_cols: &[&str]) -> bool { + base_cols.contains(&name.trim_end_matches("_attr")) +} + +/// Suffix `name` with `_attr` if it collides with `base_cols`. Because the collision check +/// strips all trailing `_attr` repeats before comparing, this single pass is already +/// collision-free: e.g. `id` and `id_attr` map to `id_attr` and `id_attr_attr` respectively. +fn suffixed_column<'a>(name: &'a str, base_cols: &[&str]) -> Cow<'a, str> { + if collides_with_base_col(name, base_cols) { + Cow::Owned(format!("{name}_attr")) + } else { + Cow::Borrowed(name) + } +} + +/// The `events` column an event attribute is stored in. `id`/`ocel_type`/`time` belong to the +/// table itself, so an attribute using one of those names takes an `_attr` suffix instead. +pub(crate) fn event_attr_column(name: &str) -> Cow<'_, str> { + suffixed_column(name, &EVENT_BASE_COLS) +} + +/// Inverse of [`event_attr_column`]: the attribute name an `events` column holds. +pub(crate) fn event_attr_name(column: &str) -> Cow<'_, str> { + match column.strip_suffix("_attr") { + Some(name) if collides_with_base_col(name, &EVENT_BASE_COLS) => Cow::Borrowed(name), + _ => Cow::Borrowed(column), + } +} + +/// The fixed columns of an `object_` wide view. +pub(crate) const OBJECT_VIEW_BASE_COLS: [&str; 1] = ["id"]; + +/// The alias an object attribute is pivoted into in an `object_` view. Mirrors +/// [`event_attr_column`]'s collision handling, but against [`OBJECT_VIEW_BASE_COLS`]. +pub(crate) fn object_attr_column(name: &str) -> Cow<'_, str> { + suffixed_column(name, &OBJECT_VIEW_BASE_COLS) +} + +/// Add a column for an event attribute that arrived after the wide schema was fixed. +pub(crate) fn add_events_column(name: &str, ty: OCELAttributeType) -> String { + format!( + "ALTER TABLE {T_EVENTS} ADD COLUMN {} {}", + quote_ident(&event_attr_column(name)), + event_attr_sql_type(ty) + ) +} + +/// Build the `CREATE TABLE events` statement: fixed columns plus one typed column per `cols` entry. +pub(crate) fn build_events_table(cols: &[(String, OCELAttributeType)]) -> String { + let mut sql = String::from( + r#"CREATE TABLE events (id TEXT PRIMARY KEY, ocel_type TEXT, "time" TIMESTAMPTZ"#, + ); + for (name, ty) in cols { + sql.push_str(", "); + sql.push_str("e_ident(&event_attr_column(name))); + sql.push(' '); + sql.push_str(event_attr_sql_type(*ty)); + } + sql.push(')'); + sql +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_schema_makes_all_static_tables() { + // `events` is absent here: the sink creates it once the event-attribute schema is known. + let con = duckdb::Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + let mut stmt = con + .prepare("SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_name") + .unwrap(); + let names: Vec = stmt + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + assert_eq!( + names, + vec![ + "e2o", + "event_attr_meta", + "o2o", + "object_attr_meta", + "object_attribute_changes", + "objects", + ] + ); + } + + #[test] + fn attribute_columns_never_collide_with_base_columns() { + let sql = build_events_table(&[ + ("time".to_string(), OCELAttributeType::Time), + ("id".to_string(), OCELAttributeType::String), + ("ocel_type".to_string(), OCELAttributeType::String), + ("time_attr".to_string(), OCELAttributeType::String), + ]); + let con = duckdb::Connection::open_in_memory().unwrap(); + con.execute_batch(&sql).unwrap(); + for name in ["time", "id", "ocel_type", "time_attr"] { + assert_eq!(event_attr_name(&event_attr_column(name)), name); + } + } + + #[test] + fn build_events_table_quotes_and_types_columns() { + let sql = build_events_table(&[ + ("amount".to_string(), OCELAttributeType::Float), + ("weird \"name\"".to_string(), OCELAttributeType::String), + ]); + assert!(sql.contains(r#"id TEXT PRIMARY KEY, ocel_type TEXT, "time" TIMESTAMPTZ"#)); + assert!(sql.contains(r#""amount" DOUBLE"#)); + assert!(sql.contains(r#""weird ""name""" VARCHAR"#)); + let con = duckdb::Connection::open_in_memory().unwrap(); + con.execute_batch(&sql).unwrap(); + } + + #[test] + fn timestamps_are_utc_anchored() { + // TIMESTAMPTZ, not a bare TIMESTAMP, so stored instants are unambiguous. + let con = duckdb::Connection::open_in_memory().unwrap(); + create_schema(&con).unwrap(); + con.execute_batch(&build_events_table(&[( + "at".to_string(), + OCELAttributeType::Time, + )])) + .unwrap(); + for (table, column) in [ + ("events", "time"), + ("events", "at"), + ("object_attribute_changes", "time"), + ] { + let ty: String = con + .query_row( + "SELECT data_type FROM information_schema.columns \ + WHERE table_name = ? AND column_name = ?", + [table, column], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(ty, "TIMESTAMP WITH TIME ZONE", "{table}.{column}"); + } + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/value.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/value.rs new file mode 100644 index 00000000..7e12af43 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/value.rs @@ -0,0 +1,204 @@ +//! Value (de)serialization: object attributes use the EAV string format +//! (`to_sql_value`/`from_sql_value`), event attributes the typed `duckdb::types::Value` converters. +use std::borrow::Cow; + +use chrono::{DateTime, FixedOffset}; +use duckdb::types::{TimeUnit, Value as DuckValue}; + +use crate::core::event_data::object_centric::ocel_struct::{OCELAttributeType, OCELAttributeValue}; +use crate::core::event_data::timestamp_utils::parse_timestamp; + +/// Convert a `DuckDB` timestamp `(unit, count)` to a [`DateTime`] at `+00:00`, or +/// `None` if the instant is out of `chrono`'s range. The original offset is not recoverable. +pub(crate) fn duck_timestamp_to_datetime(tu: TimeUnit, t: i64) -> Option> { + // Euclidean division so pre-epoch counts keep a non-negative sub-second remainder + // instead of truncating toward zero. + let (secs, nsecs) = match tu { + TimeUnit::Second => (t, 0), + TimeUnit::Millisecond => (t.div_euclid(1_000), t.rem_euclid(1_000) * 1_000_000), + TimeUnit::Microsecond => (t.div_euclid(1_000_000), t.rem_euclid(1_000_000) * 1_000), + TimeUnit::Nanosecond => (t.div_euclid(1_000_000_000), t.rem_euclid(1_000_000_000)), + }; + Some(DateTime::from_timestamp(secs, nsecs as u32)?.fixed_offset()) +} + +/// Store a timestamp as UTC microseconds for the `TIMESTAMPTZ` columns. `TIMESTAMPTZ` holds only +/// an instant, so the source offset is lost: `10:00:00+02:00` reads back as `08:00:00Z`. +pub(crate) fn datetime_to_duck_timestamp(t: DateTime) -> DuckValue { + DuckValue::Timestamp(TimeUnit::Microsecond, t.to_utc().timestamp_micros()) +} + +/// Convert an [`OCELAttributeValue`] to the `duckdb::types::Value` matching the target column's +/// [`OCELAttributeType`]. On mismatch the value is parsed from its `Display` form, or `Null`. +pub(crate) fn ocel_value_to_duck(v: &OCELAttributeValue, target: OCELAttributeType) -> DuckValue { + match target { + OCELAttributeType::String | OCELAttributeType::Null => match v { + OCELAttributeValue::Null => DuckValue::Null, + OCELAttributeValue::String(s) => DuckValue::Text(s.clone()), + other => DuckValue::Text(other.to_string()), + }, + OCELAttributeType::Integer => match v { + OCELAttributeValue::Integer(i) => DuckValue::BigInt(*i), + OCELAttributeValue::Float(f) => DuckValue::BigInt(*f as i64), + OCELAttributeValue::Null => DuckValue::Null, + other => other + .to_string() + .parse::() + .map(DuckValue::BigInt) + .unwrap_or(DuckValue::Null), + }, + OCELAttributeType::Float => match v { + OCELAttributeValue::Float(f) => DuckValue::Double(*f), + OCELAttributeValue::Integer(i) => DuckValue::Double(*i as f64), + OCELAttributeValue::Null => DuckValue::Null, + other => other + .to_string() + .parse::() + .map(DuckValue::Double) + .unwrap_or(DuckValue::Null), + }, + OCELAttributeType::Boolean => match v { + OCELAttributeValue::Boolean(b) => DuckValue::Boolean(*b), + OCELAttributeValue::Null => DuckValue::Null, + other => other + .to_string() + .parse::() + .map(DuckValue::Boolean) + .unwrap_or(DuckValue::Null), + }, + OCELAttributeType::Time => match v { + OCELAttributeValue::Time(t) => datetime_to_duck_timestamp(*t), + OCELAttributeValue::Null => DuckValue::Null, + other => parse_timestamp(&other.to_string(), None, false) + .map(datetime_to_duck_timestamp) + .unwrap_or(DuckValue::Null), + }, + } +} + +/// Reconstruct an [`OCELAttributeValue`] from a typed wide-column `duckdb::types::Value`. +pub(crate) fn duck_value_to_ocel(v: DuckValue) -> OCELAttributeValue { + match v { + DuckValue::Null => OCELAttributeValue::Null, + DuckValue::Boolean(b) => OCELAttributeValue::Boolean(b), + DuckValue::TinyInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::SmallInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::Int(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::BigInt(i) => OCELAttributeValue::Integer(i), + DuckValue::HugeInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::UTinyInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::USmallInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::UInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::UBigInt(i) => OCELAttributeValue::Integer(i as i64), + DuckValue::Float(f) => OCELAttributeValue::Float(f as f64), + DuckValue::Double(f) => OCELAttributeValue::Float(f), + DuckValue::Text(s) => OCELAttributeValue::String(s), + DuckValue::Timestamp(tu, t) => duck_timestamp_to_datetime(tu, t) + .map_or(OCELAttributeValue::Null, OCELAttributeValue::Time), + _ => OCELAttributeValue::Null, + } +} + +/// Serialize a value to its `(value, value_type)` pair for the EAV columns. +pub(crate) fn to_sql_value(v: &OCELAttributeValue) -> (Cow<'_, str>, &'static str) { + let value_type = v.get_type().as_type_str(); + match v { + OCELAttributeValue::String(s) => (Cow::Borrowed(s.as_str()), value_type), + other => (Cow::Owned(other.to_string()), value_type), + } +} + +/// Reconstruct a value from its stored `(value, value_type)`. Unparseable values become `Null`. +pub(crate) fn from_sql_value(value: &str, value_type: &str) -> OCELAttributeValue { + match OCELAttributeType::from_type_str(value_type) { + OCELAttributeType::String => OCELAttributeValue::String(value.to_owned()), + OCELAttributeType::Integer => value + .parse::() + .map(OCELAttributeValue::Integer) + .unwrap_or(OCELAttributeValue::Null), + OCELAttributeType::Float => value + .parse::() + .map(OCELAttributeValue::Float) + .unwrap_or(OCELAttributeValue::Null), + OCELAttributeType::Boolean => value + .parse::() + .map(OCELAttributeValue::Boolean) + .unwrap_or(OCELAttributeValue::Null), + OCELAttributeType::Time => parse_timestamp(value, None, false) + .map(OCELAttributeValue::Time) + .unwrap_or(OCELAttributeValue::Null), + OCELAttributeType::Null => OCELAttributeValue::Null, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::DateTime; + + fn roundtrip(v: OCELAttributeValue) -> OCELAttributeValue { + let (s, t) = to_sql_value(&v); + from_sql_value(s.as_ref(), t) + } + + #[test] + fn roundtrip_scalars() { + assert_eq!( + roundtrip(OCELAttributeValue::Integer(42)), + OCELAttributeValue::Integer(42) + ); + assert_eq!( + roundtrip(OCELAttributeValue::Float(3.5)), + OCELAttributeValue::Float(3.5) + ); + assert_eq!( + roundtrip(OCELAttributeValue::Boolean(true)), + OCELAttributeValue::Boolean(true) + ); + assert_eq!( + roundtrip(OCELAttributeValue::String("hi".into())), + OCELAttributeValue::String("hi".into()) + ); + } + + #[test] + fn roundtrip_nan_float() { + // The XML importer turns a literal `null` float into NaN, which must stay a float. + let (s, t) = to_sql_value(&OCELAttributeValue::Float(f64::NAN)); + assert_eq!(t, "float"); + match from_sql_value(s.as_ref(), t) { + OCELAttributeValue::Float(f) => assert!(f.is_nan(), "expected NaN, got {f}"), + other => panic!("expected Float(NaN), got {other:?}"), + } + } + + #[test] + fn roundtrip_time() { + let dt = DateTime::parse_from_rfc3339("2023-10-06T09:30:21+02:00").unwrap(); + assert_eq!( + roundtrip(OCELAttributeValue::Time(dt)), + OCELAttributeValue::Time(dt) + ); + } + + #[test] + fn value_type_strings() { + assert_eq!(to_sql_value(&OCELAttributeValue::Integer(1)).1, "integer"); + assert_eq!(to_sql_value(&OCELAttributeValue::Float(1.0)).1, "float"); + assert_eq!( + to_sql_value(&OCELAttributeValue::Time(chrono::Utc::now().fixed_offset())).1, + "time" + ); + } + + #[test] + fn null_roundtrips_to_empty_string_documented_caveat() { + let (s, t) = to_sql_value(&OCELAttributeValue::Null); + assert_eq!(s.as_ref(), ""); + assert_eq!(t, "string"); + assert_eq!( + from_sql_value(s.as_ref(), t), + OCELAttributeValue::String(String::new()) + ); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/views.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/views.rs new file mode 100644 index 00000000..6712ce62 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/schema/views.rs @@ -0,0 +1,306 @@ +//! On-demand generation of per-type wide views. +use duckdb::Connection; + +use super::tables::{ + event_attr_column, object_attr_column, quote_ident, T_EVENTS, T_EVENT_ATTR_META, T_OBJECTS, + T_OBJECT_ATTR_CHANGES, +}; + +/// `DuckDB` cast target for an OCEL `value_type` string. +fn cast_type(value_type: &str) -> &'static str { + match value_type { + "integer" => "BIGINT", + "float" => "DOUBLE", + "boolean" => "BOOLEAN", + "time" => "TIMESTAMPTZ", + _ => "VARCHAR", + } +} + +fn distinct_types(con: &Connection, entity_table: &str) -> Result, duckdb::Error> { + let mut stmt = con.prepare(&format!( + "SELECT DISTINCT ocel_type FROM {entity_table} ORDER BY ocel_type" + ))?; + let rows = stmt.query_map([], |r| r.get::<_, String>(0))?; + rows.collect() +} + +/// (`attr_name`, `cast_type`) for one object type. If a name has >1 `value_type`, coerce to VARCHAR. +fn object_attrs_for_type( + con: &Connection, + ocel_type: &str, +) -> Result, duckdb::Error> { + let sql = format!( + "SELECT a.name, count(DISTINCT a.value_type) AS n, any_value(a.value_type) AS vt \ + FROM {T_OBJECT_ATTR_CHANGES} a JOIN {T_OBJECTS} e ON a.id = e.id \ + WHERE e.ocel_type = ? GROUP BY a.name ORDER BY a.name" + ); + let mut stmt = con.prepare(&sql)?; + let rows = stmt.query_map([ocel_type], |r| { + let name: String = r.get(0)?; + let n: i64 = r.get(1)?; + let vt: String = r.get(2)?; + Ok((name, n, vt)) + })?; + let mut out = Vec::new(); + for row in rows { + let (name, n, vt) = row?; + out.push((name, if n > 1 { "VARCHAR" } else { cast_type(&vt) })); + } + Ok(out) +} + +/// Declared attribute column names for one event type (from `event_attr_meta`). +fn event_attrs_for_type(con: &Connection, ocel_type: &str) -> Result, duckdb::Error> { + let mut stmt = con.prepare(&format!( + "SELECT DISTINCT attr_name FROM {T_EVENT_ATTR_META} WHERE event_type = ? ORDER BY attr_name" + ))?; + let rows = stmt.query_map([ocel_type], |r| r.get::<_, String>(0))?; + rows.collect() +} + +/// Build `event_` and `object_` wide views. +/// +/// An attribute aliased to its bare name would collide with the view's base columns +/// (`id`/`time` for events, `id` for objects), so such a name takes an `_attr` suffix instead. +pub fn generate_type_views(con: &Connection) -> Result<(), duckdb::Error> { + const BASE_COLS: &[&str] = &["id", "time"]; + + // Events: project the type's typed columns straight off the wide `events` table. + for ty in distinct_types(con, T_EVENTS)? { + let attrs = event_attrs_for_type(con, &ty)?; + let cols: String = attrs + .iter() + .map(|name| { + let alias = if BASE_COLS.contains(&name.as_str()) { + quote_ident(&format!("{name}_attr")) + } else { + quote_ident(name) + }; + format!(", {} AS {alias}", quote_ident(&event_attr_column(name))) + }) + .collect(); + let view = quote_ident(&format!("event_{ty}")); + let ty_lit = ty.replace('\'', "''"); + let sql = format!( + "CREATE OR REPLACE VIEW {view} AS \ + SELECT id, \"time\"{cols} FROM {T_EVENTS} WHERE ocel_type = '{ty_lit}'" + ); + con.execute_batch(&sql)?; + } + + // Objects: pivot the EAV `object_attribute_changes` table into wide columns. + for ty in distinct_types(con, T_OBJECTS)? { + let attrs = object_attrs_for_type(con, &ty)?; + let cols: String = attrs + .iter() + .map(|(name, cast)| { + let n = name.replace('\'', "''"); + let alias = quote_ident(&object_attr_column(name)); + format!( + ", CAST(any_value(a.value) FILTER (WHERE a.name = '{n}') AS {cast}) AS {alias}" + ) + }) + .collect(); + let view = quote_ident(&format!("object_{ty}")); + let ty_lit = ty.replace('\'', "''"); + let sql = format!( + "CREATE OR REPLACE VIEW {view} AS \ + SELECT e.id{cols} \ + FROM {T_OBJECTS} e LEFT JOIN {T_OBJECT_ATTR_CHANGES} a ON a.id = e.id \ + WHERE e.ocel_type = '{ty_lit}' \ + GROUP BY e.id" + ); + con.execute_batch(&sql)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::ocel_sql::duckdb::schema::stream::stream_ocel_file_to_duckdb; + use crate::test_utils::get_test_data_path; + + #[test] + fn generates_queryable_event_views() { + let src = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let out = get_test_data_path() + .join("export") + .join("stream-views.duckdb"); + let _ = std::fs::remove_file(&out); + stream_ocel_file_to_duckdb(&src, &out).unwrap(); + let con = Connection::open(&out).unwrap(); + generate_type_views(&con).unwrap(); + + let n_views: i64 = con + .query_row( + "SELECT count(*) FROM information_schema.tables WHERE table_type = 'VIEW' AND table_name LIKE 'event_%'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(n_views >= 1, "expected at least one event_ view"); + + // The view is actually queryable, not just present in the catalog. + let view_name: String = con + .query_row( + "SELECT table_name FROM information_schema.tables WHERE table_type = 'VIEW' AND table_name LIKE 'event_%' ORDER BY table_name LIMIT 1", + [], + |r| r.get(0), + ) + .unwrap(); + con.query_row(&format!("SELECT count(*) FROM \"{view_name}\""), [], |r| { + r.get::<_, i64>(0) + }) + .unwrap(); + + // order-management objects have attributes, so this exercises a non-trivial pivot. + let object_view_name: String = con + .query_row( + "SELECT table_name FROM information_schema.tables WHERE table_type = 'VIEW' AND table_name LIKE 'object_%' ORDER BY table_name LIMIT 1", + [], + |r| r.get(0), + ) + .unwrap(); + let object_row_count: i64 = con + .query_row( + &format!("SELECT count(*) FROM \"{object_view_name}\""), + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + object_row_count > 0, + "expected {object_view_name} to return rows" + ); + } + + /// Returns (`column_name`, `data_type`) pairs for a view/table, ordered by position. + fn columns_of(con: &Connection, table_name: &str) -> Vec<(String, String)> { + let mut stmt = con + .prepare( + "SELECT column_name, data_type FROM information_schema.columns \ + WHERE table_name = ? ORDER BY ordinal_position", + ) + .unwrap(); + stmt.query_map([table_name], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap() + .map(|r| r.unwrap()) + .collect() + } + + #[test] + fn event_view_exposes_typed_attribute_column() { + let con = Connection::open_in_memory().unwrap(); + super::super::tables::create_schema(&con).unwrap(); + con.execute_batch(&super::super::tables::build_events_table(&[( + "amount".to_string(), + crate::core::event_data::object_centric::ocel_struct::OCELAttributeType::Integer, + )])) + .unwrap(); + con.execute_batch( + "INSERT INTO events VALUES ('e1', 't', TIMESTAMP '2024-01-01 00:00:00', 42); + INSERT INTO event_attr_meta VALUES ('t', 'amount', 'integer');", + ) + .unwrap(); + + generate_type_views(&con).unwrap(); + + let names: Vec = columns_of(&con, "event_t") + .into_iter() + .map(|(n, _)| n) + .collect(); + assert!(names.contains(&"id".to_string()), "cols: {names:?}"); + assert!(names.contains(&"time".to_string()), "cols: {names:?}"); + assert!(names.contains(&"amount".to_string()), "cols: {names:?}"); + + let amount: i64 = con + .query_row("SELECT amount FROM event_t WHERE id = 'e1'", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(amount, 42); + } + + #[test] + fn object_view_coerces_heterogeneous_types_to_varchar() { + let con = Connection::open_in_memory().unwrap(); + super::super::tables::create_schema(&con).unwrap(); + con.execute_batch(&super::super::tables::build_events_table(&[])) + .unwrap(); + + // Two rows give the same attribute name two different value_types. + con.execute_batch( + "INSERT INTO objects VALUES ('o1', 'o'); + INSERT INTO objects VALUES ('o2', 'o'); + INSERT INTO object_attribute_changes VALUES \ + ('o1', 'attr', TIMESTAMP '2024-01-01 00:00:00', '1', 'integer'); + INSERT INTO object_attribute_changes VALUES \ + ('o2', 'attr', TIMESTAMP '2024-01-01 00:00:00', 'foo', 'string');", + ) + .unwrap(); + + generate_type_views(&con).unwrap(); + + let object_cols = columns_of(&con, "object_o"); + let attr_type = object_cols + .iter() + .find(|(n, _)| n == "attr") + .map(|(_, t)| t.as_str()) + .unwrap_or_else(|| panic!("expected 'attr' column in object_o: {object_cols:?}")); + assert_eq!( + attr_type, "VARCHAR", + "heterogeneous value_types must coerce to VARCHAR" + ); + + let object_row_count: i64 = con + .query_row("SELECT count(*) FROM object_o", [], |r| r.get(0)) + .unwrap(); + assert_eq!(object_row_count, 2); + } + + #[test] + fn object_view_handles_id_and_id_attr_attribute_collision() { + let con = Connection::open_in_memory().unwrap(); + super::super::tables::create_schema(&con).unwrap(); + con.execute_batch(&super::super::tables::build_events_table(&[])) + .unwrap(); + + // An object with both an `id` attribute and an `id_attr` attribute must not produce + // two columns aliased to the same name. + con.execute_batch( + "INSERT INTO objects VALUES ('o1', 'o'); + INSERT INTO object_attribute_changes VALUES \ + ('o1', 'id', TIMESTAMP '2024-01-01 00:00:00', 'v1', 'string'); + INSERT INTO object_attribute_changes VALUES \ + ('o1', 'id_attr', TIMESTAMP '2024-01-01 00:00:00', 'v2', 'string');", + ) + .unwrap(); + + generate_type_views(&con).unwrap(); + + let names: Vec = columns_of(&con, "object_o") + .into_iter() + .map(|(n, _)| n) + .collect(); + assert!(names.contains(&"id".to_string()), "cols: {names:?}"); + assert!(names.contains(&"id_attr".to_string()), "cols: {names:?}"); + assert!( + names.contains(&"id_attr_attr".to_string()), + "cols: {names:?}" + ); + + let (id_attr, id_attr_attr): (String, String) = con + .query_row( + "SELECT id_attr, id_attr_attr FROM object_o WHERE id = 'o1'", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(id_attr, "v1", "the `id` attribute value"); + assert_eq!(id_attr_attr, "v2", "the `id_attr` attribute value"); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs index 0af7d3eb..e226c843 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs @@ -150,7 +150,13 @@ where fn clean_sql_name(type_name: &str) -> String { type_name .chars() - .map(|c| if c != '\'' && c != '\\' { c } else { '_' }) + .map(|c| { + if c != '\'' && c != '\\' && c != '"' { + c + } else { + '_' + } + }) .collect() } diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs index 12525a50..4108c11b 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs @@ -9,6 +9,11 @@ use chrono::DateTime; pub(crate) const OCEL_ID_COLUMN: &str = "ocel_id"; pub(crate) const OCEL_TIME_COLUMN: &str = "ocel_time"; pub(crate) const OCEL_CHANGED_FIELD: &str = "ocel_changed_field"; +/// A misspelling of [`OCEL_CHANGED_FIELD`] (missing the `d`) seen in the wild. +/// +/// Never treated as the real column: it exists only so importers can name the misspelling in the +/// diagnostic they report. +pub(crate) const OCEL_CHANGE_FIELD_MISSPELLED: &str = "ocel_change_field"; pub(crate) const IGNORED_PRAGMA_COLUMNS: [&str; 3] = [OCEL_ID_COLUMN, OCEL_TIME_COLUMN, OCEL_CHANGED_FIELD]; pub(crate) const OCEL_TYPE_MAP_COLUMN: &str = "ocel_type_map"; @@ -30,7 +35,16 @@ pub use duckdb::duckdb_ocel_export::export_ocel_duckdb_to_path; #[cfg(feature = "ocel-duckdb")] pub use duckdb::duckdb_ocel_import::import_ocel_duckdb_from_con; #[cfg(feature = "ocel-duckdb")] +pub use duckdb::duckdb_ocel_import::import_ocel_duckdb_from_con_with_options; +#[cfg(feature = "ocel-duckdb")] pub use duckdb::duckdb_ocel_import::import_ocel_duckdb_from_path; +#[cfg(feature = "ocel-duckdb")] +pub use duckdb::{ + generate_type_views, read_consolidated_ocel_from_duckdb_path, + read_consolidated_slim_ocel_from_duckdb_path, read_ocel_from_duckdb, + stream_ocel_file_to_duckdb, stream_ocel_file_to_duckdb_with, write_ocel_to_duckdb, + write_ocel_to_duckdb_with, DuckDbImportOptions, +}; #[cfg(feature = "ocel-sqlite")] pub use sqlite::sqlite_ocel_export::export_ocel_sqlite_to_path; @@ -40,12 +54,157 @@ pub use sqlite::sqlite_ocel_export::export_ocel_sqlite_to_vec; #[cfg(feature = "ocel-sqlite")] pub use sqlite::sqlite_ocel_import::import_ocel_sqlite_from_con; #[cfg(feature = "ocel-sqlite")] +pub use sqlite::sqlite_ocel_import::import_ocel_sqlite_from_con_with_options; +#[cfg(feature = "ocel-sqlite")] pub use sqlite::sqlite_ocel_import::import_ocel_sqlite_from_path; #[cfg(feature = "ocel-sqlite")] +pub use sqlite::sqlite_ocel_import::import_ocel_sqlite_from_path_with_options; +#[cfg(feature = "ocel-sqlite")] pub use sqlite::sqlite_ocel_import::import_ocel_sqlite_from_slice; use crate::core::event_data::object_centric::ocel_struct::OCELAttributeType; use crate::core::event_data::object_centric::ocel_struct::OCELType; +use crate::core::event_data::object_centric::ocel_struct::OCELTypeAttribute; + +/// Options controlling how strictly a `SQLite`/`DuckDB` OCEL 2.0 file is read. +#[derive(Debug, Clone, Copy)] +pub struct SqlOcelImportOptions { + /// When `true`, an object-type table missing its `ocel_changed_field` column is + /// tolerated: every row is read as that object's initial state, as if the column + /// existed and were `NULL` everywhere. Default: `true`. + /// + /// The OCEL 2.0 specification requires the column on every object-type table, and + /// writers in the wild leave it off types that have no attribute to change. Nothing is + /// lost by reading such a file: a table without the column records no attribute change + /// in the first place, so refusing it would only cost the caller the rest of the log. + /// Set to `false` to hold a file to the specification and reject it by name instead. + pub allow_missing_changed_field: bool, +} + +impl Default for SqlOcelImportOptions { + fn default() -> Self { + Self { + allow_missing_changed_field: true, + } + } +} + +/// A double-quoted SQL identifier, embedded quotes doubled. For table/column names that come +/// from the file being read, which nothing has sanitised. +pub(crate) fn quoted_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// A single-quoted SQL string literal, embedded quotes doubled. For the same untrusted names +/// where a literal is needed (`PRAGMA table_info('...')`). +pub(crate) fn quoted_str(name: &str) -> String { + format!("'{}'", name.replace('\'', "''")) +} + +/// Build the diagnostic for an object-type table missing its required +/// `ocel_changed_field` column, pointing out a same-shaped misspelling if one is present. +pub(crate) fn missing_changed_field_message( + db_path: Option<&str>, + ob_type: &str, + table_name: &str, + raw_column_names: &[String], +) -> String { + let where_ = db_path.map(|p| format!(" in '{p}'")).unwrap_or_default(); + let mut msg = format!( + "object type '{ob_type}' (table '{table_name}'{where_}) has no '{OCEL_CHANGED_FIELD}' \ + column. This file does not conform to the OCEL 2.0 SQLite/DuckDB specification, which \ + requires that column on every object-type table." + ); + if raw_column_names + .iter() + .any(|n| n == OCEL_CHANGE_FIELD_MISSPELLED) + { + msg.push_str(&format!( + " Note: a column named '{OCEL_CHANGE_FIELD_MISSPELLED}' (missing the 'd') is \ + present. This looks like a misspelling of '{OCEL_CHANGED_FIELD}', but rust4pm \ + does not guess at that; fix the source file to use the standard column name." + )); + } + msg +} + +/// What the columns of one object-type table mean for the import. +pub(crate) struct ObjectTablePlan { + /// The attribute columns, i.e. every column that is not OCEL-reserved. + pub attributes: Vec, + /// Query for the rows carrying an object's initial state. + pub initial_query: String, + /// Query for the rows carrying an attribute change, `None` when the table has no + /// `ocel_changed_field` column. + pub changed_query: Option, +} + +/// Interpret the `PRAGMA table_info` columns of one object-type table. +/// +/// Both SQL importers read this the same way, so a rejection comes back as a message for the +/// caller to wrap in its own error type. +/// +/// # Errors +/// The table has no `ocel_changed_field` column and `options` does not tolerate that. +pub(crate) fn plan_object_table( + db_path: Option<&str>, + ob_type: &str, + table_name: &str, + raw_ob_columns: Vec<(String, String)>, + options: SqlOcelImportOptions, +) -> Result { + let has_column = |wanted: &str| raw_ob_columns.iter().any(|(name, _)| name == wanted); + let has_changed_field = has_column(OCEL_CHANGED_FIELD); + if !has_changed_field { + if !options.allow_missing_changed_field { + let raw_ob_column_names: Vec = raw_ob_columns + .iter() + .map(|(name, _)| name.clone()) + .collect(); + return Err(missing_changed_field_message( + db_path, + ob_type, + table_name, + &raw_ob_column_names, + )); + } + if has_column(OCEL_CHANGE_FIELD_MISSPELLED) { + eprintln!( + "Warning: object type '{ob_type}' (table '{table_name}') has no \ + '{OCEL_CHANGED_FIELD}' column but does have a column named \ + '{OCEL_CHANGE_FIELD_MISSPELLED}'. This looks like a misspelling, but it is \ + imported as a regular attribute, not as change tracking. Fix the source \ + file to use '{OCEL_CHANGED_FIELD}'." + ); + } + } + let attributes: Vec = raw_ob_columns + .into_iter() + .filter(|(name, _)| !IGNORED_PRAGMA_COLUMNS.contains(&name.as_str())) + .map(|(name, atype)| OCELTypeAttribute { + name, + value_type: sql_type_to_ocel(&atype).to_type_string(), + }) + .collect(); + // Without `ocel_changed_field` there is nothing to exclude: every row is an object's + // initial (and only) state. + let table = quoted_ident(table_name); + let (initial_query, changed_query) = if has_changed_field { + ( + format!("SELECT * FROM {table} WHERE {OCEL_CHANGED_FIELD} IS NULL"), + Some(format!( + "SELECT * FROM {table} WHERE {OCEL_CHANGED_FIELD} IS NOT NULL" + )), + ) + } else { + (format!("SELECT * FROM {table}"), None) + }; + Ok(ObjectTablePlan { + attributes, + initial_query, + changed_query, + }) +} pub(crate) fn sql_type_to_ocel(s: &str) -> OCELAttributeType { match s { @@ -370,53 +529,46 @@ fn write_object_changes_sqlite( object_type: &OCELType, o: &super::ocel_struct::OCELObject, ) -> Result<(), DatabaseError> { - let initial_vals: Vec<_> = object_type - .attributes - .iter() - .map(|a| { - o.attributes - .iter() - .find(|oa| oa.name == a.name && oa.time == DateTime::UNIX_EPOCH) - .map(|v| format!("'{}'", v.value)) - .unwrap_or_else(|| "NULL".to_string()) - }) - .collect(); - let mut attr_vals = initial_vals.join(", "); - if !attr_vals.is_empty() { - attr_vals.insert_str(0, ", "); - } - connection.execute( - &format!(r#"INSERT INTO "{table_name}" VALUES (?,?,NULL{attr_vals})"#), - [&o.id, &DateTime::UNIX_EPOCH.to_rfc3339()], - )?; + let placeholders = ",?".repeat(object_type.attributes.len()); + let sql = format!(r#"INSERT INTO "{table_name}" VALUES (?,?,?{placeholders})"#); + + let initial_vals = object_type.attributes.iter().map(|a| { + o.attributes + .iter() + .find(|oa| oa.name == a.name && oa.time == DateTime::UNIX_EPOCH) + .map(|v| v.value.to_string()) + }); + let params: Vec> = [ + Some(o.id.clone()), + Some(DateTime::UNIX_EPOCH.to_rfc3339()), + None, + ] + .into_iter() + .chain(initial_vals) + .collect(); + connection.execute(&sql, rusqlite::params_from_iter(params))?; for a in o .attributes .iter() .filter(|a| a.time != DateTime::UNIX_EPOCH) { - let vals: Vec<_> = object_type - .attributes - .iter() - .map(|ot_attr| { - if a.name == ot_attr.name { - format!("'{}'", a.value) - } else { - "NULL".to_string() - } - }) - .collect(); - let mut attr_vals = vals.join(", "); - if !attr_vals.is_empty() { - attr_vals.insert_str(0, ", "); - } - connection.execute( - &format!( - r#"INSERT INTO "{table_name}" VALUES (?,?,'{}'{attr_vals})"#, - a.name - ), - [&o.id, &a.time.to_rfc3339()], - )?; + let vals = object_type.attributes.iter().map(|ot_attr| { + if a.name == ot_attr.name { + Some(a.value.to_string()) + } else { + None + } + }); + let params: Vec> = [ + Some(o.id.clone()), + Some(a.time.to_rfc3339()), + Some(a.name.clone()), + ] + .into_iter() + .chain(vals) + .collect(); + connection.execute(&sql, rusqlite::params_from_iter(params))?; } Ok(()) } @@ -487,24 +639,20 @@ fn write_event_attrs_sqlite( event_type: &OCELType, e: &super::ocel_struct::OCELEvent, ) -> Result<(), DatabaseError> { - let vals: Vec<_> = event_type - .attributes - .iter() - .map(|a| { - e.attributes - .iter() - .find(|ea| ea.name == a.name) - .map(|v| format!("'{}'", v.value)) - .unwrap_or_else(|| "NULL".to_string()) - }) + let placeholders = ",?".repeat(event_type.attributes.len()); + let vals = event_type.attributes.iter().map(|a| { + e.attributes + .iter() + .find(|ea| ea.name == a.name) + .map(|v| v.value.to_string()) + }); + let params: Vec> = [Some(e.id.clone()), Some(e.time.to_rfc3339())] + .into_iter() + .chain(vals) .collect(); - let mut attr_vals = vals.join(", "); - if !attr_vals.is_empty() { - attr_vals.insert_str(0, ", "); - } connection.execute( - &format!(r#"INSERT INTO "{table_name}" VALUES (?,?{attr_vals})"#), - [&e.id, &e.time.to_rfc3339()], + &format!(r#"INSERT INTO "{table_name}" VALUES (?,?{placeholders})"#), + rusqlite::params_from_iter(params), )?; Ok(()) } @@ -537,7 +685,8 @@ fn write_event_attrs_duckdb( Ok(()) } -#[cfg(test)] +// The only test here round-trips through SQLite, so it needs the connector, not just `test`. +#[cfg(all(test, feature = "ocel-sqlite"))] mod test { use std::fs::remove_file; diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_import.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_import.rs index 16b98cf9..fcd2383f 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_import.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_import.rs @@ -52,9 +52,22 @@ fn get_row_attribute_value( /// /// If you want to import from a filepath, see [`import_ocel_sqlite_from_path`] instead. /// +/// Rejects a file whose object-type table has no `ocel_changed_field` column. Use +/// [`import_ocel_sqlite_from_con_with_options`] to tolerate that instead. +/// /// Note: This function is only available if the `ocel-sqlite` feature is enabled. /// pub fn import_ocel_sqlite_from_con(con: Connection) -> Result { + import_ocel_sqlite_from_con_with_options(con, SqlOcelImportOptions::default()) +} + +/// Import [`OCEL`] log from `SQLite` connection, with explicit [`SqlOcelImportOptions`]. +/// +/// Note: This function is only available if the `ocel-sqlite` feature is enabled. +pub fn import_ocel_sqlite_from_con_with_options( + con: Connection, + options: SqlOcelImportOptions, +) -> Result { let mut ocel = OCEL { event_types: Vec::default(), object_types: Vec::default(), @@ -86,20 +99,21 @@ pub fn import_ocel_sqlite_from_con(con: Connection) -> Result = HashMap::new(); for (ob_type, ob_type_ocel) in ob_type_map.iter() { - let mut s = con.prepare(format!("PRAGMA table_info('object_{ob_type}')").as_str())?; + let table_name = format!("object_{ob_type}"); + let mut s = + con.prepare(format!("PRAGMA table_info({})", quoted_str(&table_name)).as_str())?; let ob_attr_query = query_all::<_>(&mut s, [])?; - let ob_type_attrs: Vec = ob_attr_query + let raw_ob_columns: Vec<(String, String)> = ob_attr_query .and_then(|x| Ok::<(String, String), rusqlite::Error>((x.get("name")?, x.get("type")?))) .flatten() - .filter(|(name, _)| !IGNORED_PRAGMA_COLUMNS.contains(&name.as_str())) - .map(|(name, atype)| OCELTypeAttribute { - name, - value_type: sql_type_to_ocel(&atype).to_type_string(), - }) .collect(); - let mut s = con.prepare( - format!("SELECT * FROM 'object_{ob_type}' WHERE {OCEL_CHANGED_FIELD} IS NULL").as_str(), - )?; + let ObjectTablePlan { + attributes: ob_type_attrs, + initial_query, + changed_query, + } = plan_object_table(con.path(), ob_type, &table_name, raw_ob_columns, options) + .map_err(rusqlite::Error::InvalidColumnName)?; + let mut s = con.prepare(initial_query.as_str())?; let objs = query_all::<_>(&mut s, [])?; objs.and_then(|x| { Ok::<(String, Vec<_>), rusqlite::Error>(( @@ -142,42 +156,41 @@ pub fn import_ocel_sqlite_from_con(con: Connection) -> Result(&mut s, [])?; - objs.and_then(|x| { - let changed_field: String = x.get(OCEL_CHANGED_FIELD)?; - let changed_val = ob_type_attrs - .iter() - .find(|at| at.name == changed_field) - .ok_or(rusqlite::Error::InvalidQuery) - .and_then(|attr| get_row_attribute_value(attr, x))?; - Ok::<(String, _, String, OCELAttributeValue), rusqlite::Error>(( - x.get(OCEL_ID_COLUMN)?, - try_get_column_date_val(x, OCEL_TIME_COLUMN)?, - changed_field, - changed_val, - )) - }) - .flatten() - .for_each(|(ob_id, time, changed_field, changed_val)| { - object_map - .entry(ob_id.clone()) - .or_insert(OCELObject { - id: ob_id, - object_type: ob_type.clone(), - attributes: Vec::default(), - relationships: Vec::default(), - }) - .attributes - .push(OCELObjectAttribute { - name: changed_field, - value: changed_val, - time, - }); - }); + if let Some(changed_query) = &changed_query { + let mut s = con.prepare(changed_query.as_str())?; + let objs = query_all::<_>(&mut s, [])?; + objs.and_then(|x| { + let changed_field: String = x.get(OCEL_CHANGED_FIELD)?; + let changed_val = ob_type_attrs + .iter() + .find(|at| at.name == changed_field) + .ok_or(rusqlite::Error::InvalidQuery) + .and_then(|attr| get_row_attribute_value(attr, x))?; + Ok::<(String, _, String, OCELAttributeValue), rusqlite::Error>(( + x.get(OCEL_ID_COLUMN)?, + try_get_column_date_val(x, OCEL_TIME_COLUMN)?, + changed_field, + changed_val, + )) + }) + .flatten() + .for_each(|(ob_id, time, changed_field, changed_val)| { + object_map + .entry(ob_id.clone()) + .or_insert(OCELObject { + id: ob_id, + object_type: ob_type.clone(), + attributes: Vec::default(), + relationships: Vec::default(), + }) + .attributes + .push(OCELObjectAttribute { + name: changed_field, + value: changed_val, + time, + }); + }); + } let t = OCELType { name: ob_type_ocel.clone(), @@ -188,7 +201,13 @@ pub fn import_ocel_sqlite_from_con(con: Connection) -> Result(&mut s, [])?; let ev_type_attrs: Vec = ev_attr_query .and_then(|x| Ok::<(String, String), rusqlite::Error>((x.get("name")?, x.get("type")?))) @@ -200,7 +219,13 @@ pub fn import_ocel_sqlite_from_con(con: Connection) -> Result(&mut s, [])?; evs.and_then(|x| { Ok::<(String, _, Vec<_>), rusqlite::Error>(( @@ -311,6 +336,18 @@ pub fn import_ocel_sqlite_from_path>( import_ocel_sqlite_from_con(con) } +/// +/// Import an [`OCEL`] `SQLite` file from the given path, with custom options +/// +/// Note: This function is only available if the `ocel-sqlite` feature is enabled. +pub fn import_ocel_sqlite_from_path_with_options>( + path: P, + options: SqlOcelImportOptions, +) -> Result { + let con = Connection::open(path)?; + import_ocel_sqlite_from_con_with_options(con, options) +} + /// /// Import an [`OCEL`] `SQLite` file from the given byte slice /// @@ -434,3 +471,162 @@ mod sqlite_tests { Ok(()) } } + +#[cfg(test)] +mod missing_changed_field_tests { + use chrono::DateTime; + use rusqlite::Connection; + + use crate::core::event_data::object_centric::{ + ocel_sql::{ + import_ocel_sqlite_from_con, import_ocel_sqlite_from_con_with_options, + SqlOcelImportOptions, + }, + ocel_struct::{OCELAttributeValue, OCELObjectAttribute}, + }; + + /// A non-conforming `SQLite` file: object type `Truck` has no `ocel_changed_field` column, + /// but does carry a misspelled `ocel_change_field` one whose value names a real attribute. + fn build_fixture(path: &std::path::Path) { + let con = Connection::open(path).unwrap(); + con.execute_batch( + " + CREATE TABLE event_map_type (ocel_type_map TEXT, ocel_type TEXT); + CREATE TABLE object_map_type (ocel_type_map TEXT, ocel_type TEXT); + CREATE TABLE event_object (ocel_event_id TEXT, ocel_object_id TEXT, ocel_qualifier TEXT); + CREATE TABLE object_object (ocel_source_id TEXT, ocel_target_id TEXT, ocel_qualifier TEXT); + CREATE TABLE event_Ev (ocel_id TEXT PRIMARY KEY, ocel_time TIMESTAMP); + CREATE TABLE object_Truck ( + ocel_id TEXT PRIMARY KEY, + ocel_time TIMESTAMP, + driver TEXT, + ocel_change_field TEXT + ); + INSERT INTO object_map_type VALUES ('Truck', 'Truck'); + INSERT INTO event_map_type VALUES ('Ev', 'Ev'); + INSERT INTO event_Ev VALUES ('e1', '2020-01-01T00:00:00+00:00'); + INSERT INTO object_Truck VALUES ('t1', '2020-01-01T00:00:00+00:00', 'Alice', NULL); + INSERT INTO object_Truck VALUES ('t2', '2020-01-02T00:00:00+00:00', 'Bob', 'driver'); + ", + ) + .unwrap(); + } + + /// The column is only missing where there is no attribute change to record, so a file + /// without it is read rather than refused. + #[test] + fn default_import_reads_a_file_without_the_changed_field_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.sqlite"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let ocel = + import_ocel_sqlite_from_con(con).expect("a file without the column must still be read"); + assert_eq!(ocel.objects.len(), 2, "both Truck objects must be present"); + } + + /// Holding the file to the specification is opt-in, and names what is wrong with it. + #[test] + fn strict_import_rejects_missing_changed_field_column() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.sqlite"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let err = import_ocel_sqlite_from_con_with_options( + con, + SqlOcelImportOptions { + allow_missing_changed_field: false, + }, + ) + .unwrap_err(); + let msg = err.to_string(); + + assert!( + msg.contains("Truck"), + "message should name the object type: {msg}" + ); + assert!( + msg.contains("object_Truck"), + "message should name the table: {msg}" + ); + assert!( + msg.contains("ocel_changed_field"), + "message should name the missing column: {msg}" + ); + assert!( + msg.contains("does not conform"), + "message should say the file is non-conforming: {msg}" + ); + assert!( + msg.contains("ocel_change_field"), + "message should mention the misspelled column it found: {msg}" + ); + } + + #[test] + fn a_file_without_the_changed_field_column_reads_as_initial_state() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no-changed-field.sqlite"); + build_fixture(&path); + + let con = Connection::open(&path).unwrap(); + let ocel = import_ocel_sqlite_from_con_with_options( + con, + SqlOcelImportOptions { + allow_missing_changed_field: true, + }, + ) + .unwrap(); + + assert_eq!(ocel.objects.len(), 2, "both Truck objects must be present"); + + let t1 = ocel.objects.iter().find(|o| o.id == "t1").unwrap(); + assert_eq!(t1.object_type, "Truck"); + assert_eq!( + t1.attributes, + vec![OCELObjectAttribute { + name: "driver".to_string(), + value: OCELAttributeValue::String("Alice".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }], + "t1's ocel_change_field cell is NULL, so it contributes no attribute; \ + it must not be invented as a change record" + ); + + let t2 = ocel.objects.iter().find(|o| o.id == "t2").unwrap(); + assert_eq!(t2.object_type, "Truck"); + let mut t2_attrs = t2.attributes.clone(); + t2_attrs.sort_by(|a, b| a.name.cmp(&b.name)); + assert_eq!( + t2_attrs, + vec![ + OCELObjectAttribute { + name: "driver".to_string(), + value: OCELAttributeValue::String("Bob".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }, + OCELObjectAttribute { + name: "ocel_change_field".to_string(), + value: OCELAttributeValue::String("driver".to_string()), + time: DateTime::UNIX_EPOCH.into(), + }, + ], + "no attribute-change row must be invented from the misspelled column" + ); + + let truck_type = ocel + .object_types + .iter() + .find(|t| t.name == "Truck") + .unwrap(); + let mut attr_names: Vec<_> = truck_type + .attributes + .iter() + .map(|a| a.name.clone()) + .collect(); + attr_names.sort(); + assert_eq!(attr_names, vec!["driver", "ocel_change_field"]); + } +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_struct.rs b/process_mining/src/core/event_data/object_centric/ocel_struct.rs index e767b6aa..1f30074a 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_struct.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_struct.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::core::event_data::{ case_centric::AttributeValue, object_centric::linked_ocel::{index_linked_ocel::ObjectIndex, IndexLinkedOCEL}, + timestamp_utils::parse_timestamp, }; /// @@ -111,6 +112,7 @@ pub struct OCELEvent { #[serde(rename = "type")] pub event_type: String, /// `DateTime` when event occured + #[serde(deserialize_with = "robust_timestamp_parsing")] pub time: DateTime, /// Event attributes #[serde(default)] @@ -189,44 +191,14 @@ pub struct OCELObjectAttribute { pub time: DateTime, } +/// Deserialize a timestamp field, accepting every format understood by +/// [`parse_timestamp`] instead of only RFC3339. fn robust_timestamp_parsing<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, { let time: String = Deserialize::deserialize(deserializer)?; - if let Ok(dt) = DateTime::parse_from_rfc3339(&time) { - return Ok(dt); - } - if let Ok(dt) = DateTime::parse_from_rfc2822(&time) { - return Ok(dt); - } - // eprintln!("Encountered weird datetime format: {:?}", time); - - // Some logs have this date: "2023-10-06 09:30:21.890421" - // Assuming that this is UTC - if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&time, "%F %T%.f") { - return Ok(dt.and_utc().into()); - } - - // Also handle "2024-10-02T07:55:15.348555" as well as "2022-01-09T15:00:00" - // Assuming UTC time zone - if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&time, "%FT%T%.f") { - return Ok(dt.and_utc().into()); - } - - // export_path - if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&time, "%F %T UTC") { - return Ok(dt.and_utc().into()); - } - - // Who made me do this? 🫣 - // Some logs have this date: "Mon Apr 03 2023 12:08:18 GMT+0200 (Mitteleuropäische Sommerzeit)" - // Below ignores the first "Mon " part (%Z) parses the rest (only if "GMT") and then parses the timezone (+0200) - // The rest of the input is ignored - if let Ok((dt, _)) = DateTime::parse_and_remainder(&time, "%Z %b %d %Y %T GMT%z") { - return Ok(dt); - } - Err(serde::de::Error::custom("Unexpected Date Format")) + parse_timestamp(&time, None, false).map_err(serde::de::Error::custom) } impl OCELObjectAttribute { @@ -300,7 +272,7 @@ impl OCELAttributeValue { /// String -> Integer (parse via `i64::from_str`) /// String -> Float (parse via `f64::from_str`) /// String -> Boolean (case-insensitive `"true"`/`"false"`) - /// String -> Time (parse via RFC3339) + /// String -> Time (parse via [`parse_timestamp`], i.e. RFC3339 and fallbacks) pub fn try_coerce_to(&self, target: OCELAttributeType) -> Option { use OCELAttributeType as T; use OCELAttributeValue::*; @@ -324,7 +296,7 @@ impl OCELAttributeValue { "false" => Some(Boolean(false)), _ => None, }, - (T::Time, String(s)) => DateTime::parse_from_rfc3339(s).ok().map(Time), + (T::Time, String(s)) => parse_timestamp(s, None, false).ok().map(Time), _ => None, } } @@ -417,7 +389,7 @@ impl> From> for OCELAttributeValue { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] /// _Types_ of attribute values in OCEL2 pub enum OCELAttributeType { /// String @@ -477,4 +449,22 @@ impl OCELAttributeType { _ => OCELAttributeType::Null, } } + + /// The narrowest type that can hold values of both `self` and `other`: identical types + /// stay put, `Null` yields the other type, `Integer` + `Float` widens to `Float`, and + /// anything else falls back to `String`. + /// + /// Used where one attribute name carries different types (e.g. across event types, or + /// across rows of an untyped source) and a single column type must cover all of them. + pub fn coalesce(self, other: Self) -> Self { + use OCELAttributeType::*; + if self == other { + return self; + } + match (self, other) { + (Null, o) | (o, Null) => o, + (Integer, Float) | (Float, Integer) => Float, + _ => String, + } + } } diff --git a/process_mining/src/core/io.rs b/process_mining/src/core/io.rs index 0a235bfb..91b4cdcb 100644 --- a/process_mining/src/core/io.rs +++ b/process_mining/src/core/io.rs @@ -167,6 +167,26 @@ pub trait Exportable { self.export_to_path_with_options(path, Self::ExportOptions::default()) } + /// Export to `path` in an explicitly named format, rather than one read back off the path. + /// + /// Exists because a path cannot always carry the format: a directory has no extension, and + /// the OCEL 2.0 bundled format's uncompressed form is a directory. A caller that already + /// knows which format the user chose, for instance one picked from + /// [`Self::known_export_formats`], should use this rather than encoding the choice into a + /// filename and having it inferred straight back out. + /// + /// # Errors + /// As [`Self::export_to_writer_with_options`], plus any I/O error creating `path`. + fn export_to_path_as>( + &self, + path: P, + format: &str, + options: Self::ExportOptions, + ) -> Result<(), Self::Error> { + let file = std::fs::File::create(path)?; + self.export_to_writer_with_options(std::io::BufWriter::new(file), format, options) + } + /// Export as a byte array with the specified options fn export_to_bytes_with_options( &self, diff --git a/process_mining/src/core/mod.rs b/process_mining/src/core/mod.rs index b219f6f2..99bac01f 100644 --- a/process_mining/src/core/mod.rs +++ b/process_mining/src/core/mod.rs @@ -5,6 +5,8 @@ pub mod event_data; /// IO Traits pub mod io; +/// Bytes of a tabular data file, held for an extraction to read. +pub mod tabular_source; pub mod process_models; diff --git a/process_mining/src/core/tabular_source.rs b/process_mining/src/core/tabular_source.rs new file mode 100644 index 00000000..e92ab412 --- /dev/null +++ b/process_mining/src/core/tabular_source.rs @@ -0,0 +1,194 @@ +//! Bytes of a tabular data file, held in the registry so an extraction can read them. + +use std::any::Any; +use std::sync::{Mutex, MutexGuard}; + +use crate::core::io::{ExtensionWithMime, Importable}; + +/// A tabular data file kept in memory: a `SQLite` database, a CSV, a Parquet file, a workbook. +/// +/// Bytes cannot travel through a binding's JSON arguments, so a dropped file is stored here and +/// named by registry id. On `wasm32` this is the only way to read a source at all. +/// +/// The opened reader is cached and held as `Box` so this type stays free of the +/// `extraction-blueprint`/`ocel-sqlite` features. It sits behind a `Mutex` because a `SQLite` +/// connection is `Send` but not `Sync`, while the registry must be `Sync`. +pub struct TabularSource { + bytes: Vec, + format: String, + opened: Mutex>>, +} + +/// Reports the byte count rather than the bytes: [`RegistryItem`](crate::bindings::RegistryItem) +/// derives `Debug`, so one `{:?}` of the registry would otherwise dump every file it holds. +impl std::fmt::Debug for TabularSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TabularSource") + .field("format", &self.format) + .field("bytes", &self.bytes.len()) + .finish() + } +} + +impl TabularSource { + /// Wrap `bytes` of a file in `format` (an extension: `sqlite`, `csv`, `parquet`, `xlsx`). + /// + /// `format` is lowercased, so [`TabularSource::format`] is comparable against a lowercase + /// literal however the caller spelled the extension. + #[must_use] + pub fn new(bytes: Vec, format: impl Into) -> Self { + Self { + bytes, + format: format.into().to_lowercase(), + opened: Mutex::new(None), + } + } + + /// The cached reader, opening it with `open` the first time. + /// + /// The guard is returned rather than a reference: the reader is not `Sync`, so it can only be + /// touched while the lock is held. + /// + /// # Errors + /// Returns `open`'s error, or a message if a cached reader is not of type `T`. Errors rather + /// than blocking while another reader of the same source is alive, since a nested call would + /// otherwise deadlock the thread against itself. + pub fn reader(&self, open: F) -> Result, String> + where + T: Any + Send, + E: std::fmt::Display, + F: FnOnce(&[u8]) -> Result, + { + let mut guard = self.opened.try_lock().map_err(|e| match e { + std::sync::TryLockError::WouldBlock => { + "data source is already open for reading elsewhere".to_string() + } + std::sync::TryLockError::Poisoned(_) => "data source lock poisoned".to_string(), + })?; + if guard.is_none() { + *guard = Some(Box::new(open(&self.bytes).map_err(|e| e.to_string())?)); + } + if guard.as_ref().is_none_or(|b| !b.is::()) { + return Err("data source was already opened as a different reader".to_string()); + } + Ok(TabularReader { + guard, + _marker: std::marker::PhantomData, + }) + } + + /// The file's bytes. + #[must_use] + pub fn bytes(&self) -> &[u8] { + &self.bytes + } + + /// The file's format, lowercased. + #[must_use] + pub fn format(&self) -> &str { + &self.format + } +} + +/// A borrowed, opened reader over a [`TabularSource`], valid while the lock is held. +#[allow(missing_debug_implementations)] +pub struct TabularReader<'a, T> { + guard: MutexGuard<'a, Option>>, + _marker: std::marker::PhantomData T>, +} + +impl TabularReader<'_, T> { + /// The opened reader. + #[must_use] + pub fn get(&self) -> &T { + self.guard + .as_ref() + .and_then(|b| b.downcast_ref::()) + .expect("reader was checked when the guard was taken") + } +} + +/// What can go wrong while importing a [`TabularSource`]. +#[derive(Debug)] +pub enum TabularSourceError { + /// An I/O error while reading the bytes. + Io(std::io::Error), +} + +impl std::fmt::Display for TabularSourceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for TabularSourceError {} + +impl From for TabularSourceError { + fn from(e: std::io::Error) -> Self { + Self::Io(e) + } +} + +impl Importable for TabularSource { + type Error = TabularSourceError; + type ImportOptions = (); + + fn import_from_reader_with_options( + mut reader: R, + data_format: &str, + _options: Self::ImportOptions, + ) -> Result { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; + Ok(Self::new(bytes, data_format)) + } + + fn known_import_formats() -> Vec { + // Only the formats this build can open, so a file dialog cannot offer an unreadable one. + vec![ + ExtensionWithMime::new("sqlite", "application/vnd.sqlite3"), + ExtensionWithMime::new("sqlite3", "application/vnd.sqlite3"), + ExtensionWithMime::new("db", "application/vnd.sqlite3"), + #[cfg(feature = "extraction-dbcon")] + ExtensionWithMime::new("csv", "text/csv"), + #[cfg(feature = "extraction-dbcon")] + ExtensionWithMime::new("tsv", "text/tab-separated-values"), + #[cfg(feature = "extraction-dbcon")] + ExtensionWithMime::new("parquet", "application/vnd.apache.parquet"), + #[cfg(feature = "extraction-dbcon")] + ExtensionWithMime::new( + "xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_format_is_lowercased_however_it_was_spelled() { + for spelling in ["SQLite", "SQLITE", "sqlite"] { + assert_eq!(TabularSource::new(Vec::new(), spelling).format(), "sqlite"); + } + assert_eq!( + TabularSource::import_from_bytes(b"a,b\n1,2\n", "CSV") + .expect("reading from a slice cannot fail") + .format(), + "csv" + ); + } + + #[test] + fn debug_reports_the_size_rather_than_the_bytes() { + let rendered = format!("{:?}", TabularSource::new(vec![0xAB; 4096], "sqlite")); + assert!( + rendered.contains("4096") && !rendered.contains("171, 171"), + "a source should render as a summary: {rendered}" + ); + } +} diff --git a/process_mining/src/lib.rs b/process_mining/src/lib.rs index 8e071534..f27d2351 100644 --- a/process_mining/src/lib.rs +++ b/process_mining/src/lib.rs @@ -8,6 +8,19 @@ )] #![doc = include_str!("../README.md")] +// Allow the `#[register_binding]` macro's generated code to reference this crate by +// its absolute name (`::process_mining::...`) even from within the crate itself. +#[allow(unused_extern_crates)] +extern crate self as process_mining; + +/// Crate re-exports used by the `#[register_binding]` macro's generated code, so downstream +/// crates registering bindings only need a dependency on `process_mining`. +#[cfg(feature = "bindings")] +#[doc(hidden)] +pub mod __private { + pub use {inventory, schemars, serde_json, uuid}; +} + pub mod analysis; pub mod conformance; pub mod core; @@ -21,7 +34,7 @@ pub use core::{EventLog, PetriNet, OCEL}; // Re-export OCEL backend traits and the streaming entry points. pub use core::event_data::object_centric::{ - appendable::AppendableOCEL, + appendable::{AppendableOCEL, StreamImportOCEL}, ocel_json::import_ocel_json_into, ocel_xml::xml_ocel_import::{import_ocel_xml_into, OCELImportOptions}, readable::{OCELLookup, ReadableOCEL}, diff --git a/process_mining/tests/ocel_bundle.rs b/process_mining/tests/ocel_bundle.rs new file mode 100644 index 00000000..e1f5db52 --- /dev/null +++ b/process_mining/tests/ocel_bundle.rs @@ -0,0 +1,1043 @@ +//! `.ocel.zip` as an ordinary OCEL format: read and written through `Importable`/`Exportable`, +//! with no connector, exactly as `.jsonocel` and `.ocel.csv` are. + +#![cfg(feature = "ocel-bundle")] + +use std::collections::BTreeMap; +use std::path::Path; + +use process_mining::core::event_data::object_centric::ocel_bundle::{ + encode_type_name, import_ocel_bundle, import_ocel_bundle_from_bytes, +}; +use process_mining::core::event_data::object_centric::ocel_json::import_ocel_json_path; +use process_mining::core::event_data::object_centric::{OCELAttributeValue, OCEL}; +use process_mining::core::io::{Exportable, Importable}; + +fn running_example() -> OCEL { + import_ocel_json_path(concat!( + env!("CARGO_MANIFEST_DIR"), + "/test_data/ocel/ocel2-p2p.json" + )) + .expect("read the running example") +} + +/// Every attribute observation, as `id/name@time=value`, so a round trip is compared exactly +/// rather than by counts. +/// +/// A `time`-typed attribute is rendered by its instant rather than by its stored text: this log +/// keeps some of them as unparsed strings (`2023-10-24 09:30:10.235452`, no timezone), and a +/// round trip through the container reads them back as real timestamps. That normalisation is +/// the reader honouring the declared type, not a difference in what the log says. +fn observations(ocel: &OCEL) -> Vec { + let time_typed: std::collections::HashSet<(&str, &str)> = ocel + .object_types + .iter() + .flat_map(|t| { + t.attributes + .iter() + .filter(|a| a.value_type == "time") + .map(move |a| (t.name.as_str(), a.name.as_str())) + }) + .collect(); + + let mut out: Vec = ocel + .objects + .iter() + .flat_map(|o| { + let time_typed = &time_typed; + o.attributes.iter().map(move |a| { + let value = if time_typed.contains(&(o.object_type.as_str(), a.name.as_str())) { + match &a.value { + OCELAttributeValue::Time(t) => t.to_rfc3339(), + OCELAttributeValue::String(s) => { + process_mining::core::event_data::timestamp_utils::parse_timestamp( + s, None, false, + ) + .map_or_else(|_| s.clone(), |t| t.to_rfc3339()) + } + other => other.to_string(), + } + } else { + a.value.to_string() + }; + format!("{}/{}@{}={}", o.id, a.name, a.time.to_rfc3339(), value) + }) + }) + .collect(); + out.sort(); + out +} + +fn relations(ocel: &OCEL) -> (Vec, Vec) { + let mut e2o: Vec = ocel + .events + .iter() + .flat_map(|e| { + e.relationships + .iter() + .map(move |r| format!("{} {} {}", e.id, r.object_id, r.qualifier)) + }) + .collect(); + let mut o2o: Vec = ocel + .objects + .iter() + .flat_map(|o| { + o.relationships + .iter() + .map(move |r| format!("{} {} {}", o.id, r.object_id, r.qualifier)) + }) + .collect(); + e2o.sort(); + o2o.sort(); + (e2o, o2o) +} + +/// The formats a caller sees, and what each produces: `{csv, parquet} x {archive, directory}`. +fn cases() -> Vec<(&'static str, &'static str)> { + let mut out = vec![ + ("csv/archive", "log.ocel.zip"), + ("csv/directory", "container"), + ]; + if cfg!(feature = "ocel-bundle-parquet") { + out.push(("parquet/archive", "log-parquet.zip")); + out.push(("parquet/directory", "container-parquet")); + } + out +} + +#[test] +fn a_bundle_round_trips_through_the_import_export_traits() { + let source = running_example(); + assert!(!source.events.is_empty() && !source.objects.is_empty()); + + for (what, name) in cases() { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join(name); + // A directory has no extension to infer from, so it has to exist before the format can + // be read off it. + if what.ends_with("directory") { + std::fs::create_dir_all(&target).expect("mkdir"); + } + + // `export_to_path` picks the format from the path, the same call any other OCEL format + // takes. + source + .export_to_path(&target) + .unwrap_or_else(|e| panic!("{what}: export: {e}")); + + let back = + OCEL::import_from_path(&target).unwrap_or_else(|e| panic!("{what}: import: {e}")); + + assert_eq!(back.events.len(), source.events.len(), "{what}: events"); + assert_eq!(back.objects.len(), source.objects.len(), "{what}: objects"); + assert_eq!( + back.event_types.len(), + source.event_types.len(), + "{what}: event types" + ); + assert_eq!( + back.object_types.len(), + source.object_types.len(), + "{what}: object types" + ); + + let (e2o, o2o) = relations(&source); + let (back_e2o, back_o2o) = relations(&back); + assert_eq!(back_e2o, e2o, "{what}: e2o"); + assert_eq!(back_o2o, o2o, "{what}: o2o"); + + let before = observations(&source); + let after = observations(&back); + if what.starts_with("parquet") { + // Parquet has a real null, so absent and empty-string are different values and the + // object/object-change split loses and invents nothing. + assert_eq!(after, before, "{what}: attributes"); + } else { + // An empty cell is a missing value, so an attribute whose value is the empty + // string cannot survive. This log has 295 of them. + let lost: Vec<&String> = before.iter().filter(|o| !after.contains(o)).collect(); + let invented: Vec<&String> = after.iter().filter(|o| !before.contains(o)).collect(); + assert!(invented.is_empty(), "{what}: invented {invented:?}"); + assert!( + lost.iter().all(|o| o.ends_with('=')), + "{what}: CSV may only lose empty-string values, lost {:?}", + lost.iter() + .filter(|o| !o.ends_with('=')) + .collect::>() + ); + assert!(!lost.is_empty(), "{what}: this log has empty-string values"); + } + } +} + +/// Every advertised export format must actually export through `export_to_bytes`, not only +/// through `export_to_path`. A host offering a download rather than a file dialog only ever calls +/// the byte path, and a format listed but not handled there fails the moment a user clicks +/// Export. +#[test] +fn every_advertised_export_format_works_through_the_byte_path() { + let source = running_example(); + for format in ::known_export_formats() { + let ext = format.extension; + // These two are genuinely path-only: both write a database file the driver opens by name. + if ext.ends_with("sqlite") || ext.ends_with("duckdb") { + continue; + } + let bytes = source + .export_to_bytes(&ext) + .unwrap_or_else(|e| panic!("{ext}: {e}")); + assert!(!bytes.is_empty(), "{ext}: produced no bytes"); + } +} + +/// Every column chunk a Parquet container holds is compressed. +/// +/// `parquet`'s writers default to `Compression::UNCOMPRESSED`, and this exporter deliberately +/// stores its Parquet entries in the ZIP rather than deflating them, so that they stay +/// seekable. With the default that combination compressed nothing anywhere, and a real 406k-event +/// log exported to 117 MB of Parquet against 37 MB of CSV. +/// +/// The codec is asserted rather than the file size: which storage comes out smaller depends on +/// the log. Parquet's per-file footer and per-column dictionary are fixed overhead, so a log with +/// many small tables (this fixture) can be larger as Parquet even fully compressed, while a log +/// with large tables is substantially smaller. +#[cfg(feature = "ocel-bundle-parquet")] +#[test] +fn every_parquet_column_chunk_is_compressed() { + use parquet::file::reader::{FileReader, SerializedFileReader}; + + let bytes = running_example() + .export_to_bytes("ocel-parquet.zip") + .expect("export"); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).expect("archive"); + + let mut checked = 0; + for i in 0..archive.len() { + let mut entry = archive.by_index(i).expect("entry"); + if !entry.name().ends_with(".parquet") { + continue; + } + let mut buf = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut buf).expect("read"); + let reader = SerializedFileReader::new(bytes::Bytes::from(buf)).expect("parquet"); + for group in reader.metadata().row_groups() { + for column in group.columns() { + assert_ne!( + column.compression(), + parquet::basic::Compression::UNCOMPRESSED, + "{}: {} is uncompressed", + entry.name(), + column.column_path() + ); + checked += 1; + } + } + } + assert!(checked > 0, "the archive had column chunks to check"); +} + +/// A directory container is opened by picking its manifest. A file dialog cannot select a +/// directory on every platform, so `ocel-meta.json` is the handle a person actually has. +#[test] +fn picking_the_manifest_opens_the_directory_it_sits_in() { + let source = running_example(); + let dir = tempfile::tempdir().expect("tempdir"); + let container = dir.path().join("container"); + std::fs::create_dir_all(&container).expect("mkdir"); + source.export_to_path(&container).expect("export"); + + let manifest = container.join("ocel-meta.json"); + assert!(manifest.is_file(), "the export wrote a manifest"); + + // Through the trait, as a host importing a picked file does. It must not be read as the + // `.json` its name ends with. + let back = OCEL::import_from_path(&manifest).expect("import via manifest"); + assert_eq!(back.events.len(), source.events.len()); + assert_eq!(back.objects.len(), source.objects.len()); +} + +/// Both names an export can produce are names an import accepts. The reader ignores which +/// storage a container uses, since the manifest says, but the filenames still have to +/// round-trip. +#[test] +fn every_name_export_writes_is_a_name_import_accepts() { + let import: Vec = ::known_import_formats() + .into_iter() + .map(|e| e.extension) + .collect(); + for exported in ::known_export_formats() { + if !exported.extension.ends_with("zip") { + continue; + } + assert!( + import.contains(&exported.extension), + "export writes .{} but import does not accept it: {import:?}", + exported.extension + ); + } +} + +/// The route a build with no filesystem takes: bytes in, bytes out, no path anywhere, through +/// both entry points a caller has. +/// +/// A directory container needs a filesystem and an archive on disk is expanded into a temp +/// directory, so on `wasm32` this is the only bundle path that works. +#[test] +fn a_container_round_trips_through_bytes_alone() { + type ImportFromBytes = fn(&[u8], &str) -> OCEL; + let routes: [(&str, ImportFromBytes); 2] = [ + ("the format registry", |bytes, format| { + OCEL::import_from_bytes(bytes, format).expect("import") + }), + ("the bundle reader", |bytes, _| { + import_ocel_bundle_from_bytes(bytes).expect("import") + }), + ]; + + let source = running_example(); + let mut formats = vec!["ocel.zip"]; + if cfg!(feature = "ocel-bundle-parquet") { + formats.push("ocel-parquet.zip"); + } + for format in formats { + let bytes = source + .export_to_bytes(format) + .unwrap_or_else(|e| panic!("{format}: export: {e}")); + for (what, import) in routes { + let back = import(&bytes, format); + assert_eq!( + back.events.len(), + source.events.len(), + "{format} via {what}: events" + ); + assert_eq!( + back.objects.len(), + source.objects.len(), + "{format} via {what}: objects" + ); + } + } +} + +/// A `.zip` is offered wherever the other OCEL formats are, so a file picker and an OS file +/// association both list it. +#[test] +fn the_bundled_format_is_advertised_alongside_the_others() { + let import: Vec = ::known_import_formats() + .into_iter() + .map(|e| e.extension) + .collect(); + assert!(import.contains(&"ocel.zip".to_string()), "{import:?}"); + assert!( + import.contains(&"json".to_string()), + "the others are still there" + ); + + let export: Vec = ::known_export_formats() + .into_iter() + .map(|e| e.extension) + .collect(); + assert!(export.contains(&"ocel.zip".to_string()), "{export:?}"); + #[cfg(feature = "ocel-bundle-parquet")] + assert!( + export.contains(&"ocel-parquet.zip".to_string()), + "Parquet storage needs a format string of its own: {export:?}" + ); +} + +/// An empty CSV cell is a missing value, which is what the format says. Pinned here because the +/// blueprint-based route over the same layout cannot tell the two apart cheaply and does not. +#[test] +fn an_empty_csv_cell_is_a_missing_value_not_an_empty_string() { + use process_mining::core::event_data::object_centric::OCELAttributeType; + use process_mining::core::event_data::object_centric::{ + OCELObject, OCELObjectAttribute, OCELType, OCELTypeAttribute, + }; + + let ocel = OCEL { + event_types: Vec::new(), + object_types: vec![OCELType { + name: "order".to_string(), + attributes: vec![ + OCELTypeAttribute::new("note", &OCELAttributeType::String), + OCELTypeAttribute::new("count", &OCELAttributeType::Integer), + ], + }], + events: Vec::new(), + // Declares two attributes, carries one. + objects: vec![OCELObject { + id: "o1".to_string(), + object_type: "order".to_string(), + attributes: vec![OCELObjectAttribute { + name: "count".to_string(), + value: OCELAttributeValue::Integer(3), + time: chrono::DateTime::from_timestamp_nanos(0).into(), + }], + relationships: Vec::new(), + }], + }; + + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join("log.ocel.zip"); + ocel.export_to_path(&target).expect("export"); + let back = OCEL::import_from_path(&target).expect("import"); + + let names: Vec<&str> = back.objects[0] + .attributes + .iter() + .map(|a| a.name.as_str()) + .collect(); + assert_eq!(names, ["count"], "the absent attribute stays absent"); + assert_eq!( + back.objects[0].attributes[0].value, + OCELAttributeValue::Integer(3) + ); +} + +// The running-example fixture is written out here rather than committed, so the test states the +// format's layout in one place and a change to the naming rules has to be made deliberately. + +/// `(type name, attribute columns, rows)` for the running example's event tables. +const EVENTS: &[(&str, &str, &[&str])] = &[ + ( + "Create Purchase Requisition", + "pr_creator", + &["e1,2022-01-09T15:00:00+00:00,Mike"], + ), + ( + "Approve Purchase Requisition", + "pr_approver", + &["e2,2022-01-09T16:30:00+00:00,Tania"], + ), + ( + "Create Purchase Order", + "po_creator", + &[ + "e3,2022-01-10T09:15:00+00:00,Mike", + "e10,2022-02-02T17:00:00+00:00,Mario", + ], + ), + ( + "Change PO Quantity", + "po_editor", + &["e4,2022-01-13T12:00:00+00:00,Mike"], + ), + ( + "Insert Invoice", + "invoice_inserter", + &[ + "e5,2022-01-14T12:00:00+00:00,Luke", + "e6,2022-01-16T11:00:00+00:00,Luke", + "e9,2022-02-02T09:00:00+00:00,Mario", + ], + ), + ( + "Insert Payment", + "payment_inserter", + &[ + "e7,2022-01-30T23:00:00+00:00,Robot", + "e8,2022-01-31T22:00:00+00:00,Robot", + "e13,2022-02-28T23:00:00+00:00,Robot", + ], + ), + ( + "Set Payment Block", + "invoice_blocker", + &["e11,2022-02-03T07:30:00+00:00,Sam"], + ), + ( + "Remove Payment Block", + "invoice_block_rem", + &["e12,2022-02-03T23:30:00+00:00,Mario"], + ), +]; + +/// An object type's fixture: its name, its attribute columns with their declared types, its +/// object rows, and its change rows. +type ObjectFixture = ( + &'static str, + &'static [(&'static str, &'static str)], + &'static [&'static str], + &'static [&'static str], +); + +const OBJECTS: &[ObjectFixture] = &[ + ( + "Purchase Requisition", + &[("pr_product", "string"), ("pr_quantity", "integer")], + &["PR1,Cows,500"], + &[], + ), + ( + "Purchase Order", + &[("po_product", "string"), ("po_quantity", "integer")], + &["PO1,Cows,500", "PO2,Notebooks,1"], + &["PO1,2022-01-13T12:00:00+00:00,po_quantity,,600"], + ), + ( + "Invoice", + &[("is_blocked", "string")], + &["R1,No", "R2,No", "R3,No"], + &[ + "R3,2022-02-03T07:30:00+00:00,is_blocked,Yes", + "R3,2022-02-03T23:30:00+00:00,is_blocked,No", + ], + ), + ("Payment", &[], &["P1", "P2", "P3"], &[]), +]; + +const E2O: &str = "\ +e1,PR1,Regular placement of PR +e2,PR1,Regular approval of PR +e3,PR1,Created order from PR +e3,PO1,Created order with identifier +e4,PO1,Change of quantity +e5,PO1,Invoice created starting from the PO +e5,R1,Invoice created with identifier +e6,PO1,Invoice created starting from the PO +e6,R2,Invoice created with identifier +e7,R1,Payment for the invoice +e7,P1,Payment inserted with identifier +e8,R2,Payment for the invoice +e8,P2,Payment inserted with identifier +e9,R3,Invoice created with identifier +e10,R3,Purchase order created with maverick buying from +e10,PO2,Purchase order created with identifier +e11,R3,Payment block due to unethical maverick buying +e12,R3,Payment block removed +e13,R3,Payment for the invoice +e13,P3,Payment inserted with identifier +"; + +const O2O: &str = "\ +PR1,PO1,PO from PR +PO1,R1,Invoice from PO +PO1,R2,Invoice from PO +R1,P1,Payment from invoice +R2,P2,Payment from invoice +PO2,R3,Maverick buying +R3,P3,Payment from invoice +"; + +/// Write the running example as a CSV container under `root`, exactly as the format lays it out. +fn write_container(root: &Path) { + let mut files: BTreeMap = BTreeMap::new(); + let mut event_types = Vec::new(); + let mut object_types = Vec::new(); + + for (ty, attr, rows) in EVENTS { + let file = format!("events/event_{}.csv", encode_type_name(ty)); + files.insert( + file.clone(), + format!("ocel_id,ocel_time,{attr}\n{}\n", rows.join("\n")), + ); + event_types.push(format!( + r#""{ty}": {{ "file": "{file}", "attributes": [{{ "name": "{attr}", "type": "string" }}] }}"# + )); + } + + for (ty, attrs, rows, changes) in OBJECTS { + let enc = encode_type_name(ty); + let file = format!("objects/object_{enc}.csv"); + let changes_file = format!("object_changes/object_changes_{enc}.csv"); + let names: Vec<&str> = attrs.iter().map(|(n, _)| *n).collect(); + + let mut header = "ocel_id".to_string(); + for n in &names { + header.push(','); + header.push_str(n); + } + files.insert(file.clone(), format!("{header}\n{}\n", rows.join("\n"))); + + let mut change_header = "ocel_id,ocel_time,ocel_changed_field".to_string(); + for n in &names { + change_header.push(','); + change_header.push_str(n); + } + // An object type with no changes still gets its table: header only, no rows. + files.insert( + changes_file.clone(), + if changes.is_empty() { + format!("{change_header}\n") + } else { + format!("{change_header}\n{}\n", changes.join("\n")) + }, + ); + + let decls: Vec = attrs + .iter() + .map(|(n, t)| format!(r#"{{ "name": "{n}", "type": "{t}" }}"#)) + .collect(); + object_types.push(format!( + r#""{ty}": {{ "file": "{file}", "changesFile": "{changes_file}", "attributes": [{}] }}"#, + decls.join(", ") + )); + } + + files.insert( + "relations/e2o.csv".to_string(), + format!("ocel_event_id,ocel_object_id,ocel_qualifier\n{E2O}"), + ); + files.insert( + "relations/o2o.csv".to_string(), + format!("ocel_source_id,ocel_target_id,ocel_qualifier\n{O2O}"), + ); + files.insert( + "ocel-meta.json".to_string(), + format!( + r#"{{ + "ocelVersion": "2.0", + "bundleFormatVersion": "1.0", + "storageFormat": "csv", + "eventTypes": {{ {} }}, + "objectTypes": {{ {} }}, + "relations": {{ "e2o": "relations/e2o.csv", "o2o": "relations/o2o.csv" }} +}}"#, + event_types.join(",\n"), + object_types.join(",\n") + ), + ); + + for (rel, contents) in files { + let path = root.join(&rel); + std::fs::create_dir_all(path.parent().expect("has a parent")).expect("mkdir"); + std::fs::write(&path, contents).expect("write"); + } +} + +/// Zip up `root`, with every entry written using `method`. +fn zip_dir(root: &Path, method: zip::CompressionMethod) -> Vec { + let mut buf = Vec::new(); + { + let mut w = zip::ZipWriter::new(std::io::Cursor::new(&mut buf)); + let opts: zip::write::FileOptions<'_, ()> = + zip::write::FileOptions::default().compression_method(method); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).expect("read_dir") { + let path = entry.expect("entry").path(); + if path.is_dir() { + stack.push(path); + continue; + } + let rel = path.strip_prefix(root).expect("under root"); + w.start_file(rel.to_string_lossy().replace('\\', "/"), opts) + .expect("start_file"); + std::io::Write::write_all(&mut w, &std::fs::read(&path).expect("read")) + .expect("write entry"); + } + } + w.finish().expect("finish"); + } + buf +} + +/// Every observation of `attr` on `object_id`, oldest first, as `time=value`. +fn attr_history(ocel: &OCEL, object_id: &str, attr: &str) -> Vec { + let ob = ocel + .objects + .iter() + .find(|o| o.id == object_id) + .unwrap_or_else(|| panic!("no object '{object_id}'")); + let mut vals: Vec<(&chrono::DateTime, &OCELAttributeValue)> = ob + .attributes + .iter() + .filter(|a| a.name == attr) + .map(|a| (&a.time, &a.value)) + .collect(); + vals.sort_by_key(|(t, _)| **t); + vals.iter() + .map(|(t, v)| format!("{}={}", t.to_rfc3339(), v)) + .collect() +} + +/// The whole of the running example, asserted on the exact object and change-table semantics +/// rather than on counts alone. +fn assert_running_example(ocel: &OCEL) { + assert_eq!(ocel.events.len(), 13, "e1..e13"); + assert_eq!(ocel.objects.len(), 9, "PR1, PO1-2, R1-3, P1-3"); + assert_eq!( + ocel.events + .iter() + .map(|e| e.relationships.len()) + .sum::(), + 20, + "every e2o row" + ); + assert_eq!( + ocel.objects + .iter() + .map(|o| o.relationships.len()) + .sum::(), + 7, + "every o2o row" + ); + + // An object table's values are initial ones, held from the epoch; a change row adds another + // value at its own instant. The unchanged attribute keeps exactly one value. + assert_eq!( + attr_history(ocel, "PO1", "po_quantity"), + [ + "1970-01-01T00:00:00+00:00=500", + "2022-01-13T12:00:00+00:00=600" + ] + ); + assert_eq!( + attr_history(ocel, "PO1", "po_product"), + ["1970-01-01T00:00:00+00:00=Cows"], + "a change to po_quantity must not record anything for po_product" + ); + assert_eq!( + attr_history(ocel, "R3", "is_blocked"), + [ + "1970-01-01T00:00:00+00:00=No", + "2022-02-03T07:30:00+00:00=Yes", + "2022-02-03T23:30:00+00:00=No" + ] + ); +} + +/// The storage formats this build can write. Reading Parquet needs the same feature. +fn storages() -> Vec { + use process_mining::core::event_data::object_centric::ocel_bundle::StorageFormat; + let mut out = vec![StorageFormat::Csv]; + if cfg!(feature = "ocel-bundle-parquet") { + out.push(StorageFormat::Parquet); + } + out +} + +#[test] +fn a_directory_container_imports_the_running_example() { + let dir = tempfile::tempdir().expect("tempdir"); + write_container(dir.path()); + + let ocel = import_ocel_bundle(dir.path()).expect("import"); + assert_running_example(&ocel); +} + +/// Every combination the format allows: {directory, archive} x {CSV, Parquet}, and for an +/// archive both compression methods, since the format names neither and a container this crate +/// did not write may use either. +/// +/// The Parquet-in-a-deflated-archive cell is the one no other test reaches: this crate's own +/// exporter stores Parquet entries uncompressed so a reader can seek them, so only a +/// third-party writer produces it. +#[test] +fn every_layout_and_storage_combination_imports_to_the_same_log() { + use process_mining::core::event_data::object_centric::ocel_bundle::{ + export_ocel_bundle, BundleExportOptions, ContainerLayout, + }; + + // The running example, read from the hand-written CSV fixture, is what every combination + // below is re-encoded from, so a difference is the encoding's, not the data's. + let fixture = tempfile::tempdir().expect("tempdir"); + write_container(fixture.path()); + let source = import_ocel_bundle(fixture.path()).expect("read the fixture"); + assert_running_example(&source); + + for storage in storages() { + let work = tempfile::tempdir().expect("tempdir"); + let as_dir = work.path().join("container"); + std::fs::create_dir_all(&as_dir).expect("mkdir"); + export_ocel_bundle( + &source, + &as_dir, + BundleExportOptions { + layout: ContainerLayout::Directory, + storage, + }, + ) + .unwrap_or_else(|e| panic!("{storage:?}: export: {e}")); + + let from_dir = import_ocel_bundle(&as_dir) + .unwrap_or_else(|e| panic!("{storage:?}/directory: import: {e}")); + assert_running_example(&from_dir); + + for method in [ + zip::CompressionMethod::Stored, + zip::CompressionMethod::Deflated, + ] { + let what = format!("{storage:?}/archive/{method:?}"); + let bytes = zip_dir(&as_dir, method); + let archive = work.path().join("log.ocel.zip"); + std::fs::write(&archive, &bytes).expect("write archive"); + + let from_archive = + import_ocel_bundle(&archive).unwrap_or_else(|e| panic!("{what}: import: {e}")); + assert_running_example(&from_archive); + + // The same archive with no path to expand beside it. + let from_memory = import_ocel_bundle_from_bytes(&bytes) + .unwrap_or_else(|e| panic!("{what}: import from bytes: {e}")); + assert_running_example(&from_memory); + + std::fs::remove_file(&archive).expect("clean up"); + } + } +} + +/// The archive this crate writes stores Parquet entries uncompressed, so a reader can seek +/// within them without expanding the archive, and deflates CSV, which is read start to finish +/// anyway. The format itself names neither method, so both are a writer's choice. +#[test] +fn a_written_archive_stores_parquet_entries_and_deflates_csv_ones() { + use process_mining::core::event_data::object_centric::ocel_bundle::{ + export_ocel_bundle, BundleExportOptions, ContainerLayout, StorageFormat, + }; + + let fixture = tempfile::tempdir().expect("tempdir"); + write_container(fixture.path()); + let source = import_ocel_bundle(fixture.path()).expect("read the fixture"); + + for storage in storages() { + let want = match storage { + StorageFormat::Csv => zip::CompressionMethod::Deflated, + StorageFormat::Parquet => zip::CompressionMethod::Stored, + }; + let dir = tempfile::tempdir().expect("tempdir"); + let archive = dir.path().join("log.ocel.zip"); + export_ocel_bundle( + &source, + &archive, + BundleExportOptions { + layout: ContainerLayout::Archive, + storage, + }, + ) + .expect("export"); + + let file = std::fs::File::open(&archive).expect("open"); + let mut zip = zip::ZipArchive::new(file).expect("read archive"); + let mut tables = 0; + for i in 0..zip.len() { + let entry = zip.by_index(i).expect("entry"); + let name = entry.name().to_string(); + if name == "ocel-meta.json" { + continue; + } + tables += 1; + assert_eq!(entry.compression(), want, "{storage:?}: {name}"); + } + assert!(tables > 0, "{storage:?}: the archive has tables"); + } +} + +/// A manifest is untrusted input, and its declared paths are joined onto the container root. +#[test] +fn a_manifest_naming_a_path_outside_the_container_reads_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + std::fs::write(root.join("secret.csv"), "ocel_id\nleaked\n").expect("write"); + std::fs::create_dir_all(root.join("container")).expect("mkdir"); + std::fs::write( + root.join("container/ocel-meta.json"), + r#"{ + "ocelVersion": "2.0", + "bundleFormatVersion": "1.0", + "storageFormat": "csv", + "eventTypes": {}, + "objectTypes": { + "Escaped": { "file": "../secret.csv", "attributes": [] } + }, + "relations": { "e2o": "relations/e2o.csv", "o2o": "relations/o2o.csv" } + }"#, + ) + .expect("write manifest"); + + let err = import_ocel_bundle(root.join("container")).expect_err("must not escape"); + let message = err.to_string(); + assert!(message.contains("../secret.csv"), "{message}"); + assert!( + message.contains("must stay inside the container"), + "{message}" + ); +} + +/// A directory with no manifest is not a container, and the error says which file was looked for. +#[test] +fn a_path_that_is_not_a_container_is_rejected_with_a_message_naming_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let err = import_ocel_bundle(dir.path()).expect_err("no manifest"); + assert!(err.to_string().contains("ocel-meta.json"), "{err}"); +} + +/// One column of a Parquet fixture table, for a container this crate did not write. +#[cfg(feature = "ocel-bundle-parquet")] +enum Col<'a> { + /// `BYTE_ARRAY` with logical type `STRING`. An empty string is written as null, which is + /// how the format spells a missing attribute value in Parquet storage. + Text(&'a [&'a str]), + /// `INT64`. + Int(&'a [i64]), + /// `INT64` with logical type `TIMESTAMP(MICROS, isAdjustedToUTC=true)`, as the format + /// requires: an exporter must not write timestamps as strings in Parquet storage. + Time(&'a [&'a str]), +} + +/// Write `columns` as a Parquet file. Fixed columns are `required`, attribute columns +/// `optional`, per the format's Parquet mapping. +#[cfg(feature = "ocel-bundle-parquet")] +fn write_parquet(path: &Path, columns: &[(&str, bool, Col<'_>)]) { + use parquet::data_type::{ByteArray, ByteArrayType, Int64Type}; + use parquet::file::properties::WriterProperties; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::parser::parse_message_type; + use std::sync::Arc; + + let fields: Vec = columns + .iter() + .map(|(name, required, col)| { + let rep = if *required { "REQUIRED" } else { "OPTIONAL" }; + match col { + Col::Text(_) => format!("{rep} BYTE_ARRAY {name} (STRING);"), + Col::Int(_) => format!("{rep} INT64 {name};"), + Col::Time(_) => format!("{rep} INT64 {name} (TIMESTAMP(MICROS,true));"), + } + }) + .collect(); + let schema = Arc::new( + parse_message_type(&format!("message row {{ {} }}", fields.join(" "))).expect("schema"), + ); + + std::fs::create_dir_all(path.parent().expect("has a parent")).expect("mkdir"); + let file = std::fs::File::create(path).expect("create"); + let mut writer = + SerializedFileWriter::new(file, schema, Arc::new(WriterProperties::new())).expect("writer"); + let mut group = writer.next_row_group().expect("row group"); + for (_, required, col) in columns { + let mut w = group.next_column().expect("column").expect("some column"); + // An optional column needs definition levels: 1 where a value is present, 0 for null. + match col { + Col::Text(values) => { + let present: Vec = values + .iter() + .filter(|v| !v.is_empty()) + .map(|v| ByteArray::from(*v)) + .collect(); + let defs: Vec = values.iter().map(|v| i16::from(!v.is_empty())).collect(); + w.typed::() + .write_batch(&present, (!required).then_some(&defs), None) + .expect("write"); + } + Col::Int(values) => { + let defs: Vec = values.iter().map(|_| 1).collect(); + w.typed::() + .write_batch(values, (!required).then_some(&defs), None) + .expect("write"); + } + Col::Time(values) => { + let micros: Vec = values + .iter() + .map(|v| { + chrono::DateTime::parse_from_rfc3339(v) + .expect("rfc3339") + .timestamp_micros() + }) + .collect(); + let defs: Vec = micros.iter().map(|_| 1).collect(); + w.typed::() + .write_batch(µs, (!required).then_some(&defs), None) + .expect("write"); + } + } + w.close().expect("close column"); + } + group.close().expect("close group"); + writer.close().expect("close writer"); +} + +/// The other storage format. Its point is that attribute types come from the file's own schema +/// rather than from the manifest, so a timestamp arrives as an instant and an integer as an +/// integer, which is also what makes a Parquet container compilable to SQL views. +#[cfg(feature = "ocel-bundle-parquet")] +#[test] +fn a_parquet_container_imports_with_types_taken_from_the_files() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + + write_parquet( + &root.join("events/event_Change%20PO%20Quantity.parquet"), + &[ + ("ocel_id", true, Col::Text(&["e4"])), + ("ocel_time", true, Col::Time(&["2022-01-13T12:00:00+00:00"])), + ("po_editor", false, Col::Text(&["Mike"])), + ], + ); + write_parquet( + &root.join("objects/object_Purchase%20Order.parquet"), + &[ + ("ocel_id", true, Col::Text(&["PO1"])), + ("po_product", false, Col::Text(&["Cows"])), + ("po_quantity", false, Col::Int(&[500])), + ], + ); + write_parquet( + &root.join("object_changes/object_changes_Purchase%20Order.parquet"), + &[ + ("ocel_id", true, Col::Text(&["PO1"])), + ("ocel_time", true, Col::Time(&["2022-01-13T12:00:00+00:00"])), + ("ocel_changed_field", true, Col::Text(&["po_quantity"])), + ("po_product", false, Col::Text(&[""])), + ("po_quantity", false, Col::Int(&[600])), + ], + ); + write_parquet( + &root.join("relations/e2o.parquet"), + &[ + ("ocel_event_id", true, Col::Text(&["e4"])), + ("ocel_object_id", true, Col::Text(&["PO1"])), + ("ocel_qualifier", true, Col::Text(&["Change of quantity"])), + ], + ); + write_parquet( + &root.join("relations/o2o.parquet"), + &[ + ("ocel_source_id", true, Col::Text(&[])), + ("ocel_target_id", true, Col::Text(&[])), + ("ocel_qualifier", true, Col::Text(&[])), + ], + ); + std::fs::write( + root.join("ocel-meta.json"), + r#"{ + "ocelVersion": "2.0", + "bundleFormatVersion": "1.0", + "storageFormat": "parquet", + "eventTypes": { + "Change PO Quantity": { "file": "events/event_Change%20PO%20Quantity.parquet", + "attributes": [{ "name": "po_editor", "type": "string" }] } + }, + "objectTypes": { + "Purchase Order": { "file": "objects/object_Purchase%20Order.parquet", + "changesFile": "object_changes/object_changes_Purchase%20Order.parquet", + "attributes": [{ "name": "po_product", "type": "string" }, + { "name": "po_quantity", "type": "integer" }] } + }, + "relations": { "e2o": "relations/e2o.parquet", "o2o": "relations/o2o.parquet" } + }"#, + ) + .expect("write manifest"); + + let ocel = import_ocel_bundle(root).expect("import"); + assert_eq!(ocel.events.len(), 1); + assert_eq!(ocel.objects.len(), 1); + assert_eq!( + attr_history(&ocel, "PO1", "po_quantity"), + [ + "1970-01-01T00:00:00+00:00=500", + "2022-01-13T12:00:00+00:00=600" + ], + "the INT64 attribute stays an integer through a change" + ); + assert_eq!( + attr_history(&ocel, "PO1", "po_product"), + ["1970-01-01T00:00:00+00:00=Cows"], + "the null cell in the change row records nothing" + ); + assert_eq!( + ocel.events[0].time.to_rfc3339(), + "2022-01-13T12:00:00+00:00", + "TIMESTAMP(MICROS) arrives as an instant, not as text to be parsed" + ); +} diff --git a/process_mining/tests/ocel_roundtrip_edge_cases.rs b/process_mining/tests/ocel_roundtrip_edge_cases.rs new file mode 100644 index 00000000..b14b1b36 --- /dev/null +++ b/process_mining/tests/ocel_roundtrip_edge_cases.rs @@ -0,0 +1,355 @@ +//! What survives a round trip through the bundled container and the flat CSV, and what the +//! formats say may not. +//! +//! The bundle stores names and values as they are and carries a schema, so a round trip through +//! it is exact. The flat CSV has three characters that structure a cell, no schema at all, and +//! rules about whitespace, so it is exact only where the format says it is. + +use std::collections::BTreeMap; + +use process_mining::core::event_data::object_centric::ocel_csv::{ + export_ocel_csv_to_string, import_ocel_csv, +}; +use process_mining::core::event_data::object_centric::{ + OCELAttributeValue, OCELEvent, OCELEventAttribute, OCELObject, OCELObjectAttribute, + OCELRelationship, OCELType, OCELTypeAttribute, OCEL, +}; + +#[cfg(feature = "ocel-bundle")] +use process_mining::core::event_data::object_centric::ocel_bundle::{ + export_ocel_bundle, import_ocel_bundle, BundleExportOptions, ContainerLayout, StorageFormat, +}; + +/// A one-event, one-object log with `text` woven through every name, id and qualifier. +fn log_named(text: &str) -> OCEL { + let event_type = format!("ev {text}"); + let object_type = format!("ob {text}"); + OCEL { + event_types: vec![OCELType { + name: event_type.clone(), + attributes: vec![OCELTypeAttribute { + name: format!("ea {text}"), + value_type: "string".into(), + }], + }], + object_types: vec![OCELType { + name: object_type.clone(), + attributes: vec![OCELTypeAttribute { + name: format!("oa {text}"), + value_type: "string".into(), + }], + }], + events: vec![OCELEvent { + id: format!("e {text}"), + event_type, + time: "2024-01-01T10:00:00+00:00".parse().unwrap(), + attributes: vec![OCELEventAttribute { + name: format!("ea {text}"), + value: OCELAttributeValue::String(format!("val {text}")), + }], + relationships: vec![OCELRelationship { + object_id: format!("o {text}"), + qualifier: format!("q {text}"), + }], + }], + objects: vec![OCELObject { + id: format!("o {text}"), + object_type, + attributes: vec![OCELObjectAttribute { + name: format!("oa {text}"), + value: OCELAttributeValue::String(format!("val {text}")), + time: "2024-01-02T10:00:00+00:00".parse().unwrap(), + }], + relationships: vec![], + }], + } +} + +/// A one-event log whose single event attribute carries `value` as text. +fn log_valued(value: &str) -> OCEL { + let mut ocel = log_named("v"); + ocel.events[0].attributes[0].value = OCELAttributeValue::String(value.to_string()); + ocel +} + +/// Everything the log says, as sorted lines, so two logs are compared exactly rather than by +/// counts. Object attributes are keyed by name only, because the CSV infers type declarations +/// rather than storing them. +fn facts(ocel: &OCEL) -> Vec { + let mut out = Vec::new(); + for t in &ocel.event_types { + out.push(format!("event type {:?}", t.name)); + } + for t in &ocel.object_types { + out.push(format!("object type {:?}", t.name)); + } + for e in &ocel.events { + out.push(format!("event {:?} of {:?}", e.id, e.event_type)); + for a in &e.attributes { + out.push(format!("event {:?} {:?} = {:?}", e.id, a.name, a.value)); + } + for r in &e.relationships { + out.push(format!( + "e2o {:?} -> {:?} [{:?}]", + e.id, r.object_id, r.qualifier + )); + } + } + for o in &ocel.objects { + out.push(format!("object {:?} of {:?}", o.id, o.object_type)); + for a in &o.attributes { + out.push(format!("object {:?} {:?} = {:?}", o.id, a.name, a.value)); + } + } + out.sort(); + out +} + +fn through_csv(ocel: &OCEL) -> OCEL { + let text = export_ocel_csv_to_string(ocel).expect("csv export"); + import_ocel_csv(text.as_bytes()).unwrap_or_else(|e| panic!("csv import: {e}\n---\n{text}")) +} + +#[cfg(feature = "ocel-bundle")] +fn through_bundle(ocel: &OCEL, label: &str) -> OCEL { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join(label); + std::fs::create_dir_all(&target).expect("mkdir"); + export_ocel_bundle( + ocel, + &target, + BundleExportOptions { + layout: ContainerLayout::Directory, + storage: if cfg!(feature = "ocel-bundle-parquet") { + StorageFormat::Parquet + } else { + StorageFormat::Csv + }, + }, + ) + .expect("bundle export"); + import_ocel_bundle(&target).expect("bundle import") +} + +/// Text that has no special meaning to either format, so both must give it back untouched. +const ORDINARY: &[(&str, &str)] = &[ + ("comma", "a,b"), + ("quote", "a\"b"), + ("newline", "a\nb"), + ("tab", "a\tb"), + ("unicode", "Ünïcödé"), + ("percent", "a%2Fb"), + ("semicolon", "a;b"), +]; + +/// The characters that structure an `ot:` cell. Written raw, `a/b` reads back as two +/// references, `a#b` as a truncated id, and `a{b` as broken JSON, so the exporter escapes them. +const RESERVED: &[(&str, &str)] = &[ + ("slash", "a/b"), + ("hash", "a#b"), + ("brace", "a{b"), + ("backslash", "a\\b"), + ("all four", "a/b#c{d\\e"), + ("only separators", "/#{"), +]; + +#[test] +fn the_csv_gives_back_ordinary_text_unchanged() { + for (label, text) in ORDINARY { + let src = log_named(text); + assert_eq!(facts(&src), facts(&through_csv(&src)), "{label}"); + } +} + +#[test] +fn the_csv_gives_back_its_own_reserved_characters() { + for (label, text) in RESERVED { + let src = log_named(text); + assert_eq!(facts(&src), facts(&through_csv(&src)), "{label}"); + } +} + +/// Column names are arbitrary text too, so recognising the `ot:`/`ea:` prefixes must not assume +/// the header opens with single-byte characters. +#[test] +fn a_header_that_opens_with_multi_byte_text_is_read_not_split() { + let csv = "id,activity,timestamp,Ünïcödé,ot:日本\n\ + e1,open,2024-01-01T10:00:00+0000,vÄ,o1"; + let ocel = import_ocel_csv(csv.as_bytes()).expect("import"); + assert_eq!(ocel.objects[0].object_type, "日本"); + assert_eq!(ocel.events[0].attributes[0].name, "Ünïcödé"); + assert_eq!( + ocel.events[0].attributes[0].value, + OCELAttributeValue::String("vÄ".into()) + ); +} + +/// A file written before the escape existed still reads the same way, because a backslash only +/// escapes the three structural characters and itself. +#[test] +fn an_unescaped_backslash_in_a_hand_written_file_is_still_a_backslash() { + let csv = "id,activity,timestamp,ot:file\n\ + e1,open,2024-01-01T10:00:00+0000,C:\\Users\\me"; + let ocel = import_ocel_csv(csv.as_bytes()).expect("import"); + assert_eq!(ocel.objects.len(), 1); + assert_eq!(ocel.objects[0].id, "C:\\Users\\me"); +} + +/// The format has no schema, so every value is retyped on the way in. A parse that cannot be +/// undone would lose the text silently, so these keep their string form. +#[test] +fn text_that_only_looks_numeric_stays_text() { + for value in [ + "007", // leading zeros are not the number 7 + "+7", // the sign is not in the canonical spelling + "1e3", // nor is an exponent + "1_2", // nor a separator + "0x1f", // nor a radix prefix + "5.", // nor a bare point + ".5", // nor a missing whole part + "-0", // renders as `0`, so it would not come back + "123456789012345678901234567890", // beyond i64: an f64 would round it + "2024-01-01", // a date with no timezone is not an instant + "2022-05-04 05:57:00", // nor is a local wall clock + // Short enough that a fixed-width look at the tail lands mid-character. + "Ünïcödé", + "aÄ", + "日本", + "aaaa̋", + ] { + let src = log_valued(value); + let back = through_csv(&src); + assert_eq!( + back.events[0].attributes[0].value, + OCELAttributeValue::String(value.to_string()), + "{value:?} should have stayed a string" + ); + } +} + +/// The other half of the same rule: text that is the canonical spelling of a number or an +/// instant is meant to be read as one. +#[test] +fn text_that_is_a_number_is_read_as_one() { + let cases: BTreeMap<&str, OCELAttributeValue> = [ + ("7", OCELAttributeValue::Integer(7)), + ("-12", OCELAttributeValue::Integer(-12)), + ("0", OCELAttributeValue::Integer(0)), + ("0.5", OCELAttributeValue::Float(0.5)), + ("-0.5", OCELAttributeValue::Float(-0.5)), + ("-12.75", OCELAttributeValue::Float(-12.75)), + // Trailing zeros are formatting, not value. + ("5.00", OCELAttributeValue::Float(5.0)), + ("true", OCELAttributeValue::Boolean(true)), + // The format compares the boolean words without regard to case. + ("TRUE", OCELAttributeValue::Boolean(true)), + ] + .into_iter() + .collect(); + + for (text, expected) in cases { + let src = log_valued(text); + let back = through_csv(&src); + assert_eq!( + back.events[0].attributes[0].value, expected, + "{text:?} should have been read as {expected:?}" + ); + } +} + +/// An `ot:` header names the exact object type, and an event attribute value is preserved +/// apart from RFC 4180 unquoting, so neither may be tidied. +#[test] +fn the_csv_keeps_the_whitespace_the_format_tells_it_to_keep() { + let mut src = log_named("x"); + src.object_types[0].name = "ob trailing ".into(); + src.objects[0].object_type = "ob trailing ".into(); + src.events[0].attributes[0].value = OCELAttributeValue::String(" padded ".into()); + + let back = through_csv(&src); + assert_eq!(back.objects[0].object_type, "ob trailing "); + assert_eq!( + back.events[0].attributes[0].value, + OCELAttributeValue::String(" padded ".into()) + ); +} + +/// The format says an id, an activity and a qualifier are read after their surrounding +/// whitespace is removed, so a name that ends in a space is not representable there. +#[test] +fn the_csv_trims_the_three_things_the_format_says_it_may() { + let mut src = log_named("x"); + src.events[0].id = "e trailing ".into(); + src.events[0].event_type = "ev trailing ".into(); + src.event_types[0].name = "ev trailing ".into(); + src.events[0].relationships[0].qualifier = "q trailing ".into(); + + let back = through_csv(&src); + assert_eq!(back.events[0].id, "e trailing"); + assert_eq!(back.events[0].event_type, "ev trailing"); + assert_eq!(back.events[0].relationships[0].qualifier, "q trailing"); +} + +/// An o2o row naming a source the file never gave a type to is dropped. `strict` turns that +/// silent loss into an error on its own, without `verbose` also being set. +#[test] +fn strict_rejects_an_unknown_o2o_source_without_needing_verbose() { + let csv = "id,activity,timestamp,ot:item\n\ + ghost,o2o,,i1#has"; + let strict = process_mining::core::event_data::object_centric::ocel_csv::OCELCSVImportOptions { + strict: true, + verbose: false, + ..Default::default() + }; + let err = + process_mining::core::event_data::object_centric::ocel_csv::import_ocel_csv_with_options( + csv.as_bytes(), + &strict, + ); + assert!(err.is_err(), "strict should reject an unknown o2o source"); + + // Without `strict` it stays a skip, so the rest of the file still reads. + let lenient = import_ocel_csv(csv.as_bytes()).expect("lenient import"); + assert!(lenient.objects.iter().all(|o| o.relationships.is_empty())); +} + +/// An empty cell is a missing value, so an attribute whose value is the empty string cannot be +/// told apart from one that is absent. +#[test] +fn an_empty_string_attribute_does_not_survive_the_csv() { + let src = log_valued(""); + let back = through_csv(&src); + assert!(back.events[0].attributes.is_empty()); +} + +#[cfg(feature = "ocel-bundle")] +#[test] +fn the_bundle_gives_everything_back_exactly() { + for (label, text) in ORDINARY.iter().chain(RESERVED) { + let src = log_named(text); + assert_eq!( + facts(&src), + facts(&through_bundle(&src, "names")), + "{label}" + ); + } + + for text in ["trailing ", " leading", " ", "007", ""] { + let mut src = log_named("x"); + src.events[0].attributes[0].value = OCELAttributeValue::String(text.to_string()); + src.objects[0].attributes[0].value = OCELAttributeValue::String(text.to_string()); + assert_eq!( + facts(&src), + facts(&through_bundle(&src, "values")), + "value {text:?}" + ); + } + + // The whitespace the CSV is required to trim, which the bundle has no reason to touch. + let mut src = log_named("x"); + src.events[0].id = "e trailing ".into(); + src.events[0].event_type = "ev trailing ".into(); + src.event_types[0].name = "ev trailing ".into(); + src.events[0].relationships[0].qualifier = "q trailing ".into(); + assert_eq!(facts(&src), facts(&through_bundle(&src, "whitespace"))); +} diff --git a/r4pm/Cargo.toml b/r4pm/Cargo.toml index 704352be..181a47ab 100644 --- a/r4pm/Cargo.toml +++ b/r4pm/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://rust4pm.aarkue.eu" repository = "https://github.com/aarkue/rust4pm" [dependencies] -process_mining = { version = "0.6.0", path = "../process_mining", features = ["bindings", "ocel-sqlite", "graphviz-export"] } +process_mining = { version = "0.6.0", path = "../process_mining", features = ["bindings", "ocel-sqlite", "graphviz-export", "ocel-duckdb"] } serde_json = "1.0.105" serde = { version = "1.0.188", features = ["derive"] } anstyle = "1.0.13" From 0fb95adb6f00e5f92bd4d793e31413128bc591e9 Mon Sep 17 00:00:00 2001 From: aarkue Date: Thu, 20 Aug 2026 16:04:30 +0200 Subject: [PATCH 2/3] Allow for non-bundled duckdb and use that in CI --- .github/workflows/test.linux.yml | 32 +++++++++++++++++++++++++++----- Cargo.lock | 4 ++-- process_mining/Cargo.toml | 19 ++++++++----------- r4pm/Cargo.toml | 2 +- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.linux.yml b/.github/workflows/test.linux.yml index d0beb80c..a6ff4f41 100644 --- a/.github/workflows/test.linux.yml +++ b/.github/workflows/test.linux.yml @@ -8,6 +8,7 @@ on: env: CARGO_TERM_COLOR: always + FEATURES: graphviz-export,ocel-sqlite,ocel-duckdb,extraction-blueprint,extraction-dbcon,extraction-dbcon-postgres,extraction-dbcon-duckdb,ocel-bundle,ocel-bundle-parquet,dataframes,log-splitting,token-based-replay,bindings jobs: Test: @@ -19,27 +20,48 @@ jobs: - name: Install Rust run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source "$HOME/.cargo/env" + source "$HOME/.cargo/env" - name: Install Graphviz run: sudo apt-get update && sudo apt-get -y install graphviz p7zip-full + - name: Download DuckDB + run: wget -O libduckdb-linux-amd64.zip https://github.com/duckdb/duckdb/releases/download/$DUCKDB_VERSION/libduckdb-linux-amd64.zip + env: + DUCKDB_VERSION: v1.3.2 + - name: Unpacking duckdb + run: 7z x ${{ github.workspace }}/libduckdb-linux-amd64.zip -o${{ github.workspace }}/libduckdb - name: Downloading test files run: wget -O process_mining/test_data/out.zip https://rwth-aachen.sciebo.de/s/4cvtTU3lLOgtxt1/download - name: Unpacking test files run: 7z x process_mining/test_data/out.zip -oprocess_mining/test_data - name: Build - run: source "$HOME/.cargo/env" && cargo build --verbose --all-features + run: source "$HOME/.cargo/env" && cargo build --verbose --features "$FEATURES" working-directory: ./process_mining + env: + DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb + DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb + LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - name: Clippy working-directory: ./process_mining - run: source "$HOME/.cargo/env" && cargo clippy --all-targets --all-features -- -D warnings + run: source "$HOME/.cargo/env" && cargo clippy --all-targets --features "$FEATURES" -- -D warnings + env: + DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb + DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb + LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb - name: Check formatting working-directory: ./process_mining run: source "$HOME/.cargo/env" && cargo fmt --all --check - name: Check docs working-directory: ./process_mining - run: source "$HOME/.cargo/env" && cargo doc --all-features --no-deps + run: source "$HOME/.cargo/env" && cargo doc --features "$FEATURES" --no-deps env: + DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb + DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb + LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb RUSTDOCFLAGS: -D warnings - name: Run tests - run: source "$HOME/.cargo/env" && cargo test --verbose --all-features + run: source "$HOME/.cargo/env" && cargo test --verbose --features "$FEATURES" working-directory: ./process_mining + env: + DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb + DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb + LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb diff --git a/Cargo.lock b/Cargo.lock index 0586b220..67a61b12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,9 +1103,9 @@ dependencies = [ [[package]] name = "dbcon" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a032a3800982b3848bfcfe8ce674f5c299507deccc1d4697c514be41b4370210" +checksum = "e8a30c7fecab2cccb75a38cf2fe1a75ab8e203f70277d086740881a92893b3ca" dependencies = [ "anyhow", "bytes", diff --git a/process_mining/Cargo.toml b/process_mining/Cargo.toml index 2453d913..07ab4f4b 100644 --- a/process_mining/Cargo.toml +++ b/process_mining/Cargo.toml @@ -16,11 +16,7 @@ rust-version = "1.88" [dependencies] macros_process_mining = { version = "0.6.0", path = "../macros_process_mining" } chrono = { version = "0.4.40", features = ["serde"] } -# `bundled` builds DuckDB from source, as `rusqlite` above does for SQLite: no system -# `libduckdb` to install, no version to keep in step with this dependency, and the result is -# statically linked, so nothing has to be findable at run time. Costs a few minutes and ~3 GB -# on a clean build, and needs a C++ compiler. -duckdb = { version = "1.2.1", optional = true, features = ["chrono", "bundled"] } +duckdb = { version = "1.2.1", optional = true, features = ["chrono"] } flate2 = "1.1.1" graphviz-rust = { version = "0.9.3", optional = true } itertools = { version = "0.14.0" } @@ -47,7 +43,7 @@ csv = "1.4.0" # Backends are selected by this crate's own `extraction-dbcon*` features, since not all of them are # portable. `dbcon`'s `default` is `[]`, and a `dbcon` with no backend recognises no connection # string, so at least one must be selected. -dbcon = {version = "0.4.0", optional = true } +dbcon = {version = "0.4.1", optional = true } hashbrown = "0.17.1" rustc-hash = "2.1.2" zip = { version = "6", default-features = false, features = ["deflate"], optional = true } @@ -75,6 +71,7 @@ graphviz-export = ["dep:graphviz-rust"] # Note: this might not work on certain architectures or machines if SQLite is not available/cannot be build ocel-sqlite = ["dep:rusqlite"] ocel-duckdb = ["dep:duckdb"] +ocel-duckdb-bundled = ["ocel-duckdb", "duckdb/bundled"] # Enables the relational-to-OCEL extraction blueprint model, validation, SQL compiler, extractor # and sinks. Carries no connector of its own: `ocel-sqlite` brings `SqliteRowProvider`, and @@ -103,9 +100,9 @@ extraction-dbcon = [ extraction-dbcon-postgres = ["extraction-dbcon", "dbcon/postgres"] # Adds DuckDB to `extraction-dbcon`. Separate for the same reason as PostgreSQL: the `duckdb` crate -# builds native code and does not build for `wasm32`. `dbcon` bundles DuckDB just as this crate -# does, so both paths to the shared `duckdb` crate agree on how it is built. +# builds native code and does not build for `wasm32`. extraction-dbcon-duckdb = ["extraction-dbcon", "dbcon/duckdb"] +extraction-dbcon-duckdb-bundled = ["extraction-dbcon-duckdb", "dbcon/duckdb-bundled"] # Enables reading and writing the OCEL 2.0 bundled CSV/Parquet format: a `.ocel.zip` archive or # a directory with the same layout, with CSV storage. @@ -130,7 +127,7 @@ bindings = [ "dep:base64" ] -all = ["graphviz-export","ocel-sqlite","ocel-duckdb","extraction-blueprint","extraction-dbcon","extraction-dbcon-postgres","extraction-dbcon-duckdb","ocel-bundle","ocel-bundle-parquet","dataframes", "log-splitting", "token-based-replay", "bindings" ] +all = ["graphviz-export","ocel-sqlite","ocel-duckdb-bundled","extraction-blueprint","extraction-dbcon","extraction-dbcon-postgres","extraction-dbcon-duckdb-bundled","ocel-bundle","ocel-bundle-parquet","dataframes", "log-splitting", "token-based-replay", "bindings" ] [package.metadata.docs.rs] # The same set as `all`, listed explicitly so a feature docs.rs cannot build can be dropped @@ -138,11 +135,11 @@ all = ["graphviz-export","ocel-sqlite","ocel-duckdb","extraction-blueprint","ext features = [ "graphviz-export", "ocel-sqlite", - "ocel-duckdb", + "ocel-duckdb-bundled", "extraction-blueprint", "extraction-dbcon", "extraction-dbcon-postgres", - "extraction-dbcon-duckdb", + "extraction-dbcon-duckdb-bundled", "ocel-bundle", "ocel-bundle-parquet", "dataframes", diff --git a/r4pm/Cargo.toml b/r4pm/Cargo.toml index 181a47ab..b8d0ab4b 100644 --- a/r4pm/Cargo.toml +++ b/r4pm/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://rust4pm.aarkue.eu" repository = "https://github.com/aarkue/rust4pm" [dependencies] -process_mining = { version = "0.6.0", path = "../process_mining", features = ["bindings", "ocel-sqlite", "graphviz-export", "ocel-duckdb"] } +process_mining = { version = "0.6.0", path = "../process_mining", features = ["bindings", "ocel-sqlite", "graphviz-export", "ocel-duckdb-bundled"] } serde_json = "1.0.105" serde = { version = "1.0.188", features = ["derive"] } anstyle = "1.0.13" From 2bad4a24b3bd2bee6eb64421f3dc31c201b3b450 Mon Sep 17 00:00:00 2001 From: aarkue Date: Thu, 20 Aug 2026 16:39:51 +0200 Subject: [PATCH 3/3] Add missing binding registrations for OC-DECLARE --- process_mining/src/discovery/object_centric/oc_declare/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process_mining/src/discovery/object_centric/oc_declare/mod.rs b/process_mining/src/discovery/object_centric/oc_declare/mod.rs index 0e7344e8..4f739dca 100644 --- a/process_mining/src/discovery/object_centric/oc_declare/mod.rs +++ b/process_mining/src/discovery/object_centric/oc_declare/mod.rs @@ -660,6 +660,7 @@ fn get_direct_or_indirect_object_involvements<'a>( /// Reduce OC-DECLARE arcs based on lossless/lossy transitive reduction /// considering paths of arbitrary length. /// Uses sequential processing to prevent mutual elimination in cycles. +#[register_binding] pub fn reduce_oc_arcs(mut arcs: Vec, lossless: bool) -> Vec { // Sorting ensures deterministic processing order arcs.sort(); @@ -810,6 +811,7 @@ fn compose_arc_labels(l1: &OCDeclareArcLabel, l2: &OCDeclareArcLabel) -> OCDecla /// /// BFSs from every target activity through non-target intermediaries, composing /// arc types and labels along the way. +#[register_binding] pub fn project_oc_arcs( arcs: Vec, activities: &HashSet,