diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..e35066abd7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!build/flow.phar diff --git a/Dockerfile b/Dockerfile index 2256b4e21c..07474868fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,21 @@ -# Stage 1: Build stage -ARG FLOW_PHP_VERSION=8.3.28 -ARG FLOW_BASE_IMAGE_TAG_SUFFIX=cli-bookworm -ARG FLOW_BASE_IMAGE_TAG=${FLOW_PHP_VERSION}-${FLOW_BASE_IMAGE_TAG_SUFFIX} -ARG FLOW_BASE_IMAGE=php:${FLOW_BASE_IMAGE_TAG} +ARG FLOW_PHP_VERSION=8.5.8 +ARG FLOW_DEBIAN_SUITE=trixie +ARG FLOW_BASE_IMAGE=php:${FLOW_PHP_VERSION}-cli-${FLOW_DEBIAN_SUITE} -FROM ${FLOW_BASE_IMAGE} AS builder +# Stage 1: Shared base so builder and final stage resolve libpq from the same apt source. +FROM ${FLOW_BASE_IMAGE} AS base + +ARG FLOW_DEBIAN_SUITE +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates gnupg curl \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/pgdg.gpg] http://apt.postgresql.org/pub/repos/apt ${FLOW_DEBIAN_SUITE}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get purge -y --auto-remove gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Stage 2: Build stage +FROM base AS builder # Install dependencies and PHP extensions RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -18,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libgmp-dev \ libpq-dev \ libsqlite3-dev \ + libzip-dev \ default-libmysqlclient-dev \ libprotobuf-dev \ libprotobuf-c-dev \ @@ -26,7 +38,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ clang \ libclang-dev \ && rm -rf /var/lib/apt/lists/* \ - && docker-php-ext-install bcmath gmp pdo_mysql pdo_pgsql pdo_sqlite \ + && docker-php-ext-install bcmath gmp pdo_mysql pdo_pgsql pdo_sqlite pgsql zip \ + && pecl install protobuf \ + && docker-php-ext-enable protobuf \ && curl -L https://github.com/php/pie/releases/latest/download/pie.phar -o /usr/local/bin/pie \ && chmod +x /usr/local/bin/pie \ && php /usr/local/bin/pie install kjdev/brotli \ @@ -38,15 +52,17 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" RUN php /usr/local/bin/pie install flow-php/arrow-ext:1.x-dev \ - && php /usr/local/bin/pie install flow-php/pg-query-ext:1.x-dev + && php /usr/local/bin/pie install flow-php/pg-query-ext:1.x-dev \ + && php /usr/local/bin/pie install flow-php/flow-php-ext:1.x-dev -# Stage 2: Final Image -FROM ${FLOW_BASE_IMAGE} AS flow +# Stage 3: Final Image +FROM base AS flow # Install runtime libraries needed by extensions RUN apt-get update && apt-get install -y --no-install-recommends \ libpq5 \ libgmp10 \ + libzip5 \ && rm -rf /var/lib/apt/lists/* # Copy the built extensions from the builder stage diff --git a/documentation/components/extensions/arrow-ext.md b/documentation/components/extensions/arrow-ext.md index 659573553b..7304a650a6 100644 --- a/documentation/components/extensions/arrow-ext.md +++ b/documentation/components/extensions/arrow-ext.md @@ -13,7 +13,7 @@ The Arrow ecosystem provides high-performance implementations for common data op Parquet, CSV, JSON, and Arrow IPC — in C++, Rust, Java, Python, and other languages. This extension brings the [Arrow Rust ecosystem](https://github.com/apache/arrow-rs) into PHP via -[ext-php-rs](https://github.com/davidcole1340/ext-php-rs). It exposes Arrow's native readers and writers through +[ext-php-rs](https://github.com/extphprs/ext-php-rs). It exposes Arrow's native readers and writers through PHP streaming interfaces, letting PHP applications benefit from Rust-level performance without leaving the PHP runtime. > [!TIP] @@ -316,7 +316,7 @@ make test ## Architecture -- Built with [ext-php-rs](https://github.com/davidcole1340/ext-php-rs), which generates PHP bindings from Rust code +- Built with [ext-php-rs](https://github.com/extphprs/ext-php-rs), which generates PHP bindings from Rust code - Uses Apache Arrow and Parquet Rust crates from the [Arrow ecosystem](https://github.com/apache/arrow-rs) - All compression codecs compiled into the extension — no external PHP compression extensions needed - PHP streaming interfaces (`RandomAccessFile`, `OutputStream`) called from Rust via ext-php-rs callbacks @@ -326,6 +326,6 @@ make test ## See Also - [parquet library](/documentation/components/libs/parquet.md) -- [ext-php-rs](https://github.com/davidcole1340/ext-php-rs) +- [ext-php-rs](https://github.com/extphprs/ext-php-rs) - [Apache Arrow Rust](https://github.com/apache/arrow-rs) - [Nix Development Environment](/documentation/contributing/nix.md) diff --git a/documentation/contributing/wasm.md b/documentation/contributing/wasm.md index da5d4dfbd3..ca9746849a 100644 --- a/documentation/contributing/wasm.md +++ b/documentation/contributing/wasm.md @@ -7,8 +7,13 @@ This document describes how to build PHP to WebAssembly (WASM) for use in the Fl ## Overview The WASM build creates a browser-compatible PHP runtime that powers the interactive playground -at [flow-php.com](https://flow-php.com). It compiles PHP 8.4 with the necessary extensions to run Flow PHP ETL pipelines -directly in the browser. +at [flow-php.com](https://flow-php.com). It compiles PHP 8.5.8 with the necessary extensions to run Flow PHP ETL +pipelines directly in the browser. The version is kept in step with the Docker image +(`Dockerfile` `FLOW_PHP_VERSION`) so both distributions ship the same PHP. + +The build targets **64-bit** wasm (`-sMEMORY64=2`), so the playground has the same `PHP_INT_MAX` as +every other distribution. See [64-bit Environment](#64-bit-environment) and +[Why MEMORY64=2 and not MEMORY64=1](#why-memory642-and-not-memory641). ## Development Setup @@ -23,18 +28,36 @@ This provides all necessary dependencies including Emscripten, autoconf, cmake, Build the WASM binary: ```bash -nix-shell --arg with-wasm true --run "cd wasm && ./build.sh" +nix-shell --arg with-wasm true --run "just wasm" ``` +Use `just wasm` rather than calling `wasm/build.sh` directly. The recipe also rebuilds `flow.phar` +into `web/landing/assets/wasm/tools/`, and the playground loads that phar — building the runtime on +its own leaves a new PHP paired with a stale library. + The build script will: 1. Download and compile libxml2 for WebAssembly 2. Download and compile libpg_query for WebAssembly 3. Download PHP source -4. Copy the pg_query and snappy extensions -5. Configure and compile PHP with required extensions -6. Link everything into `php.wasm` and `php.js` -7. Copy outputs to `web/landing/assets/wasm/` +4. Apply the php-src patches in `wasm/patches/` +5. Copy the pg_query and snappy extensions +6. Configure and compile PHP with required extensions +7. Link everything into `php.wasm` and `php.js` +8. Copy outputs to `web/landing/assets/wasm/` + +Each dependency is skipped when its build output is already present, so re-runs are cheap. Bumping +a version changes the directory name (or, for libpg_query and libzip, invalidates the archive +check), which is what triggers the rebuild. + +### libpg_query version + +`build.sh` does not pin its own libpg_query version. It reads `LIBPG_QUERY_VERSION` from +`src/extension/pg-query-ext/Makefile`, so the playground and the native extension can never +disagree about which PostgreSQL grammar parses, and fails fast if that variable cannot be read. + +The pin is a **branch** (`18-latest`), not a tag. Changing it re-clones automatically; upstream +moving the branch under a fixed version does not, and needs `rm -rf wasm/libpg_query-*`. ## Included PHP Extensions @@ -52,6 +75,7 @@ The build script will: | dom | DOM manipulation | | xmlreader | XML streaming reader | | xmlwriter | XML streaming writer | +| zip | Required by flow-php/etl-adapter-excel (XLSX files are ZIP archives) | | pg_query | PostgreSQL query parsing (Flow PHP extension) | | snappy | Snappy compression for Parquet | @@ -66,20 +90,338 @@ These are copied to `web/landing/assets/wasm/` for use by the playground. ## Architecture Notes -### 32-bit Environment +### Opcache is always compiled in, and is deliberately switched on + +PHP 8.5 made Opcache non-optional +([RFC](https://wiki.php.net/rfc/make_opcache_required), php-src `7b4c14dc1016`). The +`PHP_ARG_ENABLE([opcache], ...)` that `--disable-all` used to switch off is gone; `config.m4` now +calls `PHP_NEW_EXTENSION([opcache], ...)` unconditionally. On 8.4 `--disable-all` excluded Opcache +silently, which is why none of this was written down before. + +Two configure flags exist because of that, and **neither is redundant next to `--disable-all`** — +both pass `[no]` as their 5th `PHP_ARG_ENABLE` argument, so `--disable-all` cannot reach either: + +| Flag | Why | +|------|-----| +| `--disable-opcache-jit` | Opcache's JIT is gated on `$host_cpu`. `emconfigure` does not pass `--host`, so `config.guess` reports the **build** machine, and `config.m4` accepts every common one (`x86*`, `aarch64`, `amd64`) as JIT-capable. Without this flag `emcc` is handed `jit/ir/*.c` plus dynasm for the host ISA to compile into a wasm32 binary. Same class of workaround as `--disable-fiber-asm`. | +| `--disable-huge-code-pages` | Copying code pages into huge pages is meaningless under wasm. | + +**Do not delete `--disable-opcache-jit` because it looks redundant.** It is not. + +#### The shared-memory backend has to be forced on + +Left alone, Opcache builds but can never run. Each of its three shared-memory probes compiles a +program that calls `fork()`, which Emscripten does not implement, so all three fail and configure +defines `NO_SHM_BACKEND`: + +``` +checking for sysvipc shared memory support... no +checking for mmap() using MAP_ANON shared memory support... no +checking for mmap() using shm_open() shared memory support... no +configure: WARNING: No supported shared memory caching support was found when configuring opcache. +Opcache will be disabled. +``` + +Since php-src `e4078a6a70d5` that is a warning rather than a build failure, and at runtime +`NO_SHM_BACKEND` disables Opcache while PHP still starts. The backend sources +(`shared_alloc_mmap.c`, `shared_alloc_posix.c`, `shared_alloc_shm.c`) are each wholly inside +`#ifdef USE_MMAP` / `USE_SHM_OPEN` / `USE_SHM`, so with none defined they compile to nothing. + +`build.sh` therefore pre-sets the autoconf cache variable before `./configure`: + +```bash +export php_cv_shm_mmap_anon=yes +``` + +which turns the probe into `checking for mmap() using MAP_ANON shared memory support... (cached) yes` +and defines `HAVE_SHM_MMAP_ANON` → `USE_MMAP`. Only that one is forced; the other two stay `no`. The +three checks are siblings rather than nested (`ext/opcache/config.m4:176-234`), and `USE_MMAP` alone +is enough. + +**This is honest, not a lie to the build system.** The probe fails only on its `fork()` half. What +Opcache actually needs at runtime, it gets: + +- `mmap(MAP_SHARED|MAP_ANONYMOUS)` — Emscripten's `__syscall_mmap2` takes an anonymous branch that + **ignores `MAP_SHARED`** and serves the mapping out of linear memory + (`system/lib/libc/emscripten_mmap.c`). A single-process runtime needs no actual sharing. +- `fcntl(F_SETLK/F_SETLKW)` — Emscripten returns success by design, commenting that these are + process-level locks and a wasm program is one process (`src/lib/libsyscall.js`). +- the lock file — no `memfd_create` or `O_TMPFILE`, so it falls through to `mkstemp()` under + `opcache.lockfile_path`, and MEMFS provides `/tmp` by default. + +All three were verified with a standalone `emcc` program *before* the PHP rebuild rather than +discovered after it — worth repeating on any toolchain bump. + +#### Why this reverses the earlier decision + +An earlier revision of this file recorded the opposite decision: Opcache was left inert because "the +playground evaluates a fresh snippet per run, so a real opcode cache buys nothing." The first half of +that is true and the second does not follow. The snippet is fresh; the **3.7 MB `flow.phar` behind it +is not**, and re-compiling that class graph on every Run was the entire cost. Measured, with one PHP +request per Run: + +| | leak per Run | Runs before the page died | +|---|---|---| +| no Opcache | 2.36 MB | 813 | +| Opcache with a working SHM backend | **0.00 MB** | no death in 5,000 | + +It is also **28% faster**, not slower: a CSV → JSON → read-back workload went from a 319 ms to a +228 ms median once the phar stopped being recompiled on every Run. The artifact grew 0.03%. + +Opcache only pays off *on top of* the one-module-per-page lifecycle (see below), because its shared +memory is created at `MINIT` — under the old one-module-per-Run model it was rebuilt every Run, +which is very likely why it looked worthless. + +#### Settings, and why they are set in C + +`opcache.memory_consumption` is read once at `MINIT`, so it has to be in place before +`php_embed_init()`. `php_embed_init()` overwrites `php_embed_module.ini_entries` with its own +`HARDCODED_INI`, so the only usable hook is `php_embed_module.ini_defaults`, which is what +`pib_eval.c` sets. There is no `php.ini` in this build. + +`opcache.validate_timestamps=0`: the phar is written once by the loader and never changes for the +life of the page, and the one file that does change (`code.php`, rewritten by the Format action) is +never `require`d — snippets are compiled through `zend_compile_string`, which Opcache does not cache +either way. So the per-request `stat()` of every cached file buys nothing. + +The SHM is `mmap`ed out of wasm linear memory at `MINIT` and never returned, so +`opcache.memory_consumption` is a permanent per-page cost paid by every visitor, including ones who +never press Run. Size it from measurement, not from the upstream default. + +`opcache.enable_cli` is irrelevant here: `accel_sapi_is_cli()` matches only `cli` and `phpdbg`, and +this SAPI is named `embed`. + +### One module lifecycle per page, one request lifecycle per Run + +`pib_eval()` (`wasm/pib_eval.c`) calls `php_embed_init()` **once**, on the first Run of a page, and +then cycles only the request around every evaluation: + +``` +first Run only : php_embed_init() = sapi_startup -> MINIT -> php_request_startup +every Run : php_request_shutdown() ; php_request_startup() +never : php_embed_shutdown() +``` + +The embed SAPI splits cleanly along exactly those lines (`sapi/embed/php_embed.c`). This is what +every web SAPI does; the old model — a full `php_embed_init()`/`php_embed_shutdown()` per Run — is +the exotic one, and it cost the page ~36 MB of unreclaimable wasm memory per Run. + +**Per-Run isolation is preserved**, because user-defined classes, functions, constants and globals +are all request-scoped and request shutdown still frees them. Verified case by case against the old +artifact rather than assumed — class and function redeclaration, `$GLOBALS`, `static` variables, +`ini_set`, `declare(strict_types=1)`, autoloader registrations, unclosed file handles, `exit()`, +`register_shutdown_function`, an uncaught throwable, and output bleed all behave identically to +before. + +Two ordering details are load-bearing: + +- The request is cycled at the **end** of `pib_eval()`, not lazily at the start of the next one, so + shutdown functions and destructors are attributed to the Run that caused them. +- `php_request_shutdown()` runs **before** the `fflush()` pair, because it is what emits + `register_shutdown_function` output. + +Both `php_request_startup()` and `php_request_shutdown()` wrap their bodies in `zend_try` +internally, so neither can bail out past `pib_eval()`'s own `zend_first_try`. -WebAssembly runs as a 32-bit environment where `PHP_INT_MAX = 2147483647`. This affects: +`php_embed_shutdown()` is now never called: a browser page has no teardown hook, and the controller's +recycling path (below) discards the whole wasm instance, which reclaims everything. That is +deliberate — calling it would run `MSHUTDOWN`, and see the next paragraph for why that is a hazard +worth not courting. -- Large integer values from Thrift/Parquet metadata may overflow to floats -- The parquet library includes `(int)` casts in `fromThrift()` methods to handle this -- Hex constants like `0x80000000` and `0xFFFFFFFF` require explicit `(int)` casts +#### `MSHUTDOWN` now runs at most once — do not take that as licence + +Under the old model a non-re-entrant `MINIT`/`MSHUTDOWN` pair was fatal, and it failed in a way that +pointed nowhere near the cause: Run 1 succeeded, Run 2 died with `null function` or +`memory access out of bounds` — a wasm indirect call through a freed function pointer, with no +PHP-level stack trace. + +This already happened once. `pg_query`'s `MSHUTDOWN` called `pg_query_exit()`, which frees +libpg_query's `TopMemoryContext` while `pg_query_init()` guards on a flag it never resets — so the +second startup ran against freed memory. The fix was to not free at all (see the comment in +`src/extension/pg-query-ext/ext/pg_query.c`). Note that resetting the flag is *not* sufficient: +`MemoryContextInit()` then re-runs over freed contexts, which survives a trivial snippet and fails +under a real pipeline. + +The lifecycle change makes that class of bug much harder to hit. **It is not a reason to restore +`pg_query_exit()`** — that is a separate decision with its own history, and the recycling path still +creates fresh modules within one page. + +**When adding an extension to this build, still test with a realistic workload** — load the phar and +run a pipeline across several Runs. A `` only under `ZEND_WIN32`. clang has treated implicit function declarations as an error since 16, so the build fails without it. On 8.4 this never surfaced, because the file was never compiled. | +| `php-8.5-emscripten-mm-chunk-alignment.patch` | Zend's chunk allocator aligns by over-allocating and then partially `munmap`ing, which Emscripten cannot do — so ~2 MB was orphaned per chunk, permanently. See [the memory leak section](#the-wasm-memory-leak-and-why-the-page-no-longer-has-a-run-budget). | + +Re-check both on the next PHP bump and delete either if it has landed upstream. + +### 64-bit Environment + +**The playground is 64-bit.** `PHP_INT_MAX === 9223372036854775807` and `PHP_INT_SIZE === 8`, +verified in the browser rather than inferred from build flags. This section used to document the +opposite; the history matters, because the workarounds it justified are still in the codebase. + +What is now true: + +- `pack()`/`unpack()` 64-bit format codes (`q`, `Q`, `J`, `P`) work. `Flow\Floe` therefore works, and + so does **`sortBy()`** and everything else that spills runs to disk through + `Flow\Floe\Encoding\Int64Encoder`. Before this, every Floe write in the browser died with + `ValueError: 64-bit format codes are not available for 32-bit versions of PHP`. +- Large integers from Thrift/Parquet metadata no longer overflow to float. On the 32-bit build, + reading a parquet file emitted hundreds of + `Warning: The float 2147483648 is not representable as an int` from + `Flow\Parquet\Thrift\CompactProtocol` and `Flow\ETL\Bucketing\HashBucketing`; the 64-bit build + emits none. +- Hex constants like `0x80000000` are plain integers and need no `(int)` cast. +- `DateTimeEncoder` packs `getTimestamp()`, which now represents dates past 2038-01-19. + +The `(int)` casts in the parquet library's `fromThrift()` methods are **left in place**. They are no +longer load-bearing for the playground, but the library still supports 32-bit PHP builds generally, +and removing them is a separate decision. + +### Why `MEMORY64=2` and not `MEMORY64=1` + +`build.sh` builds with `-sMEMORY64=2` (`WASM64_MODE`). Emscripten describes the three values as +wasm32 (`0`), full end-to-end wasm64 (`1`), and wasm64 for clang/lld **lowered to wasm32 by Binaryen** +(`2`). Only `2` gives PHP an 8-byte `zend_long` without demanding Memory64 support from the engine +running it: + +| | `MEMORY64=1` | `MEMORY64=2` | +|---|---|---| +| `sizeof(long)` | 8 | 8 | +| Engine requirement | Memory64 | **runs on today's wasm32 engines** | +| Node needed to run build-time probes | ≥ 23 | **works on the nix shell's node 22** | + +The node requirement is not a detail. PHP's `configure` reports `checking whether we are cross +compiling... no` under `emconfigure` and executes its `AC_RUN_IFELSE` probes *through node*, so with +`=1` PHP's own configure would fail or misdetect before anything else was reached. And since +flow-php.com is public, `=1` would additionally need Memory64 verified in Safari and Firefox. + +`=2` is sufficient because the playground does not need a >4 GB address space — +`MAXIMUM_MEMORY=2147483648` says so. **The memory settings are deliberately unchanged from the wasm32 +build**: `=2` lowers to a 32-bit memory, so the 2 GB ceiling is a hard limit of the mode rather than a +leftover. Raising it would require `=1`. + +Switching to `=1` is a deliberate decision, not a tuning step. + +### The target flag has to reach the dependencies, not just PHP + +`WASM64_MODE` is exported as **`EMCC_CFLAGS`**, near the top of `build.sh`, and this placement is +load-bearing: + +- `build.sh` exports `CFLAGS`/`CXXFLAGS`/`LDFLAGS` only *after* libxml2, libpg_query and libzip are + already built. Putting the flag there would build PHP for wasm64 against wasm32 dependency + archives, which fails at `wasm-ld` with a machine-type mismatch — long after the mistake. +- `EMCC_CFLAGS` is appended by `emcc` itself to every invocation, so it reaches all three + dependencies uniformly regardless of whether they are driven by autotools, cmake or a plain + Makefile. + +Two consequences worth knowing before changing any of this: + +- **Emscripten's sysroot is per-target.** wasm64 system libraries and ports are generated into + `sysroot/lib/wasm64-emscripten/` alongside the wasm32 ones, so the Emscripten cache does **not** + need clearing when switching targets. But the zlib port path must follow the target — hence + `$EM_TARGET` in `build.sh` rather than a hardcoded `wasm32-emscripten`. Hardcoding it finds a real + file of the *wrong* architecture, which the existing `[ ! -f ]` guard cannot detect. +- **Switching target does not invalidate the dependency guards.** Each guard tests its own build + output (`libxml2-2.11.4/` the source dir, `libzip-1.11.3/install/lib/libzip.a`, + `libpg_query-18-latest/libpg_query.a`), and none of them knows about the target. libzip + additionally caches the resolved zlib path in `build/CMakeCache.txt`, which is why `build.sh` + removes that directory before configuring. + +### The wasm memory leak, and why the page no longer has a Run budget + +This is resolved, but the mechanism is worth keeping written down: it is a genuine Emscripten/php-src +interaction, it is invisible without a diagnostic that used to be compiled out, and anything that +regresses the lifecycle or Opcache brings it straight back. + +**The symptom.** A tab executed a fixed number of Runs and then stopped executing PHP at all — every +later Run returned empty output, including ` 221 -> 459 -> 745 -> ... -> 2019 +run 46 2048 MB = MAXIMUM_MEMORY exactly + Fatal error: Out of memory (allocated 8388608 bytes) (tried to allocate 32768 bytes) +run 47+ RangeError: Maximum call stack size exceeded <- the silent state +``` + +PHP's own heap was only 8 MB at that point, so `memory_limit` was irrelevant — the exhausted resource +was the wasm heap, held outside PHP's request allocator. `MAXIMUM_MEMORY` cannot be raised under +`MEMORY64=2`, which lowers to a 32-bit memory. + +**The cause.** `zend_mm_chunk_alloc_int()` (`Zend/zend_alloc.c`) aligns a 2 MB chunk by mapping +`size + alignment - REAL_PAGE_SIZE` and then `munmap`ing the unwanted head and tail. Emscripten's +`munmap` cannot do that: `__syscall_munmap` looks the address up in its mapping list and returns +`EINVAL` unless the length matches the *whole* recorded mapping. So both trims fail and roughly one +`ZEND_MM_CHUNK_SIZE` of alignment slack is orphaned on every chunk allocation, permanently. + +`wasm/patches/php-8.5-emscripten-mm-chunk-alignment.patch` fixes it at the source: under +`__EMSCRIPTEN__` the chunk comes from `posix_memalign()` and goes back via `free()`, so there is +nothing to trim; `chunk_truncate`/`chunk_extend` report failure, which is the branch `_WIN32` has +always taken and which callers already handle by falling back to allocate-copy-free. + +**Why it took so long to find.** `build.sh` used to compile with `-DZEND_MM_ERROR=0`, which suppresses +the `munmap() failed: [%d] %s` that this code prints on exactly this failure (`Zend/zend_alloc.c`). +That flag is gone. Errno `28` is Emscripten's `EINVAL` — its errno table is not glibc's, so the +number looks like `ENOSPC` while the text reads "Invalid argument". **Do not re-add +`-DZEND_MM_ERROR=0`**; it costs the only signal this failure produces. + +**What each change bought**, measured over the same rotation of five real pipelines: + +| build | leak per Run | Runs before death | +|---|---|---| +| module lifecycle per Run, no Opcache | 36.2 MB | 53 | +| request lifecycle per Run, no Opcache | 2.36 MB | 813 | +| + Opcache with a working SHM backend | 0.00 MB | no death in 5,000 | + +Opcache takes it to zero by removing the demand rather than fixing the bug — with the phar compiled +once into shared memory, PHP's heap never grows past its warm set, so chunks stop being allocated. +The allocator patch is what makes that flat curve structural instead of contingent on Opcache never +being outgrown or disabled. + +**Measurement trap.** Wasm memory grows in ~98 MB steps, so per-Run figures from short samples are +quantization noise, not data — during the original investigation the same code measured 24.5 and then +16.3 MB/Run, and a curve that looked like it was converging at 382 MB turned out to be dead linear +over 600 Runs. Always report a total over >= 30 Runs, measure growth over the tail rather than the +whole span (early Runs carry one-off warm-up costs), and never call a plateau from fewer than a few +hundred flat Runs. + +### The controller recycles the module as a backstop + +`web/landing/assets/controllers/wasm_controller.js` reads the wasm heap size through the +`pib_heap_bytes()` export after each Run and, past a threshold, transparently tears the module down +and builds a fresh one, preserving `/workspace`. + +With the lifecycle split, Opcache and the allocator patch all in place this should never fire. It +exists so that any future regression degrades into a brief re-initialisation instead of the silent +unrecoverable death described above — that failure mode cost a full investigation to diagnose once, +and it should not be able to reach a user again. ### Wrapper Code -The `pib_eval.c` file provides the C wrapper for evaluating PHP code: +The `pib_eval.c` file provides the C wrapper for evaluating PHP code. These three are the whole +JS-callable surface — `EXPORTED_FUNCTIONS` in `build.sh` lists exactly them: -- `pib_eval(char *code)` - Evaluates PHP code and returns the result -- `pib_force_exit()` - Forces exit from the WASM runtime +- `pib_eval(char *code)` — evaluates PHP code and returns the result +- `pib_force_exit()` — forces exit from the WASM runtime +- `pib_heap_bytes()` — current wasm heap size, for the controller's recycling backstop. Returns a + `double` rather than `size_t` because under `MEMORY64` a `size_t` return is an `i64`, which + `ccall`'s `'number'` type cannot carry without BigInt handling on the JS side. + +`php_embed_init`, `php_embed_shutdown` and `zend_eval_string` used to be exported too. They were +never called from JS, and calling them now would corrupt the module's lifecycle state, so they are +not exported any more. ## Troubleshooting @@ -93,4 +435,11 @@ Ensure all required libraries (libxml2, libpg_query) are built before PHP config ### Memory issues in browser -The playground may fail on very large datasets due to WASM memory limits. +The playground may fail on very large datasets due to WASM memory limits. `MAXIMUM_MEMORY` is 2 GB +and cannot be raised — `MEMORY64=2` lowers to a 32-bit memory, so that ceiling is a property of the +mode, not a leftover setting. + +A *steadily climbing* footprint across Runs is a different problem and a regression: memory should be +flat once the phar is warm. Check that Opcache is actually running +(`opcache_get_status(false)` must return an array, not `false`) before looking anywhere else — that +is the single thing keeping the curve flat. diff --git a/documentation/installation/docker.md b/documentation/installation/docker.md index 90e8584d04..a406ce43cf 100644 --- a/documentation/installation/docker.md +++ b/documentation/installation/docker.md @@ -2,7 +2,8 @@ [DOC_LINK:../installation.md] -Since some of the Flow adapters require additional PHP extensions, we have prepared a Docker image with all the necessary dependencies. +Since some of the Flow adapters require additional PHP extensions, we have prepared a Docker image with all the +necessary dependencies. ```shell $ docker pull ghcr.io/flow-php/flow:latest @@ -42,11 +43,78 @@ Now you can use Flow CLI as follows: flow --help ``` -If you would like to try Flow, fork this repository, navigate to it through command line interface and execute following command: +## Running a pipeline + +Write a pipeline file that *returns* a `DataFrame` — `run` executes it for you, so do not call +`->run()` yourself: + +```php +read(from_array([ + ['id' => 1, 'name' => 'User 01', 'active' => true], + ['id' => 2, 'name' => 'User 02', 'active' => false], + ])) + ->write(to_output(truncate: false)); +``` + +Save it as `pipeline.php` and run it through the image: ```shell -$ docker run -v $(pwd):/flow-workspace --rm -it flow-php/flow:latest run /flow-workspace/examples/topics/aggregations/daily_revenue.php +$ docker run -v $(pwd):/flow-workspace --rm -it ghcr.io/flow-php/flow:latest run /flow-workspace/pipeline.php ``` -Flow CLI will grab the pipeline definition from `examples/topics/aggregations/daily_revenue.php` file and execute it. +## Bundled extensions + +Alongside PHP 8.5 and the extensions Flow's adapters need — `bcmath`, `gmp`, `pdo_mysql`, +`pdo_pgsql`, `pdo_sqlite`, `pgsql`, and the `brotli`, `lz4`, `snappy`, `zstd` codecs — the image ships four extensions +that Flow detects and uses automatically: + +| Extension | Package | Effect when loaded | +|------------|-------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `flow_php` | [flow-php/flow-php-ext](/documentation/components/extensions/flow-php-ext.md) | `AdaptiveRowHydrator` and `AdaptiveFloeEncoder` run native, fusing every Floe read/write and every raw-scalar hydration into one native call per batch | +| `arrow` | [flow-php/arrow-ext](/documentation/components/extensions/arrow-ext.md) | `AdaptiveParquetEngine` selects `ArrowParquetEngine`, so Parquet reads and writes run native | +| `pg_query` | [flow-php/pg-query-ext](/documentation/components/extensions/pg-query-ext.md) | `Flow\PostgreSql\Parser` becomes usable at all — SQL parsing, normalization and AST manipulation | +| `protobuf` | `pecl/protobuf` | `Flow\PostgreSql\Parser` decodes the parse tree in C instead of pure PHP — measured ~69x faster end to end | + +`pdo_pgsql` and `pgsql` link libpq 18 from the PGDG repository, matching the PostgreSQL 18 grammar +`pg_query` is built against. PHP 8.5 additionally compiles in `lexbor`, `uri` and Zend OPcache unconditionally. + +> [!NOTE] +> `Parser::parse()` decodes a protobuf AST, and protobuf caps message nesting at 100 levels — roughly 23 levels of +> nested subqueries. Deeper SQL fails to decode regardless of whether `protobuf` is loaded; the extension changes +> speed, not that ceiling. + +### Opting out of the native path + +Engine selection happens per read and per write, so a single step can be pinned to the PHP implementation: + +```php +hydrator(new PhpRowHydrator())) + ->read(from_parquet(__DIR__ . '/input.parquet', engine: new PhpParquetEngine())) + ->write(to_floe(__DIR__ . '/output.floe', engine: FloeEngine::php)); +``` + +To take the whole container off one native path, mount an empty file over that extension's ini: + +```shell +$ docker run --rm -v /dev/null:/usr/local/etc/php/conf.d/docker-php-ext-flow_php.ini \ + -v $(pwd):/flow-workspace ghcr.io/flow-php/flow:latest run /flow-workspace/pipeline.php +``` +Once a native engine is selected it does **not** silently degrade. Extension failures surface as +`Flow\Floe\Exception\ExtensionException`, which `FloeReader` and `FloeWriter` wrap as +`Flow\Floe\Exception\FloeException`. diff --git a/src/extension/arrow-ext/README.md b/src/extension/arrow-ext/README.md index b6017a75b0..6fe75085c4 100644 --- a/src/extension/arrow-ext/README.md +++ b/src/extension/arrow-ext/README.md @@ -1,6 +1,6 @@ # Extension: Arrow -A Rust-powered PHP extension for reading and writing Apache Parquet files using [ext-php-rs](https://github.com/davidcole1340/ext-php-rs) +A Rust-powered PHP extension for reading and writing Apache Parquet files using [ext-php-rs](https://github.com/extphprs/ext-php-rs) and the official Apache Arrow/Parquet Rust crates. This extension provides streaming read/write interfaces for Parquet files with support for all flat Arrow types, nested types (LIST, STRUCT, MAP), and multiple compression algorithms (SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW). diff --git a/src/extension/pg-query-ext/ext/pg_query.c b/src/extension/pg-query-ext/ext/pg_query.c index f5eb9cca06..fb78dbccc6 100644 --- a/src/extension/pg-query-ext/ext/pg_query.c +++ b/src/extension/pg-query-ext/ext/pg_query.c @@ -65,7 +65,11 @@ PHP_MINIT_FUNCTION(pg_query) PHP_MSHUTDOWN_FUNCTION(pg_query) { - pg_query_exit(); + /* Deliberately no pg_query_exit(). It frees libpg_query's TopMemoryContext, but pg_query_init() + * guards on a flag it never resets, so the contexts are gone and cannot be validly rebuilt. + * SAPIs that start the engine more than once per process -- embed, which the WASM playground + * uses -- then crash on the second startup. MSHUTDOWN runs at process teardown, so freeing + * here bought nothing anyway. */ return SUCCESS; } diff --git a/wasm/.gitignore b/wasm/.gitignore index 1dac1bf063..65ea461b87 100644 --- a/wasm/.gitignore +++ b/wasm/.gitignore @@ -4,7 +4,7 @@ *.phar.br manifest.json /libxml2-* -/libpg_query +/libpg_query-* /php-ext-snappy /libzip-* # Build logs diff --git a/wasm/build.sh b/wasm/build.sh index a1ce7be3de..f557fa7ac7 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Build script for Flow PHP Interactive Playground -# Builds PHP 8.4.13 +# Builds PHP 8.5.8 as 64-bit wasm (see WASM64_MODE below) with a forced mmap Opcache shared-memory +# backend (see php_cv_shm_mmap_anon below). set -xeu @@ -12,9 +13,16 @@ rm -f build.log exec > >(tee -a build.log) 2>&1 echo "=== Build started at $(date) ===" -PHP_VERSION=8.4.13 +PHP_VERSION=8.5.8 PHP_PATH=php-$PHP_VERSION +# MEMORY64=2 is wasm64 for clang/lld lowered to wasm32 by Binaryen, so PHP gets an 8-byte zend_long +# (pack()'s q/Q/J/P, which Floe and sortBy() need) without requiring Memory64 in visitors' browsers. +# Set via EMCC_CFLAGS, not CFLAGS, which is exported below -- after the dependencies are built. +WASM64_MODE="-sMEMORY64=2" +EM_TARGET=wasm64-emscripten +export EMCC_CFLAGS="$WASM64_MODE" + echo "Build libxml2 for WebAssembly" LIBXML2_VERSION=2.11.4 LIBXML2_DIR=libxml2-$LIBXML2_VERSION @@ -48,20 +56,33 @@ if [ ! -d "$LIBXML2_DIR" ]; then fi echo "Build libpg_query for WebAssembly" -LIBPG_QUERY_VERSION=17-latest -LIBPG_QUERY_DIR=libpg_query +# Read from the pg_query extension's own pin, so the playground and the CLI cannot disagree about +# which PostgreSQL grammar parses. +LIBPG_QUERY_VERSION=$(sed -n 's/^LIBPG_QUERY_VERSION := //p' \ + "$PROJECT_ROOT/../src/extension/pg-query-ext/Makefile") + +if [ -z "$LIBPG_QUERY_VERSION" ]; then + echo "ERROR: could not read LIBPG_QUERY_VERSION from src/extension/pg-query-ext/Makefile" + exit 1 +fi + +# Version in the directory name so a bumped pin re-clones instead of reusing the old checkout. +# This tracks a moving branch, so an upstream branch move still needs a manual rm -rf. +LIBPG_QUERY_DIR=libpg_query-$LIBPG_QUERY_VERSION LIBPG_QUERY_INSTALL_DIR="$PROJECT_ROOT/$LIBPG_QUERY_DIR" -if [ ! -d "$LIBPG_QUERY_DIR" ]; then +# Guard on the built archive, not the directory: a part-way failure leaves a tree that a directory +# check would skip, and the missing archive would only surface at link time. +if [ ! -f "$LIBPG_QUERY_INSTALL_DIR/libpg_query.a" ]; then + rm -rf "$LIBPG_QUERY_DIR" git clone --depth=1 --branch=$LIBPG_QUERY_VERSION \ https://github.com/pganalyze/libpg_query.git "$LIBPG_QUERY_DIR" cd $LIBPG_QUERY_DIR - # Build with Emscripten - override CC and AR - # The Makefile does AR := $(AR) rs, but command-line AR overrides this - # So we must include the 'rs' flags ourselves. However, LLVM ar doesn't support 'g', - # and the Makefile @ suppresses the echo, so we pass 'rcs' which is compatible. - emmake make build -j$(nproc) CC=emcc AR="emar rcs" + # libpg_query 18 keeps AR and ARFLAGS separate, so folding the flags into AR gives `emar rcs rs`, + # which llvm-ar rejects. Both must come from the command line -- make predefines ARFLAGS, so the + # Makefile's `?=` never fires. + emmake make build -j$(nproc) CC=emcc AR=emar ARFLAGS=rcs cd $PROJECT_ROOT fi @@ -76,12 +97,16 @@ if [ ! -f "$LIBZIP_INSTALL_DIR/lib/libzip.a" ]; then # First, ensure Emscripten's zlib port is built by triggering a compile # This downloads and builds zlib to the Emscripten cache echo "int main(){return 0;}" > /tmp/zlib_test.c - emcc -sUSE_ZLIB=1 /tmp/zlib_test.c -o /tmp/zlib_test.js 2>/dev/null || true + # -flto must match CFLAGS below: emcc caches a separate port build per variant and this links + # the lto/ one. + emcc -flto -sUSE_ZLIB=1 /tmp/zlib_test.c -o /tmp/zlib_test.js 2>/dev/null || true rm -f /tmp/zlib_test.c /tmp/zlib_test.js /tmp/zlib_test.wasm # Get Emscripten cache path and locate zlib EM_CACHE=$(em-config CACHE) - ZLIB_LIBRARY="$EM_CACHE/sysroot/lib/wasm32-emscripten/lto/libz.a" + # Per-target path: hardcoding wasm32 would find a real file of the wrong architecture, which the + # guard below cannot detect. + ZLIB_LIBRARY="$EM_CACHE/sysroot/lib/$EM_TARGET/lto/libz.a" ZLIB_INCLUDE_DIR="$EM_CACHE/sysroot/include" echo "Using zlib from Emscripten cache:" @@ -99,11 +124,11 @@ if [ ! -f "$LIBZIP_INSTALL_DIR/lib/libzip.a" ]; then tar xf $LIBZIP_DIR.tar.xz cd $LIBZIP_DIR + # CMakeCache.txt pins the resolved zlib path and target, so a tree left from another target has + # to be cleared rather than reconfigured. + rm -rf build mkdir -p build && cd build - # Configure libzip for WebAssembly using CMake - # Provide explicit paths to Emscripten's zlib (from its ports system) - # Disable encryption and optional compression to minimize dependencies emcmake cmake .. \ -DCMAKE_INSTALL_PREFIX=$LIBZIP_INSTALL_DIR \ -DZLIB_LIBRARY=$ZLIB_LIBRARY \ @@ -136,6 +161,21 @@ if [ ! -d "$PHP_PATH" ]; then tar xf $PHP_PATH.tar.xz fi +echo "Patch php-src" +# Re-runs reuse an already extracted $PHP_PATH, so this has to be idempotent: a cleanly applying +# reverse patch means it is already in place. Anything else is a real failure and set -e ends here. +cd "$PHP_PATH" + +for PHP_PATCH in "$PROJECT_ROOT"/patches/*.patch; do + if patch -p1 --reverse --dry-run --force --silent <"$PHP_PATCH" >/dev/null 2>&1; then + echo " already applied: $(basename "$PHP_PATCH")" + else + patch -p1 --forward <"$PHP_PATCH" + fi +done + +cd "$PROJECT_ROOT" + echo "Add pg_query extension" PG_QUERY_EXT_SRC="$PROJECT_ROOT/../src/extension/pg-query-ext/ext" PG_QUERY_EXT_DST="$PHP_PATH/ext/pg_query" @@ -155,8 +195,10 @@ cp -r "$SNAPPY_EXT_DIR" "$SNAPPY_EXT_DST" echo "Configure PHP" -# Use -Oz for size optimization instead of -O3 for speed -export CFLAGS="-Oz -flto -fPIC -g0 -DZEND_MM_ERROR=0 -I$LIBXML2_INSTALL_DIR/include/libxml2 -I$LIBPG_QUERY_INSTALL_DIR -I$LIBPG_QUERY_INSTALL_DIR/src -I$LIBZIP_INSTALL_DIR/include -sUSE_ZLIB=1" +# -DHAVE_REALLOCARRAY=1: emcc's link probe for reallocarray() fails while its sysroot header still +# declares it, so main/php_glob.c compiles a clashing static copy (php-src#19152). +# ZEND_MM_ERROR is deliberately NOT 0 -- it gates the messages that named the chunk-alignment leak. +export CFLAGS="-Oz -flto -fPIC -g0 -DHAVE_REALLOCARRAY=1 -I$LIBXML2_INSTALL_DIR/include/libxml2 -I$LIBPG_QUERY_INSTALL_DIR -I$LIBPG_QUERY_INSTALL_DIR/src -I$LIBZIP_INSTALL_DIR/include -sUSE_ZLIB=1" export CXXFLAGS="-Oz -flto -fPIC -g0 -std=c++11 -sUSE_ZLIB=1" export LDFLAGS="-L$LIBXML2_INSTALL_DIR/lib -L$LIBPG_QUERY_INSTALL_DIR -L$LIBZIP_INSTALL_DIR/lib -sUSE_ZLIB=1" @@ -168,13 +210,7 @@ export LIBZIP_LIBS="-L$LIBZIP_INSTALL_DIR/lib -lzip" cd $PHP_PATH -# Configure with extensions required by Flow PHP -# - bcmath: required by flow-php/parquet -# - libxml: required as base for XML extensions -# - xml, dom, xmlreader, xmlwriter: required by flow-php/etl-adapter-xml -# - phar, mbstring: essential PHP extensions -# - iconv: required by symfony/polyfill-mbstring -# - zip: required by flow-php/etl-adapter-excel (XLSX files are ZIP archives) +# The extension set and the reason for each is in documentation/contributing/wasm.md. # Fix permissions for build scripts chmod +x buildconf build/config-stubs build/shtool 2>/dev/null || true @@ -184,8 +220,18 @@ bash ./buildconf --force set +e +# Opcache's shared-memory probes all call fork(), which Emscripten lacks, so configure would leave +# Opcache permanently inert. Only the fork() half really fails -- Emscripten's mmap() serves +# anonymous MAP_SHARED from linear memory and its fcntl() reports the locks taken, which is enough. +export php_cv_shm_mmap_anon=yes + +# --disable-opcache-jit and --disable-huge-code-pages are NOT redundant next to --disable-all: both +# pass [no] as their 5th PHP_ARG_ENABLE arg, so --disable-all cannot reach either. Without the JIT +# flag emcc is handed dynasm for the build machine's ISA, since emconfigure passes no --host. emconfigure ./configure \ --disable-all \ + --disable-opcache-jit \ + --disable-huge-code-pages \ --disable-cgi \ --disable-cli \ --disable-rpath \ @@ -242,9 +288,9 @@ echo "Compile pib_eval wrapper" emcc $CFLAGS -I . -I Zend -I main -I TSRM/ ../pib_eval.c -c -o pib_eval.o echo "Link everything together" -emcc $CFLAGS $LDFLAGS \ +emcc $CFLAGS $LDFLAGS $WASM64_MODE \ -s ENVIRONMENT=web \ - -s EXPORTED_FUNCTIONS='["_pib_eval", "_pib_force_exit", "_php_embed_init", "_zend_eval_string", "_php_embed_shutdown"]' \ + -s EXPORTED_FUNCTIONS='["_pib_eval", "_pib_force_exit", "_pib_heap_bytes"]' \ -s EXPORTED_RUNTIME_METHODS='["ccall","FS","UTF8ToString","lengthBytesUTF8","stringToUTF8","getValue","setValue","ENV"]' \ -s MODULARIZE=1 \ -s EXPORT_NAME="'PHP'" \ diff --git a/wasm/patches/php-8.5-emscripten-mm-chunk-alignment.patch b/wasm/patches/php-8.5-emscripten-mm-chunk-alignment.patch new file mode 100644 index 0000000000..8177c09959 --- /dev/null +++ b/wasm/patches/php-8.5-emscripten-mm-chunk-alignment.patch @@ -0,0 +1,87 @@ +Stop Zend's chunk allocator from orphaning ~2 MB of wasm memory per chunk. + +zend_mm_chunk_alloc_int() aligns a chunk by over-allocating and then munmap()ing +the unwanted head and tail. Emscripten's munmap() returns EINVAL unless the length +matches a whole mapping, so both trims fail -- and because wasm memory can never +shrink, one ZEND_MM_CHUNK_SIZE is lost per chunk, permanently. That is what gave +the playground a finite Run budget: 2.362 MB leaked per Run, and the tab stopped +executing PHP after 813 Runs. It was silent because build.sh used to compile with +-DZEND_MM_ERROR=0, which suppresses the "munmap() failed" message. + +posix_memalign() aligns directly, so there is nothing to trim. chunk_truncate() +and chunk_extend() then report failure, which is the branch _WIN32 already takes +and which callers handle by falling back to allocate-copy-free. + +Not reported upstream: php-src does not target Emscripten. Re-check on the next +PHP bump. + +--- a/Zend/zend_alloc.c ++++ b/Zend/zend_alloc.c +@@ -477,7 +477,9 @@ + #endif + } + +-#ifndef HAVE_MREMAP ++/* zend_mm_chunk_extend() is the only non-_WIN32 caller and cannot succeed under Emscripten, so ++ * without this guard the definition is left unreferenced. */ ++#if !defined(HAVE_MREMAP) && !defined(__EMSCRIPTEN__) + static void *zend_mm_mmap_fixed(void *addr, size_t size) + { + #ifdef _WIN32 +@@ -740,6 +742,18 @@ + + static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment) + { ++#ifdef __EMSCRIPTEN__ ++ /* Emscripten's munmap() cannot release part of a mapping, so the over-allocate-then-trim path ++ * below orphans one ZEND_MM_CHUNK_SIZE per chunk and wasm memory never shrinks. Early return ++ * rather than #else, so the generic path stays referenced and -Wunused-function stays quiet. */ ++ void *aligned; ++ ++ if (posix_memalign(&aligned, alignment, size) != 0) { ++ return NULL; ++ } ++ ++ return aligned; ++#endif + void *ptr = zend_mm_mmap(size); + + if (ptr == NULL) { +@@ -818,7 +832,12 @@ + return; + } + #endif ++#ifdef __EMSCRIPTEN__ ++ /* Paired with the posix_memalign() in zend_mm_chunk_alloc_int(). */ ++ free(addr); ++#else + zend_mm_munmap(addr, size); ++#endif + } + + static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size) +@@ -832,10 +851,11 @@ + } + } + #endif +-#ifndef _WIN32 ++#if !defined(_WIN32) && !defined(__EMSCRIPTEN__) + zend_mm_munmap((char*)addr + new_size, old_size - new_size); + return 1; + #else ++ /* A posix_memalign() block cannot be partially released; callers fall back to copy-and-free. */ + return 0; + #endif + } +@@ -860,9 +880,10 @@ + /* Sanity check: The mapping shouldn't have moved. */ + ZEND_ASSERT(ptr == addr); + return 1; +-#elif !defined(_WIN32) ++#elif !defined(_WIN32) && !defined(__EMSCRIPTEN__) + return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL); + #else ++ /* Emscripten's mmap() rejects a non-NULL addr hint, so a fixed extension can never succeed. */ + return 0; + #endif + } diff --git a/wasm/patches/php-8.5-opcache-unistd.patch b/wasm/patches/php-8.5-opcache-unistd.patch new file mode 100644 index 0000000000..3b5259a5d0 --- /dev/null +++ b/wasm/patches/php-8.5-opcache-unistd.patch @@ -0,0 +1,23 @@ +Add the include that ext/opcache needs for getpid(). + +zend_accelerator_debug.c calls getpid() in its non-ZTS branch but only includes +, and only under ZEND_WIN32. On PHP 8.4 this never surfaced because +--disable-all excluded opcache entirely; php-src 7b4c14dc1016 ("Make OPcache +non-optional") removed that possibility, so the file is now always compiled. +clang has treated implicit function declarations as an error since 16, and the +Emscripten toolchain is well past that, so the build fails without this. + +Same fix as WordPress Playground's packages/php-wasm/compile/php/php8.5.patch. +Reported upstream against 8.5; re-check on the next PHP bump and drop this file +if it has landed. + +--- a/ext/opcache/zend_accelerator_debug.c ++++ b/ext/opcache/zend_accelerator_debug.c +@@ -25,6 +25,8 @@ + #include + #ifdef ZEND_WIN32 + # include ++#else ++# include /* getpid(), called below outside the ZEND_WIN32 branch */ + #endif + #include "ZendAccelerator.h" diff --git a/wasm/pib_eval.c b/wasm/pib_eval.c index b6bbe41f5e..3e85ac01ba 100644 --- a/wasm/pib_eval.c +++ b/wasm/pib_eval.c @@ -3,6 +3,7 @@ #include "Zend/zend_interfaces.h" #include "Zend/zend_compile.h" #include +#include #include #include @@ -78,7 +79,7 @@ static void pib_report_exception(zend_object *ex) { zend_string_release(message); zend_string *file = zval_get_string(GET_PROPERTY_SILENT(&exception, ZEND_STR_FILE)); zend_long line = zval_get_long(GET_PROPERTY_SILENT(&exception, ZEND_STR_LINE)); - fprintf(stderr, "At %s:%d\n", ZSTR_VAL(file), line); + fprintf(stderr, "At %s:" ZEND_LONG_FMT "\n", ZSTR_VAL(file), line); zend_string_release(file); /* // Can't get this to work at the end of execution. @@ -97,20 +98,66 @@ static void pib_report_exception(zend_object *ex) { } } -// Based on code by https://github.com/oraoto/pib with modifications. -int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) { - int ret = 0; - // USE_ZEND_ALLOC prevents using fast shutdown. - // putenv("USE_ZEND_ALLOC=0"); - php_embed_init(0, NULL); +// One PHP *module* lifecycle per page, one *request* lifecycle per Run. +static int pib_module_started = 0; + +// Everything php_embed_init() does after php_request_startup(), plus this file's own per-request +// setup. sapi_activate() resets SG() on every request, so this runs for every request, not only +// the first. +static void pib_request_prepare(void) { + SG(headers_sent) = 1; + SG(request_info).no_headers = 1; + pib_cli_register_file_handles(); // Show fatal E_COMPILE_ERRORs and other errors properly (startup errors are normally hidden) - PG(display_startup_errors)=1; - PG(during_request_startup)=0; + PG(display_startup_errors) = 1; + PG(during_request_startup) = 0; // Enable error display to stdout PG(display_errors) = 1; +} + +// Opcache's shared memory is allocated once, at MINIT, and sized by opcache.memory_consumption -- +// so it has to be set before php_embed_init(). ini_defaults is the SAPI's documented hook for that +// (sapi/embed/php_embed.c:201-215); ini_entries cannot be used because php_embed_init() overwrites +// it with its own HARDCODED_INI at :216. +#define PIB_INI_DEFAULT(name, value) \ + ZVAL_NEW_STR(&tmp, zend_string_init(value, sizeof(value) - 1, 1)); \ + zend_hash_str_update(configuration_hash, name, sizeof(name) - 1, &tmp); + +static void pib_ini_defaults(HashTable *configuration_hash) { + zval tmp; + PIB_INI_DEFAULT("opcache.memory_consumption", "32") + PIB_INI_DEFAULT("opcache.interned_strings_buffer", "8") + PIB_INI_DEFAULT("opcache.validate_timestamps", "0") +} + +#undef PIB_INI_DEFAULT + +double EMSCRIPTEN_KEEPALIVE pib_heap_bytes(void) { + return (double) emscripten_get_heap_size(); +} + +// Based on code by https://github.com/oraoto/pib with modifications. +int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) { + int ret = 0; + // USE_ZEND_ALLOC prevents using fast shutdown. + // putenv("USE_ZEND_ALLOC=0"); + if (!pib_module_started) { + php_embed_module.ini_defaults = pib_ini_defaults; + + // The shipped version ignored this return value, so a failed init fell through into + // zend_first_try on a dead engine and reported nothing at all. + if (php_embed_init(0, NULL) == FAILURE) { + fprintf(stderr, "Fatal error: failed to initialize the PHP engine\n"); + fflush(stderr); + return FAILURE; + } + + pib_module_started = 1; + pib_request_prepare(); + } zend_first_try { // Set error_reporting to E_ALL @@ -148,9 +195,19 @@ int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) { } ret = EG(exit_status); } zend_end_try(); - php_embed_shutdown(); + + php_request_shutdown((void *) 0); + fflush(stdout); fflush(stderr); + + if (php_request_startup() == FAILURE) { + pib_module_started = 0; + return FAILURE; + } + + pib_request_prepare(); + return ret; } diff --git a/web/landing/assets/codemirror/completions/dataframe.js b/web/landing/assets/codemirror/completions/dataframe.js index 8a29f3fa01..7237238f7e 100644 --- a/web/landing/assets/codemirror/completions/dataframe.js +++ b/web/landing/assets/codemirror/completions/dataframe.js @@ -1,7 +1,7 @@ /** * CodeMirror Completer for Flow PHP DataFrame Methods * - * DataFrame methods: 54 + * DataFrame methods: 53 * DataFrame-returning methods from classes: 3 * * This completer triggers after DataFrame-returning methods @@ -10,7 +10,7 @@ import { CompletionContext, snippet } from "@codemirror/autocomplete" // Map of DataFrame-returning methods grouped by class -const dataframeReturningMethods = {"flow":["extract","from","process","read"],"dataframe":["aggregate","autoCast","batchBy","batchSize","cache","collect","collectRefs","constrain","crossJoin","drop","dropDuplicates","dropPartitions","duplicateRow","filter","filterPartitions","filters","join","joinEach","limit","load","map","match","mode","offset","onError","partitionBy","pivot","rename","renameEach","reorderEntries","rows","saveMode","select","sortBy","transform","until","void","with","withEntries","withEntry","write"],"groupeddataframe":["aggregate"]}; +const dataframeReturningMethods = {"flow":["extract","from","process","read"],"groupeddataframe":["aggregate"],"dataframe":["aggregate","autoCast","batchBy","batchSize","cache","collect","collectRefs","constrain","crossJoin","drop","dropDuplicates","dropPartitions","duplicateRow","filter","filterPartitions","filters","join","joinEach","limit","load","map","match","mode","offset","onError","partitionBy","rename","renameEach","reorderEntries","rows","saveMode","select","sortBy","transform","until","void","with","withEntries","withEntry","write"]}; // DataFrame methods const dataframeMethods = [ @@ -635,21 +635,6 @@ const dataframeMethods = [ }, apply: snippet("partitionBy(" + "$" + "{" + "1:entry" + "}" + ", " + "$" + "{" + "2:entries" + "}" + ")"), boost: 10 - }, { - label: "pivot", - type: "method", - detail: "Flow\\\\ETL\\\\DataFrame", - info: () => { - const div = document.createElement("div") - div.innerHTML = ` -
- pivot(Reference $ref) : self -
- ` - return div - }, - apply: snippet("pivot(" + "$" + "{" + "1:ref" + "}" + ")"), - boost: 10 }, { label: "printRows", type: "method", diff --git a/web/landing/assets/codemirror/completions/dsl.js b/web/landing/assets/codemirror/completions/dsl.js index 62d0159f2c..df60b8b97b 100644 --- a/web/landing/assets/codemirror/completions/dsl.js +++ b/web/landing/assets/codemirror/completions/dsl.js @@ -1,7 +1,7 @@ /** * CodeMirror Completer for Flow PHP DSL Functions * - * Total functions: 799 + * Total functions: 821 * * This completer provides autocompletion for all Flow PHP DSL functions: * - Extractors (flow-extractors) @@ -3992,6 +3992,21 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\PostgreSql\\DSL\\explain(" + "$" + "{" + "1:query" + "}" + ")"), boost: 10 + }, { + label: "external_sort", + type: "function", + detail: "flow\u002Ddsl\u002Ddata\u002Dframe", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ external_sort() : ExternalSortBuilder +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\DSL\\external_sort()"), + boost: 10 }, { label: "fetch", type: "function", @@ -4543,12 +4558,12 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- from_dynamic_http_requests(ClientInterface $client, NextRequestFactory $requestFactory) : PsrHttpClientDynamicExtractor + from_dynamic_http_requests(ClientInterface $client, NextRequestFactory $requestFactory, Schema $schema = null) : PsrHttpClientDynamicExtractor
` return div }, - apply: snippet("\\Flow\\ETL\\Adapter\\Http\\from_dynamic_http_requests(" + "$" + "{" + "1:client" + "}" + ", " + "$" + "{" + "2:requestFactory" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\from_dynamic_http_requests(" + "$" + "{" + "1:client" + "}" + ", " + "$" + "{" + "2:requestFactory" + "}" + ", " + "$" + "{" + "3:schema" + "}" + ")"), boost: 10 }, { label: "from_excel", @@ -4619,6 +4634,21 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\ETL\\Adapter\\GoogleSheet\\from_google_sheet_columns(" + "$" + "{" + "1:auth_config" + "}" + ", " + "$" + "{" + "2:spreadsheet_id" + "}" + ", " + "$" + "{" + "3:sheet_name" + "}" + ", " + "$" + "{" + "4:start_range_column" + "}" + ", " + "$" + "{" + "5:end_range_column" + "}" + ", " + "$" + "{" + "6:with_header" + "}" + ", " + "$" + "{" + "7:rows_per_page" + "}" + ", " + "$" + "{" + "8:options" + "}" + ")"), boost: 10 + }, { + label: "from_http_paginated", + type: "function", + detail: "flow\u002Ddsl\u002Dextractors", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ from_http_paginated(ClientInterface $client, RequestInterface $request, Paginator $paginator, Schema $schema = null) : PsrHttpClientPaginatedExtractor +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\from_http_paginated(" + "$" + "{" + "1:client" + "}" + ", " + "$" + "{" + "2:request" + "}" + ", " + "$" + "{" + "3:paginator" + "}" + ", " + "$" + "{" + "4:schema" + "}" + ")"), + boost: 10 }, { label: "from_json", type: "function", @@ -4786,7 +4816,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- from_static_http_requests(ClientInterface $client, iterable $requests) : PsrHttpClientStaticExtractor + from_static_http_requests(ClientInterface $client, iterable $requests, Schema $schema = null) : PsrHttpClientStaticExtractor
@param iterable $requests @@ -4794,7 +4824,7 @@ const dslFunctions = [ ` return div }, - apply: snippet("\\Flow\\ETL\\Adapter\\Http\\from_static_http_requests(" + "$" + "{" + "1:client" + "}" + ", " + "$" + "{" + "2:requests" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\from_static_http_requests(" + "$" + "{" + "1:client" + "}" + ", " + "$" + "{" + "2:requests" + "}" + ", " + "$" + "{" + "3:schema" + "}" + ")"), boost: 10 }, { label: "from_text", @@ -5072,6 +5102,36 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\ETL\\DSL\\hash(" + "$" + "{" + "1:value" + "}" + ", " + "$" + "{" + "2:algorithm" + "}" + ")"), boost: 10 + }, { + label: "hash_group_by", + type: "function", + detail: "flow\u002Ddsl\u002Ddata\u002Dframe", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ hash_group_by() : HashGroupByBuilder +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\DSL\\hash_group_by()"), + boost: 10 + }, { + label: "hash_join", + type: "function", + detail: "flow\u002Ddsl\u002Ddata\u002Dframe", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ hash_join() : HashJoinBuilder +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\DSL\\hash_join()"), + boost: 10 }, { label: "host_detector", type: "function", @@ -5156,6 +5216,261 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\ETL\\DSL\\html_schema(" + "$" + "{" + "1:name" + "}" + ", " + "$" + "{" + "2:nullable" + "}" + ", " + "$" + "{" + "3:metadata" + "}" + ")"), boost: 10 + }, { + label: "http_pagination_cursor", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_cursor(string $cursor_path, RequestOption $inject, bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_cursor(" + "$" + "{" + "1:cursor_path" + "}" + ", " + "$" + "{" + "2:inject" + "}" + ", " + "$" + "{" + "3:stop_on_client_error" + "}" + ", " + "$" + "{" + "4:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_pagination_last_record_cursor", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_last_record_cursor(string $record_path, RequestOption $inject, bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_last_record_cursor(" + "$" + "{" + "1:record_path" + "}" + ", " + "$" + "{" + "2:inject" + "}" + ", " + "$" + "{" + "3:stop_on_client_error" + "}" + ", " + "$" + "{" + "4:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_pagination_link_header", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_link_header(string $rel = 'next', bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_link_header(" + "$" + "{" + "1:rel" + "}" + ", " + "$" + "{" + "2:stop_on_client_error" + "}" + ", " + "$" + "{" + "3:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_pagination_next_url", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_next_url(string $path, bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_next_url(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:stop_on_client_error" + "}" + ", " + "$" + "{" + "3:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_pagination_offset", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_offset(RequestOption $offset_option, RequestOption $limit_option, int $limit, int $start_offset = 0, string $total_path = null, bool $inject_on_first_request = true, bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_offset(" + "$" + "{" + "1:offset_option" + "}" + ", " + "$" + "{" + "2:limit_option" + "}" + ", " + "$" + "{" + "3:limit" + "}" + ", " + "$" + "{" + "4:start_offset" + "}" + ", " + "$" + "{" + "5:total_path" + "}" + ", " + "$" + "{" + "6:inject_on_first_request" + "}" + ", " + "$" + "{" + "7:stop_on_client_error" + "}" + ", " + "$" + "{" + "8:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_pagination_page_number", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_pagination_page_number(RequestOption $inject, int $start_page = 1, int $page_size = null, RequestOption $size_option = null, bool $inject_on_first_request = true, string $records_path = null, bool $stop_on_client_error = true, StopWhen $stop_when = null) : Paginator +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_pagination_page_number(" + "$" + "{" + "1:inject" + "}" + ", " + "$" + "{" + "2:start_page" + "}" + ", " + "$" + "{" + "3:page_size" + "}" + ", " + "$" + "{" + "4:size_option" + "}" + ", " + "$" + "{" + "5:inject_on_first_request" + "}" + ", " + "$" + "{" + "6:records_path" + "}" + ", " + "$" + "{" + "7:stop_on_client_error" + "}" + ", " + "$" + "{" + "8:stop_when" + "}" + ")"), + boost: 10 + }, { + label: "http_request_option_body", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_request_option_body(string $path, StreamFactoryInterface $stream_factory) : RequestOption +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_request_option_body(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:stream_factory" + "}" + ")"), + boost: 10 + }, { + label: "http_request_option_header", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_request_option_header(string $name) : RequestOption +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_request_option_header(" + "$" + "{" + "1:name" + "}" + ")"), + boost: 10 + }, { + label: "http_request_option_query", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_request_option_query(string $name) : RequestOption +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_request_option_query(" + "$" + "{" + "1:name" + "}" + ")"), + boost: 10 + }, { + label: "http_request_option_uri", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_request_option_uri() : RequestOption +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_request_option_uri()"), + boost: 10 + }, { + label: "http_stop_when_empty_path", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_empty_path(string $path) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_empty_path(" + "$" + "{" + "1:path" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_flag_false", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_flag_false(string $path) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_flag_false(" + "$" + "{" + "1:path" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_flag_true", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_flag_true(string $path) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_flag_true(" + "$" + "{" + "1:path" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_max_pages", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_max_pages(int $pages) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_max_pages(" + "$" + "{" + "1:pages" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_max_results", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_max_results(int $count) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_max_results(" + "$" + "{" + "1:count" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_path_missing", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_path_missing(string $path) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_path_missing(" + "$" + "{" + "1:path" + "}" + ")"), + boost: 10 + }, { + label: "http_stop_when_total_reached", + type: "function", + detail: "flow\u002Ddsl\u002Dhelpers", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ http_stop_when_total_reached(string $total_path) : StopWhen +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\Adapter\\Http\\http_stop_when_total_reached(" + "$" + "{" + "1:total_path" + "}" + ")"), + boost: 10 }, { label: "identical", type: "function", @@ -6368,6 +6683,21 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\Telemetry\\DSL\\memory_metric_processor(" + "$" + "{" + "1:exporter" + "}" + ", " + "$" + "{" + "2:errorHandler" + "}" + ")"), boost: 10 + }, { + label: "memory_sort", + type: "function", + detail: "flow\u002Ddsl\u002Ddata\u002Dframe", + info: () => { + const div = document.createElement("div") + div.innerHTML = ` +
+ memory_sort() : MemorySortBuilder +
+ ` + return div + }, + apply: snippet("\\Flow\\ETL\\DSL\\memory_sort()"), + boost: 10 }, { label: "memory_span_processor", type: "function", diff --git a/web/landing/assets/controllers/wasm_controller.js b/web/landing/assets/controllers/wasm_controller.js index 2d8574807f..a21879cb71 100644 --- a/web/landing/assets/controllers/wasm_controller.js +++ b/web/landing/assets/controllers/wasm_controller.js @@ -1,5 +1,7 @@ import { Controller } from "@hotwired/stimulus" +const RECYCLE_THRESHOLD_BYTES = 1536 * 1024 * 1024 + export default class extends Controller { static values = { phpJs: String, @@ -12,6 +14,7 @@ export default class extends Controller { #resourcesLoaded = false #combinedOutput = '' #debug = false + #recycleCount = 0 connect() { this.#debug = this.application.debug @@ -25,48 +28,6 @@ export default class extends Controller { this.#resourcesLoaded = false } - #evaluate(code) { - if (!this.#phpModuleLoaded) { - this.#dispatchError('PHP module not loaded yet', null) - return false - } - - this.#combinedOutput = '' - - try { - try { - this.#phpModule.FS.chdir('/workspace') - this.#log('Changed working directory to /workspace') - } catch (chdirError) { - this.#logError('Failed to change directory to /workspace:', chdirError) - } - - const result = this.#phpModule.ccall( - 'pib_eval', - 'number', - ['string'], - [code] - ) - - const errorInfo = this.#parseErrorMessage(this.#combinedOutput) - if (errorInfo) { - this.#log('Parsed error info:', errorInfo) - this.#dispatchError(this.#combinedOutput, errorInfo) - return false - } - - const output = this.#combinedOutput || 'Code executed successfully (no output)' - this.#dispatchOutput(output) - return true - } catch (e) { - this.#logError('Execution error:', e) - const errorInfo = this.#parseErrorMessage(this.#combinedOutput) - this.#log('Parsed error info:', errorInfo) - this.#dispatchError('Execution error: ' + e.message + '\n' + this.#combinedOutput, errorInfo) - return false - } - } - #formatCode(code, callback) { if (!this.#phpModuleLoaded) { callback(null, 'PHP module not loaded yet', null) @@ -162,6 +123,8 @@ require '/workspace/bin/cs-fixer.php'; throw new Error('PHP module not loaded') } + await this.#recycleIfNeeded() + this.#combinedOutput = '' try { @@ -212,6 +175,8 @@ require '/workspace/bin/cs-fixer.php'; } async format(code) { + await this.#recycleIfNeeded() + return new Promise((resolve) => { this.#formatCode(code, (formattedCode, error, appliedFixers) => { if (error) { @@ -400,7 +365,9 @@ require '/workspace/bin/cs-fixer.php'; } } - async #loadResources() { + // announceReady=false while recycling: wasm:initialized makes playground-storage reload code.php + // from the example's initial code, which would throw away the user's edits. + async #loadResources(announceReady = true) { if (!this.#phpModuleLoaded) { this.#logError('Cannot load resources: PHP module not loaded yet') return false @@ -420,11 +387,18 @@ require '/workspace/bin/cs-fixer.php'; this.#log('Creating /workspace/uploads directory') this.#phpModule.FS.mkdir('/workspace/uploads') + // Examples write results under output/. Flow's filesystem creates missing directories + // on write, but PDO/sqlite does not, so the directory has to exist up front. + this.#log('Creating /workspace/output directory') + this.#phpModule.FS.mkdir('/workspace/output') + if (!this.resourcesValue || Object.keys(this.resourcesValue).length === 0) { this.#log('No resources configured to load') this.#resourcesLoaded = true - this.#dispatchResourcesLoaded() - this.#dispatchInitialized() + if (announceReady) { + this.#dispatchResourcesLoaded() + this.#dispatchInitialized() + } return true } @@ -457,8 +431,10 @@ require '/workspace/bin/cs-fixer.php'; this.#resourcesLoaded = true this.#log('All resources loaded successfully') - this.#dispatchResourcesLoaded() - this.#dispatchInitialized() + if (announceReady) { + this.#dispatchResourcesLoaded() + this.#dispatchInitialized() + } return true } catch (error) { this.#logError('Failed to load resources:', error) @@ -511,25 +487,7 @@ require '/workspace/bin/cs-fixer.php'; this.#dispatchProgress('Initializing PHP runtime...', 30) if (typeof PHP !== 'undefined') { - PHP({ - print: (text) => { - this.#log('PHP stdout:', text) - this.#combinedOutput += text + '\n' - }, - printErr: (text) => { - this.#log('PHP stderr:', text) - this.#combinedOutput += text + '\n' - }, - locateFile: (path, scriptDirectory) => { - if (path === 'php.wasm') { - this.#log('Locating php.wasm at:', this.phpWasmValue) - return this.phpWasmValue - } - return scriptDirectory + path - } - }).then(async (module) => { - this.#phpModule = module - this.#phpModuleLoaded = true + this.#instantiate().then(async () => { this.#log('PHP module initialized successfully') await this.#loadResources() @@ -551,6 +509,117 @@ require '/workspace/bin/cs-fixer.php'; document.head.appendChild(script) } + // Reused by the recycling path, which must not re-inject the