From 7e5305c4fd0d984547d641170b90de9fdbc001f6 Mon Sep 17 00:00:00 2001 From: Brooklyn Zelenka Date: Thu, 18 Jun 2026 17:19:26 -0700 Subject: [PATCH] Bump deps, resurrect README --- README.md | 354 +++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 8 +- pnpm-lock.yaml | 46 +++---- 3 files changed, 381 insertions(+), 27 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..0262046 --- /dev/null +++ b/README.md @@ -0,0 +1,354 @@ +# pushwork + +_Bidirectional directory synchronization using [Automerge](https://automerge.org/) CRDTs._ + +pushwork turns any directory into a synchronized, conflict-free replicated +folder. Initialize a directory to get a shareable `automerge:` URL, hand that +URL to another machine (or another person), and `clone` it. From then on, +`sync` reconciles both working trees through a WebSocket relay — concurrent +edits, additions, deletions, and renames all converge automatically because +every file is backed by an Automerge document. + +It feels a bit like Git, but the "merge" is a CRDT: there are no merge +conflicts to resolve by hand, and character-level edits to text files combine +cleanly. + +## How it works + +A pushwork repo is a tree of Automerge documents. One _root folder doc_ (whose +URL is what you share) references one _file doc_ per file. The folder URL is +stable for the life of the repo, so sharing it once is enough. + +```mermaid +graph TD + A["root folder doc
(automerge: URL — this is what you share)"] + A --> B["src/index.ts → file doc"] + A --> C["README.md → file doc"] + A --> D["dist/bundle.js → file doc (pinned artifact)"] + A --> E["..."] +``` + +A `sync` performs a full round trip: + +```mermaid +sequenceDiagram + participant FS as Working tree + participant PW as pushwork + participant SRV as Sync server (WebSocket) + FS->>PW: read files, honor ignore rules + PW->>PW: diff against saved tree, write changes into file docs + PW->>SRV: push/pull doc changes (Automerge merge) + SRV-->>PW: peer changes + PW->>FS: materialize merged tree back to disk +``` + +`save` (alias `commit`) runs the same pipeline _offline_ — it commits to local +CRDT storage and never contacts a server. + +## Requirements + +- Node.js `>= 24` +- [pnpm](https://pnpm.io/) `>= 8` + +## Installation + +pushwork is currently built from source: + +```sh +git clone pushwork +cd pushwork +pnpm install +pnpm build # compiles TypeScript to dist/ (required — workers run from dist/) +``` + +This produces the `pushwork` binary at `dist/cli.js`. You can then either: + +```sh +node dist/cli.js # run directly +pnpm start -- # via the start script +pnpm link --global # expose `pushwork` on your PATH +``` + +The examples below assume `pushwork` is on your PATH. + +> [!NOTE] +> A build is required even when developing, because parallel ingest spins up +> worker threads that load compiled scripts from `dist/workers/`. + +## Quick start + +```sh +# Machine A — turn a directory into a pushwork repo +cd ~/notes +pushwork init +pushwork url # prints the automerge: URL to share + +# Machine B — clone that URL into a new directory +pushwork clone automerge:2sX...e9 ~/notes-clone + +# On either machine, after editing files +pushwork sync # exchange changes with peers and merge to disk + +# Inspect local changes before syncing +pushwork status +pushwork diff +``` + +## Commands + +| Command | Description | +|---|---| +| `pushwork init [dir]` | Initialize pushwork in a directory (default `.`). | +| `pushwork clone ` | Clone an `automerge:` URL into a directory. | +| `pushwork sync` | Sync local changes with peers and merge remote changes to disk. | +| `pushwork save` (alias `commit`) | Commit local changes to local storage without contacting the server. | +| `pushwork status` | Show changes against the saved state. | +| `pushwork diff [path]` | Show textual diffs of local changes; optionally limit to `path`. | +| `pushwork url` | Print the `automerge:` URL of this repo. | +| `pushwork heads [pathspec]` | Print Automerge heads for the root folder and every file doc (offline). | +| `pushwork migrate [dir]` | Upgrade an older `.pushwork/config.json` to the current format. | +| `pushwork cut [name]` | Stash working-tree changes and reset the tree to the saved state (offline). | +| `pushwork paste [id-or-name]` | Re-apply a stashed change set (default: most recent). | +| `pushwork snarfs` (alias `clipboard`) | List stashed change sets, newest first. | +| `pushwork version` | Print pushwork and Automerge package versions. | + +### Global options + +These apply to every command: + +| Flag | Description | +|---|---| +| `--porcelain` | Machine-readable output: tab-separated lines, no spinners/colors/prompts. | +| `-q, --quiet` | Suppress progress; show only results and errors. | +| `--silent` | Suppress all output except errors (check the exit code). | +| `-v, --version` | Print version info and exit. | + +### `init` / `clone` options + +| Flag | Applies to | Description | +|---|---|---| +| `--shape ` | both | Document shape: `vfs` (default), `patchwork-folder`, or a path to a custom shape module. | +| `--artifact-dir ` | both | Directory stored as immutable, heads-pinned content. Repeatable. Defaults to `dist`. | +| `--no-sub` | both | Use the legacy WebSocket backend instead of Subduction. | +| `--legacy` | both | Alias for `--no-sub`. | + +On `clone`, the shape is normally chosen from the root doc itself +(`@patchwork.type` of `directory` → `vfs`, `folder` → `patchwork-folder`); +`--shape` is only the fallback when the type isn't recognized. + +### `sync` options + +| Flag | Description | +|---|---| +| `--nuclear` | Re-create every file/folder doc with a fresh URL before syncing, dropping references to the old URLs from this repo. | + +## Sync backends + +pushwork supports two WebSocket sync backends. **Subduction is the default.** + +| Backend | Default endpoint | Selected by | +|---|---|---| +| `subduction` | `wss://subduction.sync.inkandswitch.com` | default | +| `legacy` | `wss://sync3.automerge.org` | `--legacy` / `--no-sub` | + +Override either endpoint with an environment variable: + +```sh +PUSHWORK_SUBDUCTION_SERVER=wss://my-relay.example.com pushwork sync +PUSHWORK_LEGACY_SERVER=wss://my-relay.example.com pushwork sync --legacy +``` + +## Configuration + +pushwork stores all of its metadata under `.pushwork/` at the repo root: + +| Path | Contents | +|---|---| +| `.pushwork/config.json` | Repo configuration (see below). | +| `.pushwork/storage/` | Automerge CRDT storage (`NodeFSStorageAdapter`). | +| `.pushwork/snarf/index.json` | Local stash entries (see [Stashing changes](#stashing-changes)). | + +`config.json` is currently at version `4`: + +```json +{ + "version": 4, + "rootUrl": "automerge:2sX...e9", + "backend": "subduction", + "shape": "vfs", + "artifactDirectories": ["dist"] +} +``` + +If the config version doesn't match the installed pushwork, commands fail with +a prompt to run `pushwork migrate`. + +### Ignore files + +The following are always ignored: `.pushwork`, `.git`, and `node_modules`. +Symlinks are skipped during traversal. + +For anything else, add a `.pushworkignore` file at the repo root. It uses +gitignore syntax (blank lines and `#` comments are ignored): + +```gitignore +# .pushworkignore +*.log +tmp/ +.env +``` + +### Artifact directories + +Files inside an _artifact directory_ (default: `dist`) are treated as build +output: their content is stored as an immutable string and their doc URL is +_pinned_ to a specific set of heads, so consumers reference an exact snapshot +rather than a moving target. Configure with `--artifact-dir ` (repeatable) +at `init`/`clone` time. + +## Document shapes + +A _shape_ controls how the directory tree is encoded into Automerge documents. + +| Shape | Value | Structure | +|---|---|---| +| VFS _(default)_ | `vfs` | A single directory doc (`@patchwork.type: "directory"`) whose keys are posix file paths mapping to file-doc URLs. | +| Patchwork folder | `patchwork-folder` | A recursive folder-of-docs (`@patchwork.type: "folder"`) compatible with Patchwork and original pushwork repos. | +| Custom | _module path_ | A module with a `default` export implementing `{ encode, decode }`. | + +Select a shape with `--shape` at `init`/`clone`. + +## Stashing changes + +pushwork has a local "clipboard" for working-tree changes — handy for setting +aside in-progress edits. These stashes (called _snarfs_) are stored locally and +are never synced. + +```sh +pushwork cut "wip-refactor" # stash changes, reset tree to saved state +pushwork snarfs # list stashes (newest first) +pushwork paste # re-apply the most recent stash +pushwork paste wip-refactor # or re-apply a specific one by id or name +``` + +## Migrating older repos + +If you have a repo created by an older pushwork (or the original pushwork +"main" repo with a `.pushwork/config.json` predating versioning), upgrade it +in place: + +```sh +pushwork migrate # operates on the current directory +pushwork migrate ./repo +``` + +Migration walks the config forward one version at a time and reports the steps +taken, or that the repo is already up to date. + +## Programmatic API + +The package also exposes a library API (`import` from `pushwork`). Functions +that talk to the network accept `online: false` for fully offline operation. + +```ts +import { init, clone, sync, save, status } from "pushwork"; + +// Initialize a repo +const { url, files } = await init({ + dir: "./my-project", + backend: "subduction", // or "legacy" + shape: "vfs", +}); +console.log(`Initialized ${files} files at ${url}`); + +// Clone it elsewhere +await clone({ url, dir: "./clone-target", backend: "subduction", shape: "vfs" }); + +// Sync / inspect +await sync("./my-project"); // online by default +await save("./my-project"); // offline commit +const { diff } = await status("./my-project"); +``` + +### Exported functions + +| Function | Signature | +|---|---| +| `init` | `(opts: InitOpts, report?: Reporter) => Promise` | +| `clone` | `(opts: CloneOpts, report?: Reporter) => Promise` | +| `sync` | `(cwd: string, opts?: { nuclear?: boolean }, report?: Reporter) => Promise` | +| `save` | `(cwd: string, report?: Reporter) => Promise` | +| `status` | `(cwd: string) => Promise<{ diff: Diff }>` | +| `diff` | `(cwd: string, limitToPath?: string) => Promise>` | +| `url` | `(cwd: string) => Promise` | +| `heads` | `(cwd: string, pathspec?: string) => Promise` | +| `cutWorkdir` | `(cwd: string, opts?: { name?: string }) => Promise<{ id; entries }>` | +| `pasteSnarf` | `(cwd: string, selector?: string) => Promise<{ id; entries; name? }>` | +| `showSnarfs` | `(cwd: string) => Promise` | +| `nuclearizeRepo` | `(cwd: string) => Promise` | + +Configuration and migration helpers (`migrate`, `migrations`, `detectVersion`, +`readRawConfig`, `CONFIG_VERSION`) and shape helpers (`vfsShape`, +`patchworkFolderShape`, `pinUrl`, `stripHeads`, `isInArtifactDir`, +`normalizeArtifactDir`) are exported as well. + +Key option types: + +```ts +type Backend = "legacy" | "subduction"; + +type InitOpts = { + dir: string; + backend: Backend; + shape: string; // "vfs" | "patchwork-folder" | module path + artifactDirectories?: readonly string[]; // default: ["dist"] + online?: boolean; // default: true +}; + +type CloneOpts = InitOpts & { + url: string; + // optional interactive hooks for legacy "branches" / strategy docs: + onBranchesDoc?: (info) => AutomergeUrl | Promise; + onStrategyDoc?: (info) => boolean | Promise; +}; +``` + +## Environment variables + +| Variable | Default | Purpose | +|---|---|---| +| `PUSHWORK_SUBDUCTION_SERVER` | `wss://subduction.sync.inkandswitch.com` | Subduction sync endpoint. | +| `PUSHWORK_LEGACY_SERVER` | `wss://sync3.automerge.org` | Legacy WebSocket sync endpoint. | +| `PUSHWORK_PARALLEL_INGEST` | adaptive | `shard`/`2` forces worker pools on; `off`/`0` forces them off. | +| `PUSHWORK_WORKERS` | `8` | Maximum worker-pool size for parallel ingest/clone. | +| `DEBUG` | _off_ | Set `DEBUG=true` (rewritten to `DEBUG=*`) to enable `pushwork:*` debug logs. | + +### Parallel ingest + +For large trees, pushwork shards file ingest (push) and clone (pull) across +worker threads, each owning its own Automerge `Repo` over the shared +`.pushwork/storage`. Sharding kicks in automatically at roughly 8+ files and +falls back to the main thread on any per-worker failure. Tune or disable it +with `PUSHWORK_PARALLEL_INGEST` and `PUSHWORK_WORKERS`. + +## Development + +| Script | Description | +|---|---| +| `pnpm build` | Compile TypeScript to `dist/`. | +| `pnpm dev` | `tsc --watch`. | +| `pnpm test` | Run the Vitest suite. | +| `pnpm test:watch` / `pnpm test:coverage` | Watch / coverage modes. | +| `pnpm typecheck` | `tsc --noEmit`. | +| `pnpm lint` / `pnpm lint:fix` | ESLint over `src`. | +| `pnpm bench` | Build and run the sync benchmark harness. | + +```sh +# Example benchmark runs +npx tsx bench/sync-bench.ts --files 2000 --size 512 --text 1 --fanout 20 +npx tsx bench/sync-bench.ts --clone-local --files 3000 +``` + +## License + +MIT © Ink & Switch diff --git a/package.json b/package.json index 1b62781..15ceeee 100644 --- a/package.json +++ b/package.json @@ -45,10 +45,10 @@ ], "dependencies": { "@automerge/automerge": "3.3.0-fragments.1", - "@automerge/automerge-repo": "2.6.0-subduction.32", - "@automerge/automerge-repo-network-websocket": "2.6.0-subduction.32", - "@automerge/automerge-repo-storage-nodefs": "2.6.0-subduction.32", - "@automerge/automerge-subduction": "0.15.0", + "@automerge/automerge-repo": "2.6.0-subduction.33", + "@automerge/automerge-repo-network-websocket": "2.6.0-subduction.33", + "@automerge/automerge-repo-storage-nodefs": "2.6.0-subduction.33", + "@automerge/automerge-subduction": "0.16.0", "@clack/prompts": "^1.5.1", "@commander-js/extra-typings": "^14.0.0", "chalk": "^5.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a58e98c..64a7ba4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,17 +15,17 @@ importers: specifier: 3.3.0-fragments.1 version: 3.3.0-fragments.1 '@automerge/automerge-repo': - specifier: 2.6.0-subduction.32 - version: 2.6.0-subduction.32 + specifier: 2.6.0-subduction.33 + version: 2.6.0-subduction.33 '@automerge/automerge-repo-network-websocket': - specifier: 2.6.0-subduction.32 - version: 2.6.0-subduction.32 + specifier: 2.6.0-subduction.33 + version: 2.6.0-subduction.33 '@automerge/automerge-repo-storage-nodefs': - specifier: 2.6.0-subduction.32 - version: 2.6.0-subduction.32 + specifier: 2.6.0-subduction.33 + version: 2.6.0-subduction.33 '@automerge/automerge-subduction': - specifier: 0.15.0 - version: 0.15.0 + specifier: 0.16.0 + version: 0.16.0 '@clack/prompts': specifier: ^1.5.1 version: 1.5.1 @@ -94,20 +94,20 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} - '@automerge/automerge-repo-network-websocket@2.6.0-subduction.32': - resolution: {integrity: sha512-s+OKopsE0hmzClamdZKKPdlj1oikAkE8BK6c6T4nyOBsszyrHnRQ6sjNUzLwNCAXZkQOAJ+PV0cio0VS5HmyCQ==} + '@automerge/automerge-repo-network-websocket@2.6.0-subduction.33': + resolution: {integrity: sha512-FuJSdi8LsIeFwm/wPTTwle+rAaqv7okqzICKFGPC2EUa/8+6aw/hzwDDkYvcIuuA54M3Em8XakAuvFvEtY/Grg==} engines: {node: '>=22.13'} - '@automerge/automerge-repo-storage-nodefs@2.6.0-subduction.32': - resolution: {integrity: sha512-nSpOYQvl8A4MCN3N+SHecCyJhvQ4HikHYDRWOtcMlzSu6DBuS0vhWpUfUvYrnx7ufctsonVMSyynoo8eh7INcw==} + '@automerge/automerge-repo-storage-nodefs@2.6.0-subduction.33': + resolution: {integrity: sha512-buYS6Al6eEp5C2k8rjnEAGPHMhAMzaWz0cLEvLEF5b9fFXbA578RIg9lvm3M17Sd0pjGgF1mPv1mbx04lhzgyQ==} engines: {node: '>=22.13'} - '@automerge/automerge-repo@2.6.0-subduction.32': - resolution: {integrity: sha512-2YxU1RiwC5Xz9Fz4l1anyGPZcWhj3sY6fG3GPT5Rw/5gw9oi3t4/vNhJBz0QhOZRDhkmrU4jFGXGLtwXF8ZLIA==} + '@automerge/automerge-repo@2.6.0-subduction.33': + resolution: {integrity: sha512-AxNhmrC4wuAv6DrWCz6yikdgWFPVvEUyLRjyw08DBo4Ql8LBCPPINS0gs2Ja0dtSoJdRpk4FCGWiP03L/qNZXA==} engines: {node: '>=22.13'} - '@automerge/automerge-subduction@0.15.0': - resolution: {integrity: sha512-UxD3hfzZoH9yj2/YQZZSr9bJH7R/l+xIIZ9nMPMpsBOAlEFI5AEC4hkVOqC52QOCwq4bcqek0gTYkSWWfNvc2A==} + '@automerge/automerge-subduction@0.16.0': + resolution: {integrity: sha512-BOOn/4p8cF4LhjDUQdgM3qyZFya+M7ZuZywuC99x/kbhk4UK7dw/QcsifcHG5E1TAW4R/K17ZGFUC5nCVUSSWg==} '@automerge/automerge@3.3.0-fragments.1': resolution: {integrity: sha512-Im2n3qcYSuIsLDSjofTRo7CZyyvFn5pKLSCCO5sjWzESw+9q9G25ZIrL2mczDX43cXtyzi+w/4b9VUaiNdY56Q==} @@ -1179,9 +1179,9 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 - '@automerge/automerge-repo-network-websocket@2.6.0-subduction.32': + '@automerge/automerge-repo-network-websocket@2.6.0-subduction.33': dependencies: - '@automerge/automerge-repo': 2.6.0-subduction.32 + '@automerge/automerge-repo': 2.6.0-subduction.33 cbor-x: 1.6.4 debug: 4.4.3 eventemitter3: 5.0.4 @@ -1191,18 +1191,18 @@ snapshots: - supports-color - utf-8-validate - '@automerge/automerge-repo-storage-nodefs@2.6.0-subduction.32': + '@automerge/automerge-repo-storage-nodefs@2.6.0-subduction.33': dependencies: - '@automerge/automerge-repo': 2.6.0-subduction.32 + '@automerge/automerge-repo': 2.6.0-subduction.33 transitivePeerDependencies: - bufferutil - supports-color - utf-8-validate - '@automerge/automerge-repo@2.6.0-subduction.32': + '@automerge/automerge-repo@2.6.0-subduction.33': dependencies: '@automerge/automerge': 3.3.0-fragments.1 - '@automerge/automerge-subduction': 0.15.0 + '@automerge/automerge-subduction': 0.16.0 bs58check: 4.0.0 cbor-x: 1.6.4 debug: 4.4.3 @@ -1217,7 +1217,7 @@ snapshots: - supports-color - utf-8-validate - '@automerge/automerge-subduction@0.15.0': {} + '@automerge/automerge-subduction@0.16.0': {} '@automerge/automerge@3.3.0-fragments.1': {}