wip

.github/workflows/advisory-cron.yaml

@@ -1,4 +1,4 @@
-name: Advisories
+name: ci
 on:
   schedule:
     - cron: '0 18 * * *'

.github/workflows/ci.yaml

@@ -1,11 +1,11 @@
-name: CI
+name: ci
 on:
   push:
     branches:
-      - main
+    - experiment
   pull_request:
     branches:
-      - main
+    - experiment
 jobs:
   fmt:
     runs-on: ubuntu-latest
@@ -14,8 +14,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.67.0
-          default: true
+          toolchain: stable
           components: rustfmt
       - uses: Swatinem/rust-cache@v1
       - run: ./scripts/ci/fmt
@@ -28,8 +27,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.67.0
-          default: true
+          toolchain: stable
           components: clippy
       - uses: Swatinem/rust-cache@v1
       - run: ./scripts/ci/lint
@@ -42,14 +40,9 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.67.0
-          default: true
+          toolchain: stable
       - uses: Swatinem/rust-cache@v1
-      - name: Build rust docs
-        run: ./scripts/ci/rust-docs
-        shell: bash
-      - name: Install doxygen
-        run: sudo apt-get install -y doxygen
+      - run: ./scripts/ci/docs
         shell: bash
 
   cargo-deny:
@@ -64,88 +57,40 @@ jobs:
       - uses: actions/checkout@v2
       - uses: EmbarkStudios/cargo-deny-action@v1
         with:
-          arguments: '--manifest-path ./rust/Cargo.toml'
           command: check ${{ matrix.checks }}
 
   wasm_tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install wasm-bindgen-cli
-        run: cargo install wasm-bindgen-cli wasm-opt
-      - name: Install wasm32 target
-        run: rustup target add wasm32-unknown-unknown
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: run tests
         run: ./scripts/ci/wasm_tests
-  deno_tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: denoland/setup-deno@v1
-        with:
-          deno-version: v1.x
-      - name: Install wasm-bindgen-cli
-        run: cargo install wasm-bindgen-cli wasm-opt
-      - name: Install wasm32 target
-        run: rustup target add wasm32-unknown-unknown
-      - name: run tests
-        run: ./scripts/ci/deno_tests
-
-  js_fmt:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: install
-        run: yarn global add prettier
-      - name: format
-        run: prettier -c javascript/.prettierrc javascript
 
   js_tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install wasm-bindgen-cli
-        run: cargo install wasm-bindgen-cli wasm-opt
-      - name: Install wasm32 target
-        run: rustup target add wasm32-unknown-unknown
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: run tests
         run: ./scripts/ci/js_tests
 
-  cmake_build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2023-01-26
-          default: true
-      - uses: Swatinem/rust-cache@v1
-      - name: Install CMocka
-        run: sudo apt-get install -y libcmocka-dev
-      - name: Install/update CMake
-        uses: jwlawson/actions-setup-cmake@v1.12
-        with:
-          cmake-version: latest
-      - name: Install rust-src
-        run: rustup component add rust-src
-      - name: Build and test C bindings
-        run: ./scripts/ci/cmake-build Release Static
-        shell: bash
-
   linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         toolchain:
-          - 1.67.0
+          - stable
+          - nightly
+    continue-on-error: ${{ matrix.toolchain == 'nightly' }}
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: ${{ matrix.toolchain }}
-          default: true
       - uses: Swatinem/rust-cache@v1
       - run: ./scripts/ci/build-test
         shell: bash
@@ -157,8 +102,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.67.0
-          default: true
+          toolchain: stable
       - uses: Swatinem/rust-cache@v1
       - run: ./scripts/ci/build-test
         shell: bash
@@ -170,8 +114,8 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.67.0
-          default: true
+          toolchain: stable
       - uses: Swatinem/rust-cache@v1
       - run: ./scripts/ci/build-test
         shell: bash
+

.github/workflows/docs.yaml

@@ -1,52 +0,0 @@
-on:
-  push:
-    branches:
-      - main
-
-name: Documentation
-
-jobs:
-  deploy-docs:
-    concurrency: deploy-docs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Cache
-        uses: Swatinem/rust-cache@v1
-
-      - name: Clean docs dir
-        run: rm -rf docs
-        shell: bash
-
-      - name: Clean Rust docs dir
-        uses: actions-rs/cargo@v1
-        with:
-          command: clean
-          args: --manifest-path ./rust/Cargo.toml --doc
-
-      - name: Build Rust docs
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --manifest-path ./rust/Cargo.toml --workspace --all-features --no-deps
-
-      - name: Move Rust docs
-        run: mkdir -p docs && mv rust/target/doc/* docs/.
-        shell: bash
-
-      - name: Configure root page
-        run: echo '<meta http-equiv="refresh" content="0; url=automerge">' > docs/index.html
-
-      - name: Deploy docs
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./docs

.github/workflows/release.yaml

@@ -1,214 +0,0 @@
-name: Release
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  check_if_wasm_version_upgraded:
-    name: Check if WASM version has been upgraded
-    runs-on: ubuntu-latest
-    outputs:
-      wasm_version: ${{ steps.version-updated.outputs.current-package-version }}
-      wasm_has_updated: ${{ steps.version-updated.outputs.has-updated }}
-    steps:
-      - uses: JiPaix/package-json-updated-action@v1.0.5
-        id: version-updated
-        with:
-          path: rust/automerge-wasm/package.json
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  publish-wasm:
-    name: Publish WASM package
-    runs-on: ubuntu-latest
-    needs:
-      - check_if_wasm_version_upgraded
-    # We create release only if the version in the package.json has been upgraded
-    if: needs.check_if_wasm_version_upgraded.outputs.wasm_has_updated == 'true'
-    steps:
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16.x'
-          registry-url: 'https://registry.npmjs.org'
-      - uses: denoland/setup-deno@v1
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }}
-      - name: Get rid of local github workflows
-        run: rm -r .github/workflows
-      - name: Remove tmp_branch if it exists
-        run: git push origin :tmp_branch || true
-      - run: git checkout -b tmp_branch
-      - name: Install wasm-bindgen-cli
-        run: cargo install wasm-bindgen-cli wasm-opt
-      - name: Install wasm32 target
-        run: rustup target add wasm32-unknown-unknown
-      - name: run wasm js tests
-        id: wasm_js_tests
-        run: ./scripts/ci/wasm_tests
-      - name: run wasm deno tests
-        id: wasm_deno_tests
-        run: ./scripts/ci/deno_tests
-      - name: build release
-        id: build_release
-        run: |
-          npm --prefix $GITHUB_WORKSPACE/rust/automerge-wasm run release
-      - name: Collate deno release files
-        if: steps.wasm_js_tests.outcome == 'success' && steps.wasm_deno_tests.outcome == 'success'
-        run: |
-          mkdir $GITHUB_WORKSPACE/deno_wasm_dist
-          cp $GITHUB_WORKSPACE/rust/automerge-wasm/deno/* $GITHUB_WORKSPACE/deno_wasm_dist
-          cp $GITHUB_WORKSPACE/rust/automerge-wasm/index.d.ts $GITHUB_WORKSPACE/deno_wasm_dist
-          cp $GITHUB_WORKSPACE/rust/automerge-wasm/README.md $GITHUB_WORKSPACE/deno_wasm_dist
-          cp $GITHUB_WORKSPACE/rust/automerge-wasm/LICENSE $GITHUB_WORKSPACE/deno_wasm_dist
-          sed -i '1i /// <reference types="./index.d.ts" />' $GITHUB_WORKSPACE/deno_wasm_dist/automerge_wasm.js
-      - name: Create npm release
-        if: steps.wasm_js_tests.outcome == 'success' && steps.wasm_deno_tests.outcome == 'success'
-        run: |
-          if [ "$(npm --prefix $GITHUB_WORKSPACE/rust/automerge-wasm show . version)" = "$VERSION" ]; then
-            echo "This version is already published"
-            exit 0
-          fi
-          EXTRA_ARGS="--access public"
-          if [[ $VERSION == *"alpha."* ]] || [[ $VERSION == *"beta."* ]] || [[ $VERSION == *"rc."* ]]; then
-            echo "Is pre-release version"
-            EXTRA_ARGS="$EXTRA_ARGS --tag next"
-          fi
-          if [ "$NODE_AUTH_TOKEN" = "" ]; then
-            echo "Can't publish on NPM, You need a NPM_TOKEN secret."
-            false
-          fi
-          npm publish $GITHUB_WORKSPACE/rust/automerge-wasm  $EXTRA_ARGS
-        env:
-          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
-          VERSION: ${{ needs.check_if_wasm_version_upgraded.outputs.wasm_version }}
-      - name: Commit wasm deno release files
-        run: |
-          git config --global user.name "actions"
-          git config --global user.email actions@github.com
-          git add $GITHUB_WORKSPACE/deno_wasm_dist
-          git commit -am "Add deno release files"
-          git push origin tmp_branch
-      - name: Tag wasm release
-        if: steps.wasm_js_tests.outcome == 'success' && steps.wasm_deno_tests.outcome == 'success'
-        uses: softprops/action-gh-release@v1
-        with:
-          name: Automerge Wasm v${{ needs.check_if_wasm_version_upgraded.outputs.wasm_version }}
-          tag_name: js/automerge-wasm-${{ needs.check_if_wasm_version_upgraded.outputs.wasm_version }}
-          target_commitish: tmp_branch
-          generate_release_notes: false
-          draft: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Remove tmp_branch
-        run: git push origin :tmp_branch
-  check_if_js_version_upgraded:
-    name: Check if JS version has been upgraded
-    runs-on: ubuntu-latest
-    outputs:
-      js_version: ${{ steps.version-updated.outputs.current-package-version }}
-      js_has_updated: ${{ steps.version-updated.outputs.has-updated }}
-    steps:
-      - uses: JiPaix/package-json-updated-action@v1.0.5
-        id: version-updated
-        with:
-          path: javascript/package.json
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  publish-js:
-    name: Publish JS package
-    runs-on: ubuntu-latest
-    needs:
-      - check_if_js_version_upgraded
-      - check_if_wasm_version_upgraded
-      - publish-wasm
-    # We create release only if the version in the package.json has been upgraded and after the WASM release
-    if: |
-      (always() && ! cancelled()) &&
-      (needs.publish-wasm.result == 'success' || needs.publish-wasm.result == 'skipped') && 
-      needs.check_if_js_version_upgraded.outputs.js_has_updated == 'true'
-    steps:
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16.x'
-          registry-url: 'https://registry.npmjs.org'
-      - uses: denoland/setup-deno@v1
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }}
-      - name: Get rid of local github workflows
-        run: rm -r .github/workflows
-      - name: Remove js_tmp_branch if it exists
-        run: git push origin :js_tmp_branch || true
-      - run: git checkout -b js_tmp_branch
-      - name: check js formatting
-        run: |
-          yarn global add prettier
-          prettier -c javascript/.prettierrc javascript
-      - name: run js tests
-        id: js_tests
-        run: |
-          cargo install wasm-bindgen-cli wasm-opt
-          rustup target add wasm32-unknown-unknown
-          ./scripts/ci/js_tests
-      - name: build js release
-        id: build_release
-        run: |
-          npm --prefix $GITHUB_WORKSPACE/javascript run build
-      - name: build js deno release
-        id: build_deno_release
-        run: |
-          VERSION=$WASM_VERSION npm --prefix $GITHUB_WORKSPACE/javascript run deno:build
-        env:
-          WASM_VERSION: ${{ needs.check_if_wasm_version_upgraded.outputs.wasm_version }}
-      - name: run deno tests
-        id: deno_tests
-        run: |
-          npm --prefix $GITHUB_WORKSPACE/javascript run deno:test
-      - name: Collate deno release files
-        if: steps.js_tests.outcome == 'success' && steps.deno_tests.outcome == 'success'
-        run: |
-          mkdir $GITHUB_WORKSPACE/deno_js_dist
-          cp $GITHUB_WORKSPACE/javascript/deno_dist/* $GITHUB_WORKSPACE/deno_js_dist
-      - name: Create npm release
-        if: steps.js_tests.outcome == 'success' && steps.deno_tests.outcome == 'success'
-        run: |
-          if [ "$(npm --prefix $GITHUB_WORKSPACE/javascript show . version)" = "$VERSION" ]; then
-            echo "This version is already published"
-            exit 0
-          fi
-          EXTRA_ARGS="--access public"
-          if [[ $VERSION == *"alpha."* ]] || [[ $VERSION == *"beta."* ]] || [[ $VERSION == *"rc."* ]]; then
-            echo "Is pre-release version"
-            EXTRA_ARGS="$EXTRA_ARGS --tag next"
-          fi
-          if [ "$NODE_AUTH_TOKEN" = "" ]; then
-            echo "Can't publish on NPM, You need a NPM_TOKEN secret."
-            false
-          fi
-          npm publish $GITHUB_WORKSPACE/javascript $EXTRA_ARGS
-        env:
-          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
-          VERSION: ${{ needs.check_if_js_version_upgraded.outputs.js_version }}
-      - name: Commit js deno release files
-        run: |
-          git config --global user.name "actions"
-          git config --global user.email actions@github.com
-          git add $GITHUB_WORKSPACE/deno_js_dist
-          git commit -am "Add deno js release files"
-          git push origin js_tmp_branch
-      - name: Tag JS release
-        if: steps.js_tests.outcome == 'success' && steps.deno_tests.outcome == 'success'
-        uses: softprops/action-gh-release@v1
-        with:
-          name: Automerge v${{ needs.check_if_js_version_upgraded.outputs.js_version }}
-          tag_name: js/automerge-${{ needs.check_if_js_version_upgraded.outputs.js_version }}
-          target_commitish: js_tmp_branch
-          generate_release_notes: false
-          draft: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Remove js_tmp_branch
-        run: git push origin :js_tmp_branch

.gitignore

@@ -1,6 +1,6 @@
+/target
 /.direnv
 perf.*
 /Cargo.lock
-build/
-.vim/*
-/target
+automerge/proptest-regressions/
+.vim

.vim/coc-settings.json

@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.cargo.features": ["optree-visualisation", "storage-v2"]
+}

Cargo.toml

@@ -1,17 +1,16 @@
 [workspace]
 members = [
     "automerge",
-    "automerge-c",
     "automerge-cli",
-    "automerge-test",
     "automerge-wasm",
+    "automerge-cli",
     "edit-trace",
 ]
-resolver = "2"
 
 [profile.release]
+debug = true
 lto = true
-codegen-units = 1
+opt-level = 3
 
 [profile.bench]
 debug = true

Makefile

@@ -0,0 +1,13 @@
+rust:
+	cd automerge && cargo test
+
+wasm:
+	cd automerge-wasm && yarn
+	cd automerge-wasm && yarn build
+	cd automerge-wasm && yarn test
+	cd automerge-wasm && yarn link
+
+js: wasm
+	cd automerge-js && yarn
+	cd automerge-js && yarn link "automerge-wasm"
+	cd automerge-js && yarn test

README.md

@@ -1,147 +1,81 @@
-# Automerge
+# Automerge - NEXT
 
-<img src='./img/sign.svg' width='500' alt='Automerge logo' />
+This is pretty much a ground up rewrite of automerge-rs. The objective of this
+rewrite is to radically simplify the API. The end goal being to produce a library
+which is easy to work with both in Rust and from FFI.
 
-[![homepage](https://img.shields.io/badge/homepage-published-informational)](https://automerge.org/)
-[![main docs](https://img.shields.io/badge/docs-main-informational)](https://automerge.org/automerge-rs/automerge/)
-[![ci](https://github.com/automerge/automerge-rs/actions/workflows/ci.yaml/badge.svg)](https://github.com/automerge/automerge-rs/actions/workflows/ci.yaml)
-[![docs](https://github.com/automerge/automerge-rs/actions/workflows/docs.yaml/badge.svg)](https://github.com/automerge/automerge-rs/actions/workflows/docs.yaml)
+## How?
 
-Automerge is a library which provides fast implementations of several different
-CRDTs, a compact compression format for these CRDTs, and a sync protocol for
-efficiently transmitting those changes over the network. The objective of the
-project is to support [local-first](https://www.inkandswitch.com/local-first/) applications in the same way that relational
-databases support server applications - by providing mechanisms for persistence
-which allow application developers to avoid thinking about hard distributed
-computing problems. Automerge aims to be PostgreSQL for your local-first app.
+The current iteration of automerge-rs is complicated to work with because it
+adopts the frontend/backend split architecture of the JS implementation. This
+architecture was necessary due to basic operations on the automerge opset being
+too slow to perform on the UI thread. Recently @orionz has been able to improve
+the performance to the point where the split is no longer necessary. This means
+we can adopt a much simpler mutable API.
 
-If you're looking for documentation on the JavaScript implementation take a look
-at https://automerge.org/docs/hello/. There are other implementations in both
-Rust and C, but they are earlier and don't have documentation yet. You can find
-them in `rust/automerge` and `rust/automerge-c` if you are comfortable
-reading the code and tests to figure out how to use them.
-
-If you're familiar with CRDTs and interested in the design of Automerge in
-particular take a look at https://automerge.org/docs/how-it-works/backend/
-
-Finally, if you want to talk to us about this project please [join the
-Slack](https://join.slack.com/t/automerge/shared_invite/zt-e4p3760n-kKh7r3KRH1YwwNfiZM8ktw)
+The architecture is now built around the `OpTree`. This is a data structure
+which supports efficiently inserting new operations and realising values of
+existing operations. Most interactions with the `OpTree` are in the form of
+implementations of `TreeQuery` - a trait which can be used to traverse the
+optree and producing state of some kind. User facing operations are exposed on
+an `Automerge` object, under the covers these operations typically instantiate
+some `TreeQuery` and run it over the `OpTree`.
 
 ## Status
 
-This project is formed of a core Rust implementation which is exposed via FFI in
-javascript+WASM, C, and soon other languages. Alex
-([@alexjg](https://github.com/alexjg/)]) is working full time on maintaining
-automerge, other members of Ink and Switch are also contributing time and there
-are several other maintainers. The focus is currently on shipping the new JS
-package. We expect to be iterating the API and adding new features over the next
-six months so there will likely be several major version bumps in all packages
-in that time.
+We have working code which passes all of the tests in the JS test suite. We're
+now working on writing a bunch more tests and cleaning up the API.
 
-In general we try and respect semver.
+## Development
 
-### JavaScript
+### Running CI
 
-A stable release of the javascript package is currently available as
-`@automerge/automerge@2.0.0` where. pre-release verisions of the `2.0.1` are
-available as `2.0.1-alpha.n`. `2.0.1*` packages are also available for Deno at
-https://deno.land/x/automerge
+The steps CI will run are all defined in `./scripts/ci`. Obviously CI will run
+everything when you submit a PR, but if you want to run everything locally
+before you push you can run `./scripts/ci/run` to run everything.
 
-### Rust
+### Running the JS tests
 
-The rust codebase is currently oriented around producing a performant backend
-for the Javascript wrapper and as such the API for Rust code is low level and
-not well documented. We will be returning to this over the next few months but
-for now you will need to be comfortable reading the tests and asking questions
-to figure out how to use it. If you are looking to build rust applications which
-use automerge you may want to look into
-[autosurgeon](https://github.com/alexjg/autosurgeon)
+You will need to have [node](https://nodejs.org/en/), [yarn](https://yarnpkg.com/getting-started/install), [rust](https://rustup.rs/) and [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) installed.
 
-## Repository Organisation
+To build and test the rust library:
 
-- `./rust` - the rust rust implementation and also the Rust components of
-  platform specific wrappers (e.g. `automerge-wasm` for the WASM API or
-  `automerge-c` for the C FFI bindings)
-- `./javascript` - The javascript library which uses `automerge-wasm`
-  internally but presents a more idiomatic javascript interface
-- `./scripts` - scripts which are useful to maintenance of the repository.
-  This includes the scripts which are run in CI.
-- `./img` - static assets for use in `.md` files
-
-## Building
-
-To build this codebase you will need:
-
-- `rust`
-- `node`
-- `yarn`
-- `cmake`
-- `cmocka`
-
-You will also need to install the following with `cargo install`
-
-- `wasm-bindgen-cli`
-- `wasm-opt`
-- `cargo-deny`
-
-And ensure you have added the `wasm32-unknown-unknown` target for rust cross-compilation.
-
-The various subprojects (the rust code, the wrapper projects) have their own
-build instructions, but to run the tests that will be run in CI you can run
-`./scripts/ci/run`.
-
-### For macOS
-
-These instructions worked to build locally on macOS 13.1 (arm64) as of
-Nov 29th 2022.
-
-```bash
-# clone the repo
-git clone https://github.com/automerge/automerge-rs
-cd automerge-rs
-
-# install rustup
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-
-# install homebrew
-/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-
-# install cmake, node, cmocka
-brew install cmake node cmocka
-
-# install yarn
-npm install --global yarn
-
-# install javascript dependencies
-yarn --cwd ./javascript
-
-# install rust dependencies
-cargo install wasm-bindgen-cli wasm-opt cargo-deny
-
-# get nightly rust to produce optimized automerge-c builds
-rustup toolchain install nightly
-rustup component add rust-src --toolchain nightly
-
-# add wasm target in addition to current architecture
-rustup target add wasm32-unknown-unknown
-
-# Run ci script
-./scripts/ci/run
+```shell
+  $ cd automerge
+  $ cargo test
 ```
 
-If your build fails to find `cmocka.h` you may need to teach it about homebrew's
-installation location:
+To build and test the wasm library:
 
-```
-export CPATH=/opt/homebrew/include
-export LIBRARY_PATH=/opt/homebrew/lib
-./scripts/ci/run
+```shell
+  ## setup
+  $ cd automerge-wasm
+  $ yarn
+
+  ## building or testing
+  $ yarn build
+  $ yarn test
+
+  ## without this the js library wont automatically use changes
+  $ yarn link
+
+  ## cutting a release or doing benchmarking
+  $ yarn release
+  $ yarn opt ## or set `wasm-opt = false` in Cargo.toml on supported platforms (not arm64 osx)
 ```
 
-## Contributing
+And finally to test the js library. This is where most of the tests reside.
 
-Please try and split your changes up into relatively independent commits which
-change one subsystem at a time and add good commit messages which describe what
-the change is and why you're making it (err on the side of longer commit
-messages). `git blame` should give future maintainers a good idea of why
-something is the way it is.
+```shell
+  ## setup
+  $ cd automerge-js
+  $ yarn
+  $ yarn link "automerge-wasm"
+
+  ## testing
+  $ yarn test
+```
+
+## Benchmarking
+
+The `edit-trace` folder has the main code for running the edit trace benchmarking.

TODO.md

@@ -0,0 +1,32 @@
+### next steps:
+  1. C API
+  2. port rust command line tool
+  3. fast load
+
+### ergonomics:
+  1. value() -> () or something that into's a value
+
+### automerge:
+  1. single pass (fast) load
+  2. micro-patches / bare bones observation API / fully hydrated documents
+
+### future:
+  1. handle columns with unknown data in and out
+  2. branches with different indexes
+
+### Peritext
+  1. add mark / remove mark -- type, start/end elemid (inclusive,exclusive)
+  2. track any formatting ops that start or end on a character
+  3. ops right before the character, ops right after that character
+  4. query a single character - character, plus marks that start or end on that character
+     what is its current formatting,
+     what are the ops that include that in their span,
+     None = same as last time, Set( bold, italic ),
+     keep these on index
+  5. op probably belongs with the start character - possible packed at the beginning or end of the list
+
+### maybe:
+  1. tables
+
+### no:
+  1. cursors

automerge-cli/Cargo.lock

@@ -0,0 +1,857 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd"
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "automerge"
+version = "0.1.0"
+dependencies = [
+ "flate2",
+ "fxhash",
+ "hex",
+ "itertools",
+ "js-sys",
+ "leb128",
+ "nonzero_ext",
+ "rand",
+ "serde",
+ "sha2",
+ "smol_str",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+ "unicode-segmentation",
+ "uuid",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "automerge-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "atty",
+ "automerge",
+ "clap",
+ "colored_json",
+ "combine",
+ "duct",
+ "maplit",
+ "serde_json",
+ "thiserror",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "bytes"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "3.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced1892c55c910c1219e98d6fc8d71f6bddba7905866ce740066d8bfea859312"
+dependencies = [
+ "atty",
+ "bitflags",
+ "clap_derive",
+ "indexmap",
+ "lazy_static",
+ "os_str_bytes",
+ "strsim",
+ "termcolor",
+ "textwrap",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "colored_json"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd32eb54d016e203b7c2600e3a7802c75843a92e38ccc4869aefeca21771a64"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "libc",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "combine"
+version = "4.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50b727aacc797f9fc28e355d21f34709ac4fc9adecfe470ad07b8f4464f53062"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
+name = "duct"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fc6a0a59ed0888e0041cf708e66357b7ae1a82f1c67247e1f93b5e0818f7d8d"
+dependencies = [
+ "libc",
+ "once_cell",
+ "os_pipe",
+ "shared_child",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
+[[package]]
+name = "flate2"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "wasi",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "heck"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "indexmap"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+
+[[package]]
+name = "js-sys"
+version = "0.3.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "leb128"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
+
+[[package]]
+name = "libc"
+version = "0.2.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
+dependencies = [
+ "adler",
+ "autocfg",
+]
+
+[[package]]
+name = "nonzero_ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44a1290799eababa63ea60af0cbc3f03363e328e58f32fb0294798ed3e85f444"
+
+[[package]]
+name = "once_cell"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
+
+[[package]]
+name = "os_pipe"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "os_str_bytes"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+
+[[package]]
+name = "serde"
+version = "1.0.136"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.136"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sharded-slab"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "shared_child"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6be9f7d5565b1483af3e72975e2dee33879b3b86bd48c0929fccf6585d79e65a"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+
+[[package]]
+name = "smol_str"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61d15c83e300cce35b7c8cd39ff567c1ef42dde6d4a1a38dbdbf9a59902261bd"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
+
+[[package]]
+name = "thiserror"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+
+[[package]]
+name = "tracing"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f"
+dependencies = [
+ "cfg-if",
+ "log",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8276d9a4a3a558d7b7ad5303ad50b53d58264641b82914b7ada36bd762e7a716"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23"
+dependencies = [
+ "lazy_static",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3"
+dependencies = [
+ "lazy_static",
+ "log",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce"
+dependencies = [
+ "ansi_term",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing-core",
+ "tracing-log",
+]
+
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom",
+ "serde",
+]
+
+[[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
+dependencies = [
+ "bumpalo",
+ "lazy_static",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
+
+[[package]]
+name = "web-sys"
+version = "0.3.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

automerge-cli/Cargo.toml

@@ -4,7 +4,6 @@ version = "0.1.0"
 authors = ["Alex Good <alex@memoryandthought.me>"]
 edition = "2018"
 license = "MIT"
-rust-version = "1.57.0"
 
 [[bin]]
 name = "automerge"
@@ -13,18 +12,17 @@ bench = false
 doc = false
 
 [dependencies]
-clap = {version = "~4", features = ["derive"]}
+clap = {version = "~3.1", features = ["derive"]}
 serde_json = "^1.0"
 anyhow = "1.0"
+atty = "^0.2"
 thiserror = "^1.0"
 combine = "^4.5"
 maplit = "^1.0"
+colored_json = "^2.1"
 tracing-subscriber = "~0.3"
 
 automerge = { path = "../automerge" }
-is-terminal = "0.4.1"
-termcolor = "1.1.3"
-serde = "1.0.150"
 
 [dev-dependencies]
 duct = "^0.13"

automerge-cli/src/examine.rs

@@ -1,8 +1,6 @@
 use automerge as am;
 use thiserror::Error;
 
-use crate::{color_json::print_colored_json, SkipVerifyFlag};
-
 #[derive(Error, Debug)]
 pub enum ExamineError {
     #[error("Error reading change file: {:?}", source)]
@@ -22,29 +20,22 @@ pub enum ExamineError {
     },
 }
 
-pub(crate) fn examine(
+pub fn examine(
     mut input: impl std::io::Read,
     mut output: impl std::io::Write,
-    skip: SkipVerifyFlag,
     is_tty: bool,
 ) -> Result<(), ExamineError> {
     let mut buf: Vec<u8> = Vec::new();
     input
         .read_to_end(&mut buf)
         .map_err(|e| ExamineError::ReadingChanges { source: e })?;
-    let doc = skip
-        .load(&buf)
+    let doc = am::Automerge::load(&buf)
         .map_err(|e| ExamineError::ApplyingInitialChanges { source: e })?;
-    let uncompressed_changes: Vec<_> = doc
-        .get_changes(&[])
-        .unwrap()
-        .iter()
-        .map(|c| c.decode())
-        .collect();
+    let uncompressed_changes: Vec<_> = doc.get_changes(&[]).iter().map(|c| c.decode()).collect();
     if is_tty {
         let json_changes = serde_json::to_value(uncompressed_changes).unwrap();
-        print_colored_json(&json_changes).unwrap();
-        writeln!(output).unwrap();
+        colored_json::write_colored_json(&json_changes, &mut output).unwrap();
+        writeln!(&mut output).unwrap();
     } else {
         let json_changes = serde_json::to_string_pretty(&uncompressed_changes).unwrap();
         output

automerge-cli/src/export.rs

@@ -1,14 +1,11 @@
 use anyhow::Result;
 use automerge as am;
-use automerge::ReadDoc;
-
-use crate::{color_json::print_colored_json, SkipVerifyFlag};
 
 pub(crate) fn map_to_json(doc: &am::Automerge, obj: &am::ObjId) -> serde_json::Value {
     let keys = doc.keys(obj);
     let mut map = serde_json::Map::new();
     for k in keys {
-        let val = doc.get(obj, &k);
+        let val = doc.value(obj, &k);
         match val {
             Ok(Some((am::Value::Object(o), exid)))
                 if o == am::ObjType::Map || o == am::ObjType::Table =>
@@ -31,7 +28,7 @@ fn list_to_json(doc: &am::Automerge, obj: &am::ObjId) -> serde_json::Value {
     let len = doc.length(obj);
     let mut array = Vec::new();
     for i in 0..len {
-        let val = doc.get(obj, i);
+        let val = doc.value(obj, i as usize);
         match val {
             Ok(Some((am::Value::Object(o), exid)))
                 if o == am::ObjType::Map || o == am::ObjType::Table =>
@@ -53,13 +50,11 @@ fn list_to_json(doc: &am::Automerge, obj: &am::ObjId) -> serde_json::Value {
 fn scalar_to_json(val: &am::ScalarValue) -> serde_json::Value {
     match val {
         am::ScalarValue::Str(s) => serde_json::Value::String(s.to_string()),
-        am::ScalarValue::Bytes(b) | am::ScalarValue::Unknown { bytes: b, .. } => {
-            serde_json::Value::Array(
-                b.iter()
-                    .map(|byte| serde_json::Value::Number((*byte).into()))
-                    .collect(),
-            )
-        }
+        am::ScalarValue::Bytes(b) => serde_json::Value::Array(
+            b.iter()
+                .map(|byte| serde_json::Value::Number((*byte).into()))
+                .collect(),
+        ),
         am::ScalarValue::Int(n) => serde_json::Value::Number((*n).into()),
         am::ScalarValue::Uint(n) => serde_json::Value::Number((*n).into()),
         am::ScalarValue::F64(n) => serde_json::Number::from_f64(*n)
@@ -72,23 +67,22 @@ fn scalar_to_json(val: &am::ScalarValue) -> serde_json::Value {
     }
 }
 
-fn get_state_json(input_data: Vec<u8>, skip: SkipVerifyFlag) -> Result<serde_json::Value> {
-    let doc = skip.load(&input_data).unwrap(); // FIXME
+fn get_state_json(input_data: Vec<u8>) -> Result<serde_json::Value> {
+    let doc = am::Automerge::load(&input_data).unwrap(); // FIXME
     Ok(map_to_json(&doc, &am::ObjId::Root))
 }
 
-pub(crate) fn export_json(
+pub fn export_json(
     mut changes_reader: impl std::io::Read,
     mut writer: impl std::io::Write,
-    skip: SkipVerifyFlag,
     is_tty: bool,
 ) -> Result<()> {
     let mut input_data = vec![];
     changes_reader.read_to_end(&mut input_data)?;
 
-    let state_json = get_state_json(input_data, skip)?;
+    let state_json = get_state_json(input_data)?;
     if is_tty {
-        print_colored_json(&state_json).unwrap();
+        colored_json::write_colored_json(&state_json, &mut writer).unwrap();
         writeln!(writer).unwrap();
     } else {
         writeln!(
@@ -107,10 +101,7 @@ mod tests {
 
     #[test]
     fn cli_export_with_empty_input() {
-        assert_eq!(
-            get_state_json(vec![], Default::default()).unwrap(),
-            serde_json::json!({})
-        )
+        assert_eq!(get_state_json(vec![]).unwrap(), serde_json::json!({}))
     }
 
     #[test]
@@ -124,7 +115,7 @@ mod tests {
         let mut backend = initialize_from_json(&initial_state_json).unwrap();
         let change_bytes = backend.save();
         assert_eq!(
-            get_state_json(change_bytes, Default::default()).unwrap(),
+            get_state_json(change_bytes).unwrap(),
             serde_json::json!({"sparrows": 15.0})
         )
     }
@@ -151,7 +142,7 @@ mod tests {
         */
         let change_bytes = backend.save();
         assert_eq!(
-            get_state_json(change_bytes, Default::default()).unwrap(),
+            get_state_json(change_bytes).unwrap(),
             serde_json::json!({
                 "birds": {
                     "wrens": 3.0,

automerge-cli/src/import.rs

@@ -3,14 +3,14 @@ use automerge::transaction::Transactable;
 
 pub(crate) fn initialize_from_json(
     json_value: &serde_json::Value,
-) -> anyhow::Result<am::AutoCommit> {
+) -> Result<am::AutoCommit, am::AutomergeError> {
     let mut doc = am::AutoCommit::new();
     match json_value {
         serde_json::Value::Object(m) => {
             import_map(&mut doc, &am::ObjId::Root, m)?;
             Ok(doc)
         }
-        _ => anyhow::bail!("expected an object"),
+        _ => Err(am::AutomergeError::Decoding),
     }
 }
 
@@ -18,35 +18,35 @@ fn import_map(
     doc: &mut am::AutoCommit,
     obj: &am::ObjId,
     map: &serde_json::Map<String, serde_json::Value>,
-) -> anyhow::Result<()> {
+) -> Result<(), am::AutomergeError> {
     for (key, value) in map {
         match value {
             serde_json::Value::Null => {
-                doc.put(obj, key, ())?;
+                doc.set(obj, key, ())?;
             }
             serde_json::Value::Bool(b) => {
-                doc.put(obj, key, *b)?;
+                doc.set(obj, key, *b)?;
             }
             serde_json::Value::String(s) => {
-                doc.put(obj, key, s)?;
+                doc.set(obj, key, s.as_ref())?;
             }
             serde_json::Value::Array(vec) => {
-                let id = doc.put_object(obj, key, am::ObjType::List)?;
+                let id = doc.set_object(obj, key, am::ObjType::List)?;
                 import_list(doc, &id, vec)?;
             }
             serde_json::Value::Number(n) => {
                 if let Some(m) = n.as_i64() {
-                    doc.put(obj, key, m)?;
+                    doc.set(obj, key, m)?;
                 } else if let Some(m) = n.as_u64() {
-                    doc.put(obj, key, m)?;
+                    doc.set(obj, key, m)?;
                 } else if let Some(m) = n.as_f64() {
-                    doc.put(obj, key, m)?;
+                    doc.set(obj, key, m)?;
                 } else {
-                    anyhow::bail!("not a number");
+                    return Err(am::AutomergeError::Decoding);
                 }
             }
             serde_json::Value::Object(map) => {
-                let id = doc.put_object(obj, key, am::ObjType::Map)?;
+                let id = doc.set_object(obj, key, am::ObjType::Map)?;
                 import_map(doc, &id, map)?;
             }
         }
@@ -58,7 +58,7 @@ fn import_list(
     doc: &mut am::AutoCommit,
     obj: &am::ObjId,
     list: &[serde_json::Value],
-) -> anyhow::Result<()> {
+) -> Result<(), am::AutomergeError> {
     for (i, value) in list.iter().enumerate() {
         match value {
             serde_json::Value::Null => {
@@ -68,7 +68,7 @@ fn import_list(
                 doc.insert(obj, i, *b)?;
             }
             serde_json::Value::String(s) => {
-                doc.insert(obj, i, s)?;
+                doc.insert(obj, i, s.as_ref())?;
             }
             serde_json::Value::Array(vec) => {
                 let id = doc.insert_object(obj, i, am::ObjType::List)?;
@@ -82,7 +82,7 @@ fn import_list(
                 } else if let Some(m) = n.as_f64() {
                     doc.insert(obj, i, m)?;
                 } else {
-                    anyhow::bail!("not a number");
+                    return Err(am::AutomergeError::Decoding);
                 }
             }
             serde_json::Value::Object(map) => {

automerge-cli/src/main.rs

@@ -1,15 +1,10 @@
 use std::{fs::File, path::PathBuf, str::FromStr};
 
 use anyhow::{anyhow, Result};
-use clap::{
-    builder::{BoolishValueParser, TypedValueParser, ValueParserFactory},
-    Parser,
-};
-use is_terminal::IsTerminal;
+use clap::Parser;
 
-mod color_json;
+//mod change;
 mod examine;
-mod examine_sync;
 mod export;
 mod import;
 mod merge;
@@ -21,50 +16,12 @@ struct Opts {
     cmd: Command,
 }
 
-#[derive(clap::ValueEnum, Clone, Debug)]
+#[derive(Debug)]
 enum ExportFormat {
     Json,
     Toml,
 }
 
-#[derive(Copy, Clone, Default, Debug)]
-pub(crate) struct SkipVerifyFlag(bool);
-
-impl SkipVerifyFlag {
-    fn load(&self, buf: &[u8]) -> Result<automerge::Automerge, automerge::AutomergeError> {
-        if self.0 {
-            automerge::Automerge::load(buf)
-        } else {
-            automerge::Automerge::load_unverified_heads(buf)
-        }
-    }
-}
-
-#[derive(Clone)]
-struct SkipVerifyFlagParser;
-impl ValueParserFactory for SkipVerifyFlag {
-    type Parser = SkipVerifyFlagParser;
-
-    fn value_parser() -> Self::Parser {
-        SkipVerifyFlagParser
-    }
-}
-
-impl TypedValueParser for SkipVerifyFlagParser {
-    type Value = SkipVerifyFlag;
-
-    fn parse_ref(
-        &self,
-        cmd: &clap::Command,
-        arg: Option<&clap::Arg>,
-        value: &std::ffi::OsStr,
-    ) -> Result<Self::Value, clap::Error> {
-        BoolishValueParser::new()
-            .parse_ref(cmd, arg, value)
-            .map(SkipVerifyFlag)
-    }
-}
-
 impl FromStr for ExportFormat {
     type Err = anyhow::Error;
 
@@ -86,15 +43,12 @@ enum Command {
         format: ExportFormat,
 
         /// Path that contains Automerge changes
+        #[clap(parse(from_os_str))]
         changes_file: Option<PathBuf>,
 
         /// The file to write to. If omitted assumes stdout
-        #[clap(long("out"), short('o'))]
+        #[clap(parse(from_os_str), long("out"), short('o'))]
         output_file: Option<PathBuf>,
-
-        /// Whether to verify the head hashes of a compressed document
-        #[clap(long, action = clap::ArgAction::SetFalse)]
-        skip_verifying_heads: SkipVerifyFlag,
     },
 
     Import {
@@ -102,37 +56,69 @@ enum Command {
         #[clap(long, short, default_value = "json")]
         format: ExportFormat,
 
+        #[clap(parse(from_os_str))]
         input_file: Option<PathBuf>,
 
         /// Path to write Automerge changes to
-        #[clap(long("out"), short('o'))]
+        #[clap(parse(from_os_str), long("out"), short('o'))]
         changes_file: Option<PathBuf>,
     },
 
-    /// Read an automerge document and print a JSON representation of the changes in it to stdout
-    Examine {
+    /// Read an automerge document from a file or stdin, perform a change on it and write a new
+    /// document to stdout or the specified output file.
+    Change {
+        /// The change script to perform. Change scripts have the form <command> <path> [<JSON value>].
+        /// The possible commands are 'set', 'insert', 'delete', and 'increment'.
+        ///
+        /// Paths look like this: $["mapkey"][0]. They always lways start with a '$', then each
+        /// subsequent segment of the path is either a string in double quotes to index a key in a
+        /// map, or an integer index to address an array element.
+        ///
+        /// Examples
+        ///
+        /// ## set
+        ///
+        /// > automerge change 'set $["someobject"] {"items": []}' somefile
+        ///
+        /// ## insert
+        ///
+        /// > automerge change 'insert $["someobject"]["items"][0] "item1"' somefile
+        ///
+        /// ## increment
+        ///
+        /// > automerge change 'increment $["mycounter"]'
+        ///
+        /// ## delete
+        ///
+        /// > automerge change 'delete $["someobject"]["items"]' somefile
+        script: String,
+
+        /// The file to change, if omitted will assume stdin
+        #[clap(parse(from_os_str))]
         input_file: Option<PathBuf>,
-        skip_verifying_heads: SkipVerifyFlag,
+
+        /// Path to write Automerge changes to, if omitted will write to stdout
+        #[clap(parse(from_os_str), long("out"), short('o'))]
+        output_file: Option<PathBuf>,
     },
 
-    /// Read an automerge sync messaage and print a JSON representation of it
-    ExamineSync { input_file: Option<PathBuf> },
+    /// Read an automerge document and print a JSON representation of the changes in it to stdout
+    Examine { input_file: Option<PathBuf> },
 
     /// Read one or more automerge documents and output a merged, compacted version of them
     Merge {
         /// The file to write to. If omitted assumes stdout
-        #[clap(long("out"), short('o'))]
+        #[clap(parse(from_os_str), long("out"), short('o'))]
         output_file: Option<PathBuf>,
-
         /// The file(s) to compact. If empty assumes stdin
         input: Vec<PathBuf>,
     },
 }
 
 fn open_file_or_stdin(maybe_path: Option<PathBuf>) -> Result<Box<dyn std::io::Read>> {
-    if std::io::stdin().is_terminal() {
+    if atty::is(atty::Stream::Stdin) {
         if let Some(path) = maybe_path {
-            Ok(Box::new(File::open(path).unwrap()))
+            Ok(Box::new(File::open(&path).unwrap()))
         } else {
             Err(anyhow!(
                 "Must provide file path if not providing input via stdin"
@@ -144,9 +130,9 @@ fn open_file_or_stdin(maybe_path: Option<PathBuf>) -> Result<Box<dyn std::io::Re
 }
 
 fn create_file_or_stdout(maybe_path: Option<PathBuf>) -> Result<Box<dyn std::io::Write>> {
-    if std::io::stdout().is_terminal() {
+    if atty::is(atty::Stream::Stdout) {
         if let Some(path) = maybe_path {
-            Ok(Box::new(File::create(path).unwrap()))
+            Ok(Box::new(File::create(&path).unwrap()))
         } else {
             Err(anyhow!("Must provide file path if not piping to stdout"))
         }
@@ -163,22 +149,16 @@ fn main() -> Result<()> {
             changes_file,
             format,
             output_file,
-            skip_verifying_heads,
         } => {
             let output: Box<dyn std::io::Write> = if let Some(output_file) = output_file {
-                Box::new(File::create(output_file)?)
+                Box::new(File::create(&output_file)?)
             } else {
                 Box::new(std::io::stdout())
             };
             match format {
                 ExportFormat::Json => {
                     let mut in_buffer = open_file_or_stdin(changes_file)?;
-                    export::export_json(
-                        &mut in_buffer,
-                        output,
-                        skip_verifying_heads,
-                        std::io::stdout().is_terminal(),
-                    )
+                    export::export_json(&mut in_buffer, output, atty::is(atty::Stream::Stdout))
                 }
                 ExportFormat::Toml => unimplemented!(),
             }
@@ -195,30 +175,23 @@ fn main() -> Result<()> {
             }
             ExportFormat::Toml => unimplemented!(),
         },
-        Command::Examine {
-            input_file,
-            skip_verifying_heads,
+        Command::Change { ..
+            //input_file,
+            //output_file,
+            //script,
         } => {
+            unimplemented!()
+/*
             let in_buffer = open_file_or_stdin(input_file)?;
-            let out_buffer = std::io::stdout();
-            match examine::examine(
-                in_buffer,
-                out_buffer,
-                skip_verifying_heads,
-                std::io::stdout().is_terminal(),
-            ) {
-                Ok(()) => {}
-                Err(e) => {
-                    eprintln!("Error: {:?}", e);
-                }
-            }
-            Ok(())
+            let mut out_buffer = create_file_or_stdout(output_file)?;
+                        change::change(in_buffer, &mut out_buffer, script.as_str())
+                            .map_err(|e| anyhow::format_err!("Unable to make changes: {:?}", e))
+*/
         }
-        Command::ExamineSync { input_file } => {
+        Command::Examine { input_file } => {
             let in_buffer = open_file_or_stdin(input_file)?;
             let out_buffer = std::io::stdout();
-            match examine_sync::examine_sync(in_buffer, out_buffer, std::io::stdout().is_terminal())
-            {
+            match examine::examine(in_buffer, out_buffer, atty::is(atty::Stream::Stdout)) {
                 Ok(()) => {}
                 Err(e) => {
                     eprintln!("Error: {:?}", e);

automerge-js/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+/yarn.lock

automerge-js/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "automerge-js",
+  "version": "0.1.0",
+  "main": "src/index.js",
+  "license": "MIT",
+  "scripts": {
+    "test": "mocha --bail --full-trace"
+  },
+  "devDependencies": {
+    "mocha": "^9.1.1"
+  },
+  "dependencies": {
+    "automerge-wasm": "file:../automerge-wasm/dev",
+    "fast-sha256": "^1.3.0",
+    "pako": "^2.0.4",
+    "uuid": "^8.3"
+  }
+}

automerge-js/src/constants.js

@@ -0,0 +1,18 @@
+// Properties of the document root object
+//const OPTIONS   = Symbol('_options')   // object containing options passed to init()
+//const CACHE     = Symbol('_cache')     // map from objectId to immutable object
+const STATE      = Symbol('_state')     // object containing metadata about current state (e.g. sequence numbers)
+const HEADS      = Symbol('_heads')     // object containing metadata about current state (e.g. sequence numbers)
+const OBJECT_ID  = Symbol('_objectId')     // object containing metadata about current state (e.g. sequence numbers)
+const READ_ONLY  = Symbol('_readOnly')     // object containing metadata about current state (e.g. sequence numbers)
+const FROZEN     = Symbol('_frozen')     // object containing metadata about current state (e.g. sequence numbers)
+
+// Properties of all Automerge objects
+//const OBJECT_ID = Symbol('_objectId')  // the object ID of the current object (string)
+//const CONFLICTS = Symbol('_conflicts') // map or list (depending on object type) of conflicts
+//const CHANGE    = Symbol('_change')    // the context object on proxy objects used in change callback
+//const ELEM_IDS  = Symbol('_elemIds')   // list containing the element ID of each list element
+
+module.exports = {
+  STATE, HEADS, OBJECT_ID, READ_ONLY, FROZEN
+}

automerge-js/src/counter.js

@@ -1,16 +1,12 @@
-import { Automerge, type ObjID, type Prop } from "@automerge/automerge-wasm"
-import { COUNTER } from "./constants"
 /**
  * The most basic CRDT: an integer value that can be changed only by
  * incrementing and decrementing. Since addition of integers is commutative,
  * the value trivially converges.
  */
-export class Counter {
-  value: number
-
-  constructor(value?: number) {
+class Counter {
+  constructor(value) {
     this.value = value || 0
-    Reflect.defineProperty(this, COUNTER, { value: true })
+    Object.freeze(this)
   }
 
   /**
@@ -21,7 +17,7 @@ export class Counter {
    * concatenating it with another string, as in `x + ''`.
    * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/valueOf
    */
-  valueOf(): number {
+  valueOf() {
     return this.value
   }
 
@@ -30,7 +26,7 @@ export class Counter {
    * this method is called e.g. when you do `['value: ', x].join('')` or when
    * you use string interpolation: `value: ${x}`.
    */
-  toString(): string {
+  toString() {
     return this.valueOf().toString()
   }
 
@@ -38,7 +34,7 @@ export class Counter {
    * Returns the counter value, so that a JSON serialization of an Automerge
    * document represents the counter simply as an integer.
    */
-  toJSON(): number {
+  toJSON() {
     return this.value
   }
 }
@@ -48,32 +44,13 @@ export class Counter {
  * callback.
  */
 class WriteableCounter extends Counter {
-  context: Automerge
-  path: Prop[]
-  objectId: ObjID
-  key: Prop
-
-  constructor(
-    value: number,
-    context: Automerge,
-    path: Prop[],
-    objectId: ObjID,
-    key: Prop
-  ) {
-    super(value)
-    this.context = context
-    this.path = path
-    this.objectId = objectId
-    this.key = key
-  }
-
   /**
    * Increases the value of the counter by `delta`. If `delta` is not given,
    * increases the value of the counter by 1.
    */
-  increment(delta: number): number {
-    delta = typeof delta === "number" ? delta : 1
-    this.context.increment(this.objectId, this.key, delta)
+  increment(delta) {
+    delta = typeof delta === 'number' ? delta : 1
+    this.context.inc(this.objectId, this.key, delta)
     this.value += delta
     return this.value
   }
@@ -82,8 +59,8 @@ class WriteableCounter extends Counter {
    * Decreases the value of the counter by `delta`. If `delta` is not given,
    * decreases the value of the counter by 1.
    */
-  decrement(delta: number): number {
-    return this.increment(typeof delta === "number" ? -delta : -1)
+  decrement(delta) {
+    return this.inc(typeof delta === 'number' ? -delta : -1)
   }
 }
 
@@ -93,15 +70,15 @@ class WriteableCounter extends Counter {
  * `objectId` is the ID of the object containing the counter, and `key` is
  * the property name (key in map, or index in list) where the counter is
  * located.
- */
-export function getWriteableCounter(
-  value: number,
-  context: Automerge,
-  path: Prop[],
-  objectId: ObjID,
-  key: Prop
-): WriteableCounter {
-  return new WriteableCounter(value, context, path, objectId, key)
+*/
+function getWriteableCounter(value, context, path, objectId, key) {
+  const instance = Object.create(WriteableCounter.prototype)
+  instance.value = value
+  instance.context = context
+  instance.path = path
+  instance.objectId = objectId
+  instance.key = key
+  return instance
 }
 
-//module.exports = { Counter, getWriteableCounter }
+module.exports = { Counter, getWriteableCounter }

automerge-js/src/index.js

@@ -0,0 +1,372 @@
+const AutomergeWASM = require("automerge-wasm")
+const uuid = require('./uuid')
+
+let { rootProxy, listProxy, textProxy, mapProxy } = require("./proxies")
+let { Counter  } = require("./counter")
+let { Text } = require("./text")
+let { Int, Uint, Float64  } = require("./numbers")
+let { STATE, HEADS, OBJECT_ID, READ_ONLY, FROZEN  } = require("./constants")
+
+function init(actor) {
+  if (typeof actor != 'string') {
+    actor = null
+  }
+  const state = AutomergeWASM.create(actor)
+  return rootProxy(state, true);
+}
+
+function clone(doc) {
+  const state = doc[STATE].clone()
+  return rootProxy(state, true);
+}
+
+function free(doc) {
+  return doc[STATE].free()
+}
+
+function from(data, actor) {
+    let doc1 = init(actor)
+    let doc2 = change(doc1, (d) => Object.assign(d, data))
+    return doc2
+}
+
+function change(doc, options, callback) {
+  if (callback === undefined) {
+    // FIXME implement options
+    callback = options
+    options = {}
+  }
+  if (typeof options === "string") {
+    options = { message: options }
+  }
+  if (doc === undefined || doc[STATE] === undefined || doc[OBJECT_ID] !== "_root") {
+    throw new RangeError("must be the document root");
+  }
+  if (doc[FROZEN] === true) {
+    throw new RangeError("Attempting to use an outdated Automerge document")
+  }
+  if (!!doc[HEADS] === true) {
+    throw new RangeError("Attempting to change an out of date document");
+  }
+  if (doc[READ_ONLY] === false) {
+    throw new RangeError("Calls to Automerge.change cannot be nested")
+  }
+  const state = doc[STATE]
+  const heads = state.getHeads()
+  try {
+    doc[HEADS] = heads
+    doc[FROZEN] = true
+    let root = rootProxy(state);
+    callback(root)
+    if (state.pendingOps() === 0) {
+      doc[FROZEN] = false
+      doc[HEADS] = undefined
+      return doc
+    } else {
+      state.commit(options.message, options.time)
+      return rootProxy(state, true);
+    }
+  } catch (e) {
+    //console.log("ERROR: ",e)
+    doc[FROZEN] = false
+    doc[HEADS] = undefined
+    state.rollback()
+    throw e
+  }
+}
+
+function emptyChange(doc, options) {
+  if (options === undefined) {
+    options = {}
+  }
+  if (typeof options === "string") {
+    options = { message: options }
+  }
+
+  if (doc === undefined || doc[STATE] === undefined || doc[OBJECT_ID] !== "_root") {
+    throw new RangeError("must be the document root");
+  }
+  if (doc[FROZEN] === true) {
+    throw new RangeError("Attempting to use an outdated Automerge document")
+  }
+  if (doc[READ_ONLY] === false) {
+    throw new RangeError("Calls to Automerge.change cannot be nested")
+  }
+
+  const state = doc[STATE]
+  state.commit(options.message, options.time)
+  return rootProxy(state, true);
+}
+
+function load(data, actor) {
+  const state = AutomergeWASM.loadDoc(data, actor)
+  return rootProxy(state, true);
+}
+
+function save(doc) {
+  const state = doc[STATE]
+  return state.save()
+}
+
+function merge(local, remote) {
+  if (local[HEADS] === true) {
+    throw new RangeError("Attempting to change an out of date document");
+  }
+  const localState = local[STATE]
+  const heads = localState.getHeads()
+  const remoteState = remote[STATE]
+  const changes = localState.getChangesAdded(remoteState)
+  localState.applyChanges(changes)
+  local[HEADS] = heads
+  return rootProxy(localState, true)
+}
+
+function getActorId(doc) {
+  const state = doc[STATE]
+  return state.getActorId()
+}
+
+function conflictAt(context, objectId, prop) {
+      let values = context.values(objectId, prop)
+      if (values.length <= 1) {
+        return
+      }
+      let result = {}
+      for (const conflict of values) {
+        const datatype = conflict[0]
+        const value = conflict[1]
+        switch (datatype) {
+          case "map":
+            result[value] = mapProxy(context, value, [ prop ], true)
+            break;
+          case "list":
+            result[value] = listProxy(context, value, [ prop ], true)
+            break;
+          case "text":
+            result[value] = textProxy(context, value, [ prop ], true)
+            break;
+          //case "table":
+          //case "cursor":
+          case "str":
+          case "uint":
+          case "int":
+          case "f64":
+          case "boolean":
+          case "bytes":
+          case "null":
+            result[conflict[2]] = value
+            break;
+          case "counter":
+            result[conflict[2]] = new Counter(value)
+            break;
+          case "timestamp":
+            result[conflict[2]] = new Date(value)
+            break;
+          default:
+            throw RangeError(`datatype ${datatype} unimplemented`)
+        }
+      }
+      return result
+}
+
+function getConflicts(doc, prop) {
+  const state = doc[STATE]
+  const objectId = doc[OBJECT_ID]
+  return conflictAt(state, objectId, prop)
+}
+
+function getLastLocalChange(doc) {
+  const state = doc[STATE]
+  try {
+    return state.getLastLocalChange()
+  } catch (e) {
+    return
+  }
+}
+
+function getObjectId(doc) {
+  return doc[OBJECT_ID]
+}
+
+function getChanges(oldState, newState) {
+  const o = oldState[STATE]
+  const n = newState[STATE]
+  const heads = oldState[HEADS]
+  return n.getChanges(heads || o.getHeads())
+}
+
+function getAllChanges(doc) {
+  const state = doc[STATE]
+  return state.getChanges([])
+}
+
+function applyChanges(doc, changes) {
+  if (doc === undefined || doc[STATE] === undefined || doc[OBJECT_ID] !== "_root") {
+    throw new RangeError("must be the document root");
+  }
+  if (doc[FROZEN] === true) {
+    throw new RangeError("Attempting to use an outdated Automerge document")
+  }
+  if (doc[READ_ONLY] === false) {
+    throw new RangeError("Calls to Automerge.change cannot be nested")
+  }
+  const state = doc[STATE]
+  const heads = state.getHeads()
+  state.applyChanges(changes)
+  doc[HEADS] = heads
+  return [rootProxy(state, true)];
+}
+
+function getHistory(doc) {
+  const actor = getActorId(doc)
+  const history = getAllChanges(doc)
+  return history.map((change, index) => ({
+      get change () {
+        return decodeChange(change)
+      },
+      get snapshot () {
+        const [state] = applyChanges(init(), history.slice(0, index + 1))
+        return state
+      }
+    })
+  )
+}
+
+function equals() {
+  if (!isObject(val1) || !isObject(val2)) return val1 === val2
+  const keys1 = Object.keys(val1).sort(), keys2 = Object.keys(val2).sort()
+  if (keys1.length !== keys2.length) return false
+  for (let i = 0; i < keys1.length; i++) {
+    if (keys1[i] !== keys2[i]) return false
+    if (!equals(val1[keys1[i]], val2[keys2[i]])) return false
+  }
+  return true
+}
+
+function encodeSyncMessage(msg) {
+  return AutomergeWASM.encodeSyncMessage(msg)
+}
+
+function decodeSyncMessage(msg) {
+  return AutomergeWASM.decodeSyncMessage(msg)
+}
+
+function encodeSyncState(state) {
+  return AutomergeWASM.encodeSyncState(AutomergeWASM.importSyncState(state))
+}
+
+function decodeSyncState(state) {
+  return AutomergeWASM.exportSyncState(AutomergeWASM.decodeSyncState(state))
+}
+
+function generateSyncMessage(doc, inState) {
+  const state = doc[STATE]
+  const syncState = AutomergeWASM.importSyncState(inState)
+  const message = state.generateSyncMessage(syncState)
+  const outState = AutomergeWASM.exportSyncState(syncState)
+  return [ outState, message ]
+}
+
+function receiveSyncMessage(doc, inState, message) {
+  const syncState = AutomergeWASM.importSyncState(inState)
+  if (doc === undefined || doc[STATE] === undefined || doc[OBJECT_ID] !== "_root") {
+    throw new RangeError("must be the document root");
+  }
+  if (doc[FROZEN] === true) {
+    throw new RangeError("Attempting to use an outdated Automerge document")
+  }
+  if (!!doc[HEADS] === true) {
+    throw new RangeError("Attempting to change an out of date document");
+  }
+  if (doc[READ_ONLY] === false) {
+    throw new RangeError("Calls to Automerge.change cannot be nested")
+  }
+  const state = doc[STATE]
+  const heads = state.getHeads()
+  state.receiveSyncMessage(syncState, message)
+  const outState = AutomergeWASM.exportSyncState(syncState)
+  doc[HEADS] = heads
+  return [rootProxy(state, true), outState, null];
+}
+
+function initSyncState() {
+  return AutomergeWASM.exportSyncState(AutomergeWASM.initSyncState(change))
+}
+
+function encodeChange(change) {
+  return AutomergeWASM.encodeChange(change)
+}
+
+function decodeChange(data) {
+  return AutomergeWASM.decodeChange(data)
+}
+
+function encodeSyncMessage(change) {
+  return AutomergeWASM.encodeSyncMessage(change)
+}
+
+function decodeSyncMessage(data) {
+  return AutomergeWASM.decodeSyncMessage(data)
+}
+
+function getMissingDeps(doc, heads) {
+  const state = doc[STATE]
+  return state.getMissingDeps(heads)
+}
+
+function getHeads(doc) {
+  const state = doc[STATE]
+  return doc[HEADS] || state.getHeads()
+}
+
+function dump(doc) {
+  const state = doc[STATE]
+  state.dump()
+}
+
+function toJS(doc) {
+  if (typeof doc === "object") {
+    if (doc instanceof Uint8Array) {
+      return doc
+    }
+    if (doc === null) {
+      return doc
+    }
+    if (doc instanceof Array) {
+      return doc.map((a) => toJS(a))
+    }
+    if (doc instanceof Text) {
+      return doc.map((a) => toJS(a))
+    }
+    let tmp = {}
+    for (index in doc) {
+      tmp[index] = toJS(doc[index])
+    }
+    return tmp
+  } else {
+    return doc
+  }
+}
+
+module.exports = {
+    init, from, change, emptyChange, clone, free,
+    load, save, merge, getChanges, getAllChanges, applyChanges,
+    getLastLocalChange, getObjectId, getActorId, getConflicts,
+    encodeChange, decodeChange, equals, getHistory, getHeads, uuid,
+    generateSyncMessage, receiveSyncMessage, initSyncState,
+    decodeSyncMessage, encodeSyncMessage, decodeSyncState, encodeSyncState,
+    getMissingDeps,
+    dump, Text, Counter, Int, Uint, Float64, toJS,
+}
+
+// depricated
+// Frontend, setDefaultBackend, Backend
+
+// more...
+/*
+for (let name of ['getObjectId', 'getObjectById',
+       'setActorId',
+       'Text', 'Table', 'Counter', 'Observable' ]) {
+    module.exports[name] = Frontend[name]
+}
+*/

automerge-js/src/numbers.js

@@ -0,0 +1,33 @@
+// Convience classes to allow users to stricly specify the number type they want
+
+class Int {
+  constructor(value) {
+    if (!(Number.isInteger(value) && value <= Number.MAX_SAFE_INTEGER && value >= Number.MIN_SAFE_INTEGER)) {
+      throw new RangeError(`Value ${value} cannot be a uint`)
+    }
+    this.value = value
+    Object.freeze(this)
+  }
+}
+
+class Uint {
+  constructor(value) {
+    if (!(Number.isInteger(value) && value <= Number.MAX_SAFE_INTEGER && value >= 0)) {
+      throw new RangeError(`Value ${value} cannot be a uint`)
+    }
+    this.value = value
+    Object.freeze(this)
+  }
+}
+
+class Float64 {
+  constructor(value) {
+    if (typeof value !== 'number') {
+      throw new RangeError(`Value ${value} cannot be a float64`)
+    }
+    this.value = value || 0.0
+    Object.freeze(this)
+  }
+}
+
+module.exports = { Int, Uint, Float64 }

automerge-js/src/proxies.js

@@ -0,0 +1,623 @@
+
+const AutomergeWASM = require("automerge-wasm")
+const { Int, Uint, Float64 } = require("./numbers");
+const { Counter, getWriteableCounter } = require("./counter");
+const { Text } = require("./text");
+const { STATE, HEADS, FROZEN, OBJECT_ID, READ_ONLY } = require("./constants")
+
+function parseListIndex(key) {
+  if (typeof key === 'string' && /^[0-9]+$/.test(key)) key = parseInt(key, 10)
+  if (typeof key !== 'number') {
+    // throw new TypeError('A list index must be a number, but you passed ' + JSON.stringify(key))
+    return key
+  }
+  if (key < 0 || isNaN(key) || key === Infinity || key === -Infinity) {
+    throw new RangeError('A list index must be positive, but you passed ' + key)
+  }
+  return key
+}
+
+function valueAt(target, prop) {
+      const { context, objectId, path, readonly, heads} = target
+      let value = context.value(objectId, prop, heads)
+      if (value === undefined) {
+        return
+      }
+      const datatype = value[0]
+      const val = value[1]
+      switch (datatype) {
+        case undefined: return;
+        case "map": return mapProxy(context, val, [ ... path, prop ], readonly, heads);
+        case "list": return listProxy(context, val, [ ... path, prop ], readonly, heads);
+        case "text": return textProxy(context, val, [ ... path, prop ], readonly, heads);
+        //case "table":
+        //case "cursor":
+        case "str": return val;
+        case "uint": return val;
+        case "int": return val;
+        case "f64": return val;
+        case "boolean": return val;
+        case "null": return null;
+        case "bytes": return val;
+        case "timestamp": return val;
+        case "counter": {
+          if (readonly) {
+            return new Counter(val);
+          } else {
+            return getWriteableCounter(val, context, path, objectId, prop)
+          }
+        }
+        default:
+          throw RangeError(`datatype ${datatype} unimplemented`)
+      }
+}
+
+function import_value(value) {
+    switch (typeof value) {
+      case 'object':
+        if (value == null) {
+          return [ null, "null"]
+        } else if (value instanceof Uint) {
+          return [ value.value, "uint" ]
+        } else if (value instanceof Int) {
+          return [ value.value, "int" ]
+        } else if (value instanceof Float64) {
+          return [ value.value, "f64" ]
+        } else if (value instanceof Counter) {
+          return [ value.value, "counter" ]
+        } else if (value instanceof Date) {
+          return [ value.getTime(), "timestamp" ]
+        } else if (value instanceof Uint8Array) {
+          return [ value, "bytes" ]
+        } else if (value instanceof Array) {
+          return [ value, "list" ]
+        } else if (value instanceof Text) {
+          return [ value, "text" ]
+        } else if (value[OBJECT_ID]) {
+          throw new RangeError('Cannot create a reference to an existing document object')
+        } else {
+          return [ value, "map" ]
+        }
+        break;
+      case 'boolean':
+        return [ value, "boolean" ]
+      case 'number':
+        if (Number.isInteger(value)) {
+          return [ value, "int" ]
+        } else {
+          return [ value, "f64" ]
+        }
+        break;
+      case 'string':
+        return [ value ]
+        break;
+      default:
+        throw new RangeError(`Unsupported type of value: ${typeof value}`)
+    }
+}
+
+const MapHandler = {
+  get (target, key) {
+    const { context, objectId, path, readonly, frozen, heads, cache } = target
+    if (key === Symbol.toStringTag) { return target[Symbol.toStringTag] }
+    if (key === OBJECT_ID) return objectId
+    if (key === READ_ONLY) return readonly
+    if (key === FROZEN) return frozen
+    if (key === HEADS) return heads
+    if (key === STATE) return context;
+    if (!cache[key]) {
+      cache[key] = valueAt(target, key)
+    }
+    return cache[key]
+  },
+
+  set (target, key, val) {
+    let { context, objectId, path, readonly, frozen} = target
+    target.cache = {} // reset cache on set
+    if (val && val[OBJECT_ID]) {
+          throw new RangeError('Cannot create a reference to an existing document object')
+    }
+    if (key === FROZEN) {
+      target.frozen = val
+      return
+    }
+    if (key === HEADS) {
+      target.heads = val
+      return
+    }
+    let [ value, datatype ] = import_value(val)
+    if (frozen) {
+      throw new RangeError("Attempting to use an outdated Automerge document")
+    }
+    if (readonly) {
+      throw new RangeError(`Object property "${key}" cannot be modified`)
+    }
+    switch (datatype) {
+      case "list":
+        const list = context.set_object(objectId, key, [])
+        const proxyList = listProxy(context, list, [ ... path, key ], readonly );
+        for (let i = 0; i < value.length; i++) {
+          proxyList[i] = value[i]
+        }
+        break;
+      case "text":
+        const text = context.set_object(objectId, key, "", "text")
+        const proxyText = textProxy(context, text, [ ... path, key ], readonly );
+        for (let i = 0; i < value.length; i++) {
+          proxyText[i] = value.get(i)
+        }
+        break;
+      case "map":
+        const map = context.set_object(objectId, key, {})
+        const proxyMap = mapProxy(context, map, [ ... path, key ], readonly );
+        for (const key in value) {
+          proxyMap[key] = value[key]
+        }
+        break;
+      default:
+        context.set(objectId, key, value, datatype)
+    }
+    return true
+  },
+
+  deleteProperty (target, key) {
+    const { context, objectId, path, readonly, frozen } = target
+    target.cache = {} // reset cache on delete
+    if (readonly) {
+      throw new RangeError(`Object property "${key}" cannot be modified`)
+    }
+    context.del(objectId, key)
+    return true
+  },
+
+  has (target, key) {
+    const value = this.get(target, key)
+    return value !== undefined
+  },
+
+  getOwnPropertyDescriptor (target, key) {
+    const { context, objectId } = target
+    const value = this.get(target, key)
+    if (typeof value !== 'undefined') {
+      return {
+        configurable: true, enumerable: true, value
+      }
+    }
+  },
+
+  ownKeys (target) {
+    const { context, objectId, heads} = target
+    return context.keys(objectId, heads)
+  },
+}
+
+
+const ListHandler = {
+  get (target, index) {
+    const {context, objectId, path, readonly, frozen, heads } = target
+    index = parseListIndex(index)
+    if (index === Symbol.hasInstance) { return (instance) => { return [].has(instance) } }
+    if (index === Symbol.toStringTag) { return target[Symbol.toStringTag] }
+    if (index === OBJECT_ID) return objectId
+    if (index === READ_ONLY) return readonly
+    if (index === FROZEN) return frozen
+    if (index === HEADS) return heads
+    if (index === STATE) return context;
+    if (index === 'length') return context.length(objectId, heads);
+    if (index === Symbol.iterator) {
+      let i = 0;
+      return function *() {
+        // FIXME - ugly
+        let value = valueAt(target, i)
+        while (value !== undefined) {
+            yield value
+            i += 1
+            value = valueAt(target, i)
+        }
+      }
+    }
+    if (typeof index === 'number') {
+      return valueAt(target, index)
+    } else {
+      return listMethods(target)[index]
+    }
+  },
+
+  set (target, index, val) {
+    let {context, objectId, path, readonly, frozen } = target
+    index = parseListIndex(index)
+    if (val && val[OBJECT_ID]) {
+      throw new RangeError('Cannot create a reference to an existing document object')
+    }
+    if (index === FROZEN) {
+      target.frozen = val
+      return
+    }
+    if (index === HEADS) {
+      target.heads = val
+      return
+    }
+    if (typeof index == "string") {
+      throw new RangeError('list index must be a number')
+    }
+    const [ value, datatype] = import_value(val)
+    if (frozen) {
+      throw new RangeError("Attempting to use an outdated Automerge document")
+    }
+    if (readonly) {
+      throw new RangeError(`Object property "${index}" cannot be modified`)
+    }
+    switch (datatype) {
+      case "list":
+        let list
+        if (index >= context.length(objectId)) {
+          list = context.insert_object(objectId, index, [])
+        } else {
+          list = context.set_object(objectId, index, [])
+        }
+        const proxyList = listProxy(context, list, [ ... path, index ], readonly);
+        proxyList.splice(0,0,...value)
+        break;
+      case "text":
+        let text
+        if (index >= context.length(objectId)) {
+          text = context.insert_object(objectId, index, "", "text")
+        } else {
+          text = context.set_object(objectId, index, "", "text")
+        }
+        const proxyText = textProxy(context, text, [ ... path, index ], readonly);
+        proxyText.splice(0,0,...value)
+        break;
+      case "map":
+        let map
+        if (index >= context.length(objectId)) {
+          map = context.insert_object(objectId, index, {})
+        } else {
+          map = context.set_object(objectId, index, {})
+        }
+        const proxyMap = mapProxy(context, map, [ ... path, index ], readonly);
+        for (const key in value) {
+          proxyMap[key] = value[key]
+        }
+        break;
+      default:
+        if (index >= context.length(objectId)) {
+          context.insert(objectId, index, value, datatype)
+        } else {
+          context.set(objectId, index, value, datatype)
+        }
+    }
+    return true
+  },
+
+  deleteProperty (target, index) {
+    const {context, objectId} = target
+    index = parseListIndex(index)
+    if (context.value(objectId, index)[0] == "counter") {
+      throw new TypeError('Unsupported operation: deleting a counter from a list')
+    }
+    context.del(objectId, index)
+    return true
+  },
+
+  has (target, index) {
+    const {context, objectId, heads} = target
+    index = parseListIndex(index)
+    if (typeof index === 'number') {
+      return index < context.length(objectId, heads)
+    }
+    return index === 'length'
+  },
+
+  getOwnPropertyDescriptor (target, index) {
+    const {context, objectId, path, readonly, frozen, heads} = target
+
+    if (index === 'length') return {writable: true, value: context.length(objectId, heads) }
+    if (index === OBJECT_ID) return {configurable: false, enumerable: false, value: objectId}
+
+    index = parseListIndex(index)
+
+    let value = valueAt(target, index)
+    return { configurable: true, enumerable: true, value }
+  },
+
+  getPrototypeOf(target) { return Object.getPrototypeOf([]) },
+  ownKeys (target) {
+    const {context, objectId, heads } = target
+    let keys = []
+    // uncommenting this causes assert.deepEqual() to fail when comparing to a pojo array
+    // but not uncommenting it causes for (i in list) {} to not enumerate values properly
+    //for (let i = 0; i < target.context.length(objectId, heads); i++) { keys.push(i.toString()) }
+    keys.push("length");
+    return keys
+  }
+}
+
+const TextHandler = Object.assign({}, ListHandler, {
+  get (target, index) {
+    // FIXME this is a one line change from ListHandler.get()
+    const {context, objectId, path, readonly, frozen, heads } = target
+    index = parseListIndex(index)
+    if (index === Symbol.toStringTag) { return target[Symbol.toStringTag] }
+    if (index === Symbol.hasInstance) { return (instance) => { return [].has(instance) } }
+    if (index === OBJECT_ID) return objectId
+    if (index === READ_ONLY) return readonly
+    if (index === FROZEN) return frozen
+    if (index === HEADS) return heads
+    if (index === STATE) return context;
+    if (index === 'length') return context.length(objectId, heads);
+    if (index === Symbol.iterator) {
+      let i = 0;
+      return function *() {
+        let value = valueAt(target, i)
+        while (value !== undefined) {
+            yield value
+            i += 1
+            value = valueAt(target, i)
+        }
+      }
+    }
+    if (typeof index === 'number') {
+      return valueAt(target, index)
+    } else {
+      return textMethods(target)[index] || listMethods(target)[index]
+    }
+  },
+  getPrototypeOf(target) {
+    return Object.getPrototypeOf(new Text())
+  },
+})
+
+function mapProxy(context, objectId, path, readonly, heads) {
+  return new Proxy({context, objectId, path, readonly: !!readonly, frozen: false, heads, cache: {}}, MapHandler)
+}
+
+function listProxy(context, objectId, path, readonly, heads) {
+  let target = []
+  Object.assign(target, {context, objectId, path, readonly: !!readonly, frozen: false, heads, cache: {}})
+  return new Proxy(target, ListHandler)
+}
+
+function textProxy(context, objectId, path, readonly, heads) {
+  let target = []
+  Object.assign(target, {context, objectId, path, readonly: !!readonly, frozen: false, heads, cache: {}})
+  return new Proxy(target, TextHandler)
+}
+
+function rootProxy(context, readonly) {
+  return mapProxy(context, "_root", [], readonly)
+}
+
+function listMethods(target) {
+  const {context, objectId, path, readonly, frozen, heads} = target
+  const methods = {
+    deleteAt(index, numDelete) {
+      if (typeof numDelete === 'number') {
+        context.splice(objectId, index, numDelete)
+      } else {
+        context.del(objectId, index)
+      }
+      return this
+    },
+
+    fill(val, start, end) {
+      // FIXME
+      let list = context.getObject(objectId)
+      let [value, datatype] = valueAt(target, index)
+      for (let index = parseListIndex(start || 0); index < parseListIndex(end || list.length); index++) {
+        context.set(objectId, index, value, datatype)
+      }
+      return this
+    },
+
+    indexOf(o, start = 0) {
+      // FIXME
+      const id = o[OBJECT_ID]
+      if (id) {
+        const list = context.getObject(objectId)
+        for (let index = start; index < list.length; index++) {
+          if (list[index][OBJECT_ID] === id) {
+            return index
+          }
+        }
+        return -1
+      } else {
+        return context.indexOf(objectId, o, start)
+      }
+    },
+
+    insertAt(index, ...values) {
+      this.splice(index, 0, ...values)
+      return this
+    },
+
+    pop() {
+      let length = context.length(objectId)
+      if (length == 0) {
+        return undefined
+      }
+      let last = valueAt(target, length - 1)
+      context.del(objectId, length - 1)
+      return last
+    },
+
+    push(...values) {
+      let len = context.length(objectId)
+      this.splice(len, 0, ...values)
+      return context.length(objectId)
+    },
+
+    shift() {
+      if (context.length(objectId) == 0) return
+      const first = valueAt(target, 0)
+      context.del(objectId, 0)
+      return first
+    },
+
+    splice(index, del, ...vals) {
+      index = parseListIndex(index)
+      del = parseListIndex(del)
+      for (let val of vals) {
+        if (val && val[OBJECT_ID]) {
+              throw new RangeError('Cannot create a reference to an existing document object')
+        }
+      }
+      if (frozen) {
+        throw new RangeError("Attempting to use an outdated Automerge document")
+      }
+      if (readonly) {
+        throw new RangeError("Sequence object cannot be modified outside of a change block")
+      }
+      let result = []
+      for (let i = 0; i < del; i++) {
+        let value = valueAt(target, index)
+        result.push(value)
+        context.del(objectId, index)
+      }
+      const values = vals.map((val) => import_value(val))
+      for (let [value,datatype] of values) {
+        switch (datatype) {
+          case "list":
+            const list = context.insert_object(objectId, index, [])
+            const proxyList = listProxy(context, list, [ ... path, index ], readonly);
+            proxyList.splice(0,0,...value)
+            break;
+          case "text":
+            const text = context.insert_object(objectId, index, "", "text")
+            const proxyText = textProxy(context, text, [ ... path, index ], readonly);
+            proxyText.splice(0,0,...value)
+            break;
+          case "map":
+            const map = context.insert_object(objectId, index, {})
+            const proxyMap = mapProxy(context, map, [ ... path, index ], readonly);
+            for (const key in value) {
+              proxyMap[key] = value[key]
+            }
+            break;
+          default:
+            context.insert(objectId, index, value, datatype)
+        }
+        index += 1
+      }
+      return result
+    },
+
+    unshift(...values) {
+      this.splice(0, 0, ...values)
+      return context.length(objectId)
+    },
+
+    entries() {
+      let i = 0;
+      const iterator = {
+        next: () => {
+          let value = valueAt(target, i)
+          if (value === undefined) {
+            return { value: undefined, done: true }
+          } else {
+            return { value: [ i, value ], done: false }
+          }
+        }
+      }
+      return iterator
+    },
+
+    keys() {
+      let i = 0;
+      let len = context.length(objectId, heads)
+      const iterator = {
+        next: () => {
+          let value = undefined
+          if (i < len) { value = i; i++ }
+          return { value, done: true }
+        }
+      }
+      return iterator
+    },
+
+    values() {
+      let i = 0;
+      const iterator = {
+        next: () => {
+          let value = valueAt(target, i)
+          if (value === undefined) {
+            return { value: undefined, done: true }
+          } else {
+            return { value, done: false }
+          }
+        }
+      }
+      return iterator
+    }
+  }
+
+  // Read-only methods that can delegate to the JavaScript built-in implementations
+  // FIXME - super slow
+  for (let method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes',
+                      'join', 'lastIndexOf', 'map', 'reduce', 'reduceRight',
+                      'slice', 'some', 'toLocaleString', 'toString']) {
+    methods[method] = (...args) => {
+      const list = []
+      while (true) {
+        let value =  valueAt(target, list.length)
+        if (value == undefined) {
+          break
+        }
+        list.push(value)
+      }
+
+      return list[method](...args)
+    }
+  }
+
+  return methods
+}
+
+function textMethods(target) {
+  const {context, objectId, path, readonly, frozen} = target
+  const methods = {
+    set (index, value) {
+      return this[index] = value
+    },
+    get (index) {
+      return this[index]
+    },
+    toString () {
+      let str = ''
+      let length = this.length
+      for (let i = 0; i < length; i++) {
+        const value = this.get(i)
+        if (typeof value === 'string') str += value
+      }
+      return str
+    },
+    toSpans () {
+      let spans = []
+      let chars = ''
+      let length = this.length
+      for (let i = 0; i < length; i++) {
+        const value = this[i]
+        if (typeof value === 'string') {
+          chars += value
+        } else {
+          if (chars.length > 0) {
+            spans.push(chars)
+            chars = ''
+          }
+          spans.push(value)
+        }
+      }
+      if (chars.length > 0) {
+        spans.push(chars)
+      }
+      return spans
+    },
+    toJSON () {
+      return this.toString()
+    }
+  }
+  return methods
+}
+
+
+module.exports = { rootProxy, textProxy, listProxy, mapProxy, MapHandler, ListHandler, TextHandler }

automerge-js/src/sync.js

@@ -16,15 +16,11 @@
  * last sync to disk), and we fall back to sending the entire document in this case.
  */
 
-const Backend = null //require('./backend')
-const {
-  hexStringToBytes,
-  bytesToHexString,
-  Encoder,
-  Decoder,
-} = require("./encoding")
-const { decodeChangeMeta } = require("./columnar")
-const { copyObject } = require("./common")
+//const Backend = require('./backend')
+const Backend = {} //require('./backend')
+const { hexStringToBytes, bytesToHexString, Encoder, Decoder } = require('./encoding')
+const { decodeChangeMeta } = require('./columnar')
+const { copyObject } = require('../src/common')
 
 const HASH_SIZE = 32 // 256 bits = 32 bytes
 const MESSAGE_TYPE_SYNC = 0x42 // first byte of a sync message, for identification
@@ -33,8 +29,7 @@ const PEER_STATE_TYPE = 0x43 // first byte of an encoded peer state, for identif
 // These constants correspond to a 1% false positive rate. The values can be changed without
 // breaking compatibility of the network protocol, since the parameters used for a particular
 // Bloom filter are encoded in the wire format.
-const BITS_PER_ENTRY = 10,
-  NUM_PROBES = 7
+const BITS_PER_ENTRY = 10, NUM_PROBES = 7
 
 /**
  * A Bloom filter implementation that can be serialised to a byte array for transmission
@@ -42,15 +37,13 @@ const BITS_PER_ENTRY = 10,
  * so this implementation does not perform its own hashing.
  */
 class BloomFilter {
-  constructor(arg) {
+  constructor (arg) {
     if (Array.isArray(arg)) {
       // arg is an array of SHA256 hashes in hexadecimal encoding
       this.numEntries = arg.length
       this.numBitsPerEntry = BITS_PER_ENTRY
       this.numProbes = NUM_PROBES
-      this.bits = new Uint8Array(
-        Math.ceil((this.numEntries * this.numBitsPerEntry) / 8)
-      )
+      this.bits = new Uint8Array(Math.ceil(this.numEntries * this.numBitsPerEntry / 8))
       for (let hash of arg) this.addHash(hash)
     } else if (arg instanceof Uint8Array) {
       if (arg.byteLength === 0) {
@@ -63,12 +56,10 @@ class BloomFilter {
         this.numEntries = decoder.readUint32()
         this.numBitsPerEntry = decoder.readUint32()
         this.numProbes = decoder.readUint32()
-        this.bits = decoder.readRawBytes(
-          Math.ceil((this.numEntries * this.numBitsPerEntry) / 8)
-        )
+        this.bits = decoder.readRawBytes(Math.ceil(this.numEntries * this.numBitsPerEntry / 8))
       }
     } else {
-      throw new TypeError("invalid argument")
+      throw new TypeError('invalid argument')
     }
   }
 
@@ -96,32 +87,12 @@ class BloomFilter {
    * http://www.ccis.northeastern.edu/home/pete/pub/bloom-filters-verification.pdf
    */
   getProbes(hash) {
-    const hashBytes = hexStringToBytes(hash),
-      modulo = 8 * this.bits.byteLength
-    if (hashBytes.byteLength !== 32)
-      throw new RangeError(`Not a 256-bit hash: ${hash}`)
+    const hashBytes = hexStringToBytes(hash), modulo = 8 * this.bits.byteLength
+    if (hashBytes.byteLength !== 32) throw new RangeError(`Not a 256-bit hash: ${hash}`)
     // on the next three lines, the right shift means interpret value as unsigned
-    let x =
-      ((hashBytes[0] |
-        (hashBytes[1] << 8) |
-        (hashBytes[2] << 16) |
-        (hashBytes[3] << 24)) >>>
-        0) %
-      modulo
-    let y =
-      ((hashBytes[4] |
-        (hashBytes[5] << 8) |
-        (hashBytes[6] << 16) |
-        (hashBytes[7] << 24)) >>>
-        0) %
-      modulo
-    let z =
-      ((hashBytes[8] |
-        (hashBytes[9] << 8) |
-        (hashBytes[10] << 16) |
-        (hashBytes[11] << 24)) >>>
-        0) %
-      modulo
+    let x = ((hashBytes[0] | hashBytes[1] << 8 | hashBytes[2]  << 16 | hashBytes[3]  << 24) >>> 0) % modulo
+    let y = ((hashBytes[4] | hashBytes[5] << 8 | hashBytes[6]  << 16 | hashBytes[7]  << 24) >>> 0) % modulo
+    let z = ((hashBytes[8] | hashBytes[9] << 8 | hashBytes[10] << 16 | hashBytes[11] << 24) >>> 0) % modulo
     const probes = [x]
     for (let i = 1; i < this.numProbes; i++) {
       x = (x + y) % modulo
@@ -158,14 +129,12 @@ class BloomFilter {
  * Encodes a sorted array of SHA-256 hashes (as hexadecimal strings) into a byte array.
  */
 function encodeHashes(encoder, hashes) {
-  if (!Array.isArray(hashes)) throw new TypeError("hashes must be an array")
+  if (!Array.isArray(hashes)) throw new TypeError('hashes must be an array')
   encoder.appendUint32(hashes.length)
   for (let i = 0; i < hashes.length; i++) {
-    if (i > 0 && hashes[i - 1] >= hashes[i])
-      throw new RangeError("hashes must be sorted")
+    if (i > 0 && hashes[i - 1] >= hashes[i]) throw new RangeError('hashes must be sorted')
     const bytes = hexStringToBytes(hashes[i])
-    if (bytes.byteLength !== HASH_SIZE)
-      throw new TypeError("heads hashes must be 256 bits")
+    if (bytes.byteLength !== HASH_SIZE) throw new TypeError('heads hashes must be 256 bits')
     encoder.appendRawBytes(bytes)
   }
 }
@@ -175,8 +144,7 @@ function encodeHashes(encoder, hashes) {
  * array of hex strings.
  */
 function decodeHashes(decoder) {
-  let length = decoder.readUint32(),
-    hashes = []
+  let length = decoder.readUint32(), hashes = []
   for (let i = 0; i < length; i++) {
     hashes.push(bytesToHexString(decoder.readRawBytes(HASH_SIZE)))
   }
@@ -216,11 +184,11 @@ function decodeSyncMessage(bytes) {
   const heads = decodeHashes(decoder)
   const need = decodeHashes(decoder)
   const haveCount = decoder.readUint32()
-  let message = { heads, need, have: [], changes: [] }
+  let message = {heads, need, have: [], changes: []}
   for (let i = 0; i < haveCount; i++) {
     const lastSync = decodeHashes(decoder)
     const bloom = decoder.readPrefixedBytes(decoder)
-    message.have.push({ lastSync, bloom })
+    message.have.push({lastSync, bloom})
   }
   const changeCount = decoder.readUint32()
   for (let i = 0; i < changeCount; i++) {
@@ -267,7 +235,7 @@ function decodeSyncState(bytes) {
 function makeBloomFilter(backend, lastSync) {
   const newChanges = Backend.getChanges(backend, lastSync)
   const hashes = newChanges.map(change => decodeChangeMeta(change, true).hash)
-  return { lastSync, bloom: new BloomFilter(hashes).bytes }
+  return {lastSync, bloom: new BloomFilter(hashes).bytes}
 }
 
 /**
@@ -278,26 +246,20 @@ function makeBloomFilter(backend, lastSync) {
  */
 function getChangesToSend(backend, have, need) {
   if (have.length === 0) {
-    return need
-      .map(hash => Backend.getChangeByHash(backend, hash))
-      .filter(change => change !== undefined)
+    return need.map(hash => Backend.getChangeByHash(backend, hash)).filter(change => change !== undefined)
   }
 
-  let lastSyncHashes = {},
-    bloomFilters = []
+  let lastSyncHashes = {}, bloomFilters = []
   for (let h of have) {
     for (let hash of h.lastSync) lastSyncHashes[hash] = true
     bloomFilters.push(new BloomFilter(h.bloom))
   }
 
   // Get all changes that were added since the last sync
-  const changes = Backend.getChanges(backend, Object.keys(lastSyncHashes)).map(
-    change => decodeChangeMeta(change, true)
-  )
+  const changes = Backend.getChanges(backend, Object.keys(lastSyncHashes))
+    .map(change => decodeChangeMeta(change, true))
 
-  let changeHashes = {},
-    dependents = {},
-    hashesToSend = {}
+  let changeHashes = {}, dependents = {}, hashesToSend = {}
   for (let change of changes) {
     changeHashes[change.hash] = true
 
@@ -331,8 +293,7 @@ function getChangesToSend(backend, have, need) {
   let changesToSend = []
   for (let hash of need) {
     hashesToSend[hash] = true
-    if (!changeHashes[hash]) {
-      // Change is not among those returned by getMissingChanges()?
+    if (!changeHashes[hash]) { // Change is not among those returned by getMissingChanges()?
       const change = Backend.getChangeByHash(backend, hash)
       if (change) changesToSend.push(change)
     }
@@ -357,7 +318,7 @@ function initSyncState() {
 }
 
 function compareArrays(a, b) {
-  return a.length === b.length && a.every((v, i) => v === b[i])
+    return (a.length === b.length) && a.every((v, i) => v === b[i])
 }
 
 /**
@@ -369,19 +330,10 @@ function generateSyncMessage(backend, syncState) {
     throw new Error("generateSyncMessage called with no Automerge document")
   }
   if (!syncState) {
-    throw new Error(
-      "generateSyncMessage requires a syncState, which can be created with initSyncState()"
-    )
+    throw new Error("generateSyncMessage requires a syncState, which can be created with initSyncState()")
   }
 
-  let {
-    sharedHeads,
-    lastSentHeads,
-    theirHeads,
-    theirNeed,
-    theirHave,
-    sentHashes,
-  } = syncState
+  let { sharedHeads, lastSentHeads, theirHeads, theirNeed, theirHave, sentHashes } = syncState
   const ourHeads = Backend.getHeads(backend)
 
   // Hashes to explicitly request from the remote peer: any missing dependencies of unapplied
@@ -405,28 +357,18 @@ function generateSyncMessage(backend, syncState) {
     const lastSync = theirHave[0].lastSync
     if (!lastSync.every(hash => Backend.getChangeByHash(backend, hash))) {
       // we need to queue them to send us a fresh sync message, the one they sent is uninteligible so we don't know what they need
-      const resetMsg = {
-        heads: ourHeads,
-        need: [],
-        have: [{ lastSync: [], bloom: new Uint8Array(0) }],
-        changes: [],
-      }
+      const resetMsg = {heads: ourHeads, need: [], have: [{ lastSync: [], bloom: new Uint8Array(0) }], changes: []}
       return [syncState, encodeSyncMessage(resetMsg)]
     }
   }
 
   // XXX: we should limit ourselves to only sending a subset of all the messages, probably limited by a total message size
   //      these changes should ideally be RLE encoded but we haven't implemented that yet.
-  let changesToSend =
-    Array.isArray(theirHave) && Array.isArray(theirNeed)
-      ? getChangesToSend(backend, theirHave, theirNeed)
-      : []
+  let changesToSend = Array.isArray(theirHave) && Array.isArray(theirNeed) ? getChangesToSend(backend, theirHave, theirNeed) : []
 
   // If the heads are equal, we're in sync and don't need to do anything further
-  const headsUnchanged =
-    Array.isArray(lastSentHeads) && compareArrays(ourHeads, lastSentHeads)
-  const headsEqual =
-    Array.isArray(theirHeads) && compareArrays(ourHeads, theirHeads)
+  const headsUnchanged = Array.isArray(lastSentHeads) && compareArrays(ourHeads, lastSentHeads)
+  const headsEqual = Array.isArray(theirHeads) && compareArrays(ourHeads, theirHeads)
   if (headsUnchanged && headsEqual && changesToSend.length === 0) {
     // no need to send a sync message if we know we're synced!
     return [syncState, null]
@@ -434,19 +376,12 @@ function generateSyncMessage(backend, syncState) {
 
   // TODO: this recomputes the SHA-256 hash of each change; we should restructure this to avoid the
   // unnecessary recomputation
-  changesToSend = changesToSend.filter(
-    change => !sentHashes[decodeChangeMeta(change, true).hash]
-  )
+  changesToSend = changesToSend.filter(change => !sentHashes[decodeChangeMeta(change, true).hash])
 
   // Regular response to a sync message: send any changes that the other node
   // doesn't have. We leave the "have" field empty because the previous message
   // generated by `syncStart` already indicated what changes we have.
-  const syncMessage = {
-    heads: ourHeads,
-    have: ourHave,
-    need: ourNeed,
-    changes: changesToSend,
-  }
+  const syncMessage = {heads: ourHeads, have: ourHave, need: ourNeed, changes: changesToSend}
   if (changesToSend.length > 0) {
     sentHashes = copyObject(sentHashes)
     for (const change of changesToSend) {
@@ -454,10 +389,7 @@ function generateSyncMessage(backend, syncState) {
     }
   }
 
-  syncState = Object.assign({}, syncState, {
-    lastSentHeads: ourHeads,
-    sentHashes,
-  })
+  syncState = Object.assign({}, syncState, {lastSentHeads: ourHeads, sentHashes})
   return [syncState, encodeSyncMessage(syncMessage)]
 }
 
@@ -475,14 +407,13 @@ function generateSyncMessage(backend, syncState) {
  * another peer, that means that peer had those changes, and therefore we now both know about them.
  */
 function advanceHeads(myOldHeads, myNewHeads, ourOldSharedHeads) {
-  const newHeads = myNewHeads.filter(head => !myOldHeads.includes(head))
-  const commonHeads = ourOldSharedHeads.filter(head =>
-    myNewHeads.includes(head)
-  )
+  const newHeads = myNewHeads.filter((head) => !myOldHeads.includes(head))
+  const commonHeads = ourOldSharedHeads.filter((head) => myNewHeads.includes(head))
   const advancedHeads = [...new Set([...newHeads, ...commonHeads])].sort()
   return advancedHeads
 }
 
+
 /**
  * Given a backend, a message message and the state of our peer, apply any changes, update what
  * we believe about the peer, and (if there were applied changes) produce a patch for the frontend
@@ -492,13 +423,10 @@ function receiveSyncMessage(backend, oldSyncState, binaryMessage) {
     throw new Error("generateSyncMessage called with no Automerge document")
   }
   if (!oldSyncState) {
-    throw new Error(
-      "generateSyncMessage requires a syncState, which can be created with initSyncState()"
-    )
+    throw new Error("generateSyncMessage requires a syncState, which can be created with initSyncState()")
   }
 
-  let { sharedHeads, lastSentHeads, sentHashes } = oldSyncState,
-    patch = null
+  let { sharedHeads, lastSentHeads, sentHashes } = oldSyncState, patch = null
   const message = decodeSyncMessage(binaryMessage)
   const beforeHeads = Backend.getHeads(backend)
 
@@ -507,27 +435,18 @@ function receiveSyncMessage(backend, oldSyncState, binaryMessage) {
   // changes without applying them. The set of changes may also be incomplete if the sender decided
   // to break a large set of changes into chunks.
   if (message.changes.length > 0) {
-    ;[backend, patch] = Backend.applyChanges(backend, message.changes)
-    sharedHeads = advanceHeads(
-      beforeHeads,
-      Backend.getHeads(backend),
-      sharedHeads
-    )
+    [backend, patch] = Backend.applyChanges(backend, message.changes)
+    sharedHeads = advanceHeads(beforeHeads, Backend.getHeads(backend), sharedHeads)
   }
 
   // If heads are equal, indicate we don't need to send a response message
-  if (
-    message.changes.length === 0 &&
-    compareArrays(message.heads, beforeHeads)
-  ) {
+  if (message.changes.length === 0 && compareArrays(message.heads, beforeHeads)) {
     lastSentHeads = message.heads
   }
 
   // If all of the remote heads are known to us, that means either our heads are equal, or we are
   // ahead of the remote peer. In this case, take the remote heads to be our shared heads.
-  const knownHeads = message.heads.filter(head =>
-    Backend.getChangeByHash(backend, head)
-  )
+  const knownHeads = message.heads.filter(head => Backend.getChangeByHash(backend, head))
   if (knownHeads.length === message.heads.length) {
     sharedHeads = message.heads
     // If the remote peer has lost all its data, reset our state to perform a full resync
@@ -549,18 +468,14 @@ function receiveSyncMessage(backend, oldSyncState, binaryMessage) {
     theirHave: message.have, // the information we need to calculate the changes they need
     theirHeads: message.heads,
     theirNeed: message.need,
-    sentHashes,
+    sentHashes
   }
   return [backend, syncState, patch]
 }
 
 module.exports = {
-  receiveSyncMessage,
-  generateSyncMessage,
-  encodeSyncMessage,
-  decodeSyncMessage,
-  initSyncState,
-  encodeSyncState,
-  decodeSyncState,
-  BloomFilter, // BloomFilter is a private API, exported only for testing purposes
+  receiveSyncMessage, generateSyncMessage,
+  encodeSyncMessage, decodeSyncMessage,
+  initSyncState, encodeSyncState, decodeSyncState,
+  BloomFilter // BloomFilter is a private API, exported only for testing purposes
 }

automerge-js/src/text.js

@@ -0,0 +1,132 @@
+const { OBJECT_ID } = require('./constants')
+const { isObject } = require('../src/common')
+
+class Text {
+  constructor (text) {
+    const instance = Object.create(Text.prototype)
+    if (typeof text === 'string') {
+      instance.elems = [...text]
+    } else if (Array.isArray(text)) {
+      instance.elems = text
+    } else if (text === undefined) {
+      instance.elems = []
+    } else {
+      throw new TypeError(`Unsupported initial value for Text: ${text}`)
+    }
+    return instance
+  }
+
+  get length () {
+    return this.elems.length
+  }
+
+  get (index) {
+    return this.elems[index]
+  }
+
+  getElemId (index) {
+    return undefined
+  }
+
+  /**
+   * Iterates over the text elements character by character, including any
+   * inline objects.
+   */
+  [Symbol.iterator] () {
+    let elems = this.elems, index = -1
+    return {
+      next () {
+        index += 1
+        if (index < elems.length) {
+          return {done: false, value: elems[index]}
+        } else {
+          return {done: true}
+        }
+      }
+    }
+  }
+
+  /**
+   * Returns the content of the Text object as a simple string, ignoring any
+   * non-character elements.
+   */
+  toString() {
+    // Concatting to a string is faster than creating an array and then
+    // .join()ing for small (<100KB) arrays.
+    // https://jsperf.com/join-vs-loop-w-type-test
+    let str = ''
+    for (const elem of this.elems) {
+      if (typeof elem === 'string') str += elem
+    }
+    return str
+  }
+
+  /**
+   * Returns the content of the Text object as a sequence of strings,
+   * interleaved with non-character elements.
+   *
+   * For example, the value ['a', 'b', {x: 3}, 'c', 'd'] has spans:
+   * => ['ab', {x: 3}, 'cd']
+   */
+  toSpans() {
+    let spans = []
+    let chars = ''
+    for (const elem of this.elems) {
+      if (typeof elem === 'string') {
+        chars += elem
+      } else {
+        if (chars.length > 0) {
+          spans.push(chars)
+          chars = ''
+        }
+        spans.push(elem)
+      }
+    }
+    if (chars.length > 0) {
+      spans.push(chars)
+    }
+    return spans
+  }
+
+  /**
+   * Returns the content of the Text object as a simple string, so that the
+   * JSON serialization of an Automerge document represents text nicely.
+   */
+  toJSON() {
+    return this.toString()
+  }
+
+  /**
+   * Updates the list item at position `index` to a new value `value`.
+   */
+  set (index, value) {
+    this.elems[index] = value
+  }
+
+  /**
+   * Inserts new list items `values` starting at position `index`.
+   */
+  insertAt(index, ...values) {
+    this.elems.splice(index, 0, ... values)
+  }
+
+  /**
+   * Deletes `numDelete` list items starting at position `index`.
+   * if `numDelete` is not given, one item is deleted.
+   */
+  deleteAt(index, numDelete = 1) {
+    this.elems.splice(index, numDelete)
+  }
+}
+
+// Read-only methods that can delegate to the JavaScript built-in array
+for (let method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes',
+                    'indexOf', 'join', 'lastIndexOf', 'map', 'reduce', 'reduceRight',
+                    'slice', 'some', 'toLocaleString']) {
+  Text.prototype[method] = function (...args) {
+    const array = [...this]
+    return array[method](...args)
+  }
+}
+
+module.exports = { Text }

automerge-js/src/uuid.js

@@ -0,0 +1,16 @@
+const { v4: uuid } = require('uuid')
+
+function defaultFactory() {
+  return uuid().replace(/-/g, '')
+}
+
+let factory = defaultFactory
+
+function makeUuid() {
+  return factory()
+}
+
+makeUuid.setFactory = newFactory => { factory = newFactory }
+makeUuid.reset = () => { factory = defaultFactory }
+
+module.exports = makeUuid

automerge-js/test/basic_test.js

@@ -0,0 +1,164 @@
+
+const assert = require('assert')
+const util = require('util')
+const Automerge = require('..')
+
+describe('Automerge', () => {
+    describe('basics', () => {
+        it('should init clone and free', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.clone(doc1);
+        })
+
+        it('handle basic set and read on root object', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.hello = "world"
+              d.big = "little"
+              d.zip = "zop"
+              d.app = "dap"
+            assert.deepEqual(d, {  hello: "world", big: "little", zip: "zop", app: "dap" })
+            })
+            assert.deepEqual(doc2, {  hello: "world", big: "little", zip: "zop", app: "dap" })
+        })
+
+        it('handle basic sets over many changes', () => {
+            let doc1 = Automerge.init()
+            let timestamp = new Date();
+            let counter = new Automerge.Counter(100);
+            let bytes = new Uint8Array([10,11,12]);
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.hello = "world"
+            })
+            let doc3 = Automerge.change(doc2, (d) => {
+              d.counter1 = counter
+            })
+            let doc4 = Automerge.change(doc3, (d) => {
+              d.timestamp1 = timestamp
+            })
+            let doc5 = Automerge.change(doc4, (d) => {
+              d.app = null
+            })
+            let doc6 = Automerge.change(doc5, (d) => {
+              d.bytes1 = bytes
+            })
+            let doc7 = Automerge.change(doc6, (d) => {
+              d.uint = new Automerge.Uint(1)
+              d.int = new Automerge.Int(-1)
+              d.float64 = new Automerge.Float64(5.5)
+              d.number1 = 100
+              d.number2 = -45.67
+              d.true = true
+              d.false = false
+            })
+
+            assert.deepEqual(doc7, {  hello: "world", true: true, false: false, int: -1, uint: 1, float64: 5.5, number1: 100, number2: -45.67, counter1: counter, timestamp1: timestamp, bytes1: bytes, app: null })
+
+            let changes = Automerge.getAllChanges(doc7)
+            let t1 = Automerge.init()
+            ;let [t2] = Automerge.applyChanges(t1, changes)
+            assert.deepEqual(doc7,t2)
+        })
+
+        it('handle overwrites to values', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.hello = "world1"
+            })
+            let doc3 = Automerge.change(doc2, (d) => {
+              d.hello = "world2"
+            })
+            let doc4 = Automerge.change(doc3, (d) => {
+              d.hello = "world3"
+            })
+            let doc5 = Automerge.change(doc4, (d) => {
+              d.hello = "world4"
+            })
+            assert.deepEqual(doc5, {  hello: "world4" } )
+        })
+
+        it('handle set with object value', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.subobj = { hello: "world", subsubobj: { zip: "zop" } }
+            })
+            assert.deepEqual(doc2, { subobj:  { hello: "world", subsubobj: { zip: "zop" } } })
+        })
+
+        it('handle simple list creation', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => d.list = [])
+            assert.deepEqual(doc2, { list: []})
+        })
+
+        it('handle simple lists', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.list = [ 1, 2, 3 ]
+            })
+            assert.deepEqual(doc2.list.length, 3)
+            assert.deepEqual(doc2.list[0], 1)
+            assert.deepEqual(doc2.list[1], 2)
+            assert.deepEqual(doc2.list[2], 3)
+            assert.deepEqual(doc2, { list: [1,2,3] })
+           // assert.deepStrictEqual(Automerge.toJS(doc2), { list: [1,2,3] })
+
+            let doc3 = Automerge.change(doc2, (d) => {
+              d.list[1] = "a"
+            })
+
+            assert.deepEqual(doc3.list.length, 3)
+            assert.deepEqual(doc3.list[0], 1)
+            assert.deepEqual(doc3.list[1], "a")
+            assert.deepEqual(doc3.list[2], 3)
+            assert.deepEqual(doc3, { list: [1,"a",3] })
+        })
+        it('handle simple lists', () => {
+            let doc1 = Automerge.init()
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.list = [ 1, 2, 3 ]
+            })
+            let changes = Automerge.getChanges(doc1, doc2)
+            let docB1 = Automerge.init()
+            ;let [docB2] = Automerge.applyChanges(docB1, changes)
+            assert.deepEqual(docB2, doc2);
+        })
+        it('handle text', () => {
+            let doc1 = Automerge.init()
+            let tmp = new Automerge.Text("hello")
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.list = new Automerge.Text("hello")
+              d.list.insertAt(2,"Z")
+            })
+            let changes = Automerge.getChanges(doc1, doc2)
+            let docB1 = Automerge.init()
+            ;let [docB2] = Automerge.applyChanges(docB1, changes)
+            assert.deepEqual(docB2, doc2);
+        })
+
+        it('have many list methods', () => {
+            let doc1 = Automerge.from({ list: [1,2,3] })
+            assert.deepEqual(doc1, { list: [1,2,3] });
+            let doc2 = Automerge.change(doc1, (d) => {
+              d.list.splice(1,1,9,10)
+            })
+            assert.deepEqual(doc2, { list: [1,9,10,3] });
+            let doc3 = Automerge.change(doc2, (d) => {
+              d.list.push(11,12)
+            })
+            assert.deepEqual(doc3, { list: [1,9,10,3,11,12] });
+            let doc4 = Automerge.change(doc3, (d) => {
+              d.list.unshift(2,2)
+            })
+            assert.deepEqual(doc4, { list: [2,2,1,9,10,3,11,12] });
+            let doc5 = Automerge.change(doc4, (d) => {
+              d.list.shift()
+            })
+            assert.deepEqual(doc5, { list: [2,1,9,10,3,11,12] });
+            let doc6 = Automerge.change(doc5, (d) => {
+              d.list.insertAt(3,100,101)
+            })
+            assert.deepEqual(doc6, { list: [2,1,9,100,101,10,3,11,12] });
+        })
+    })
+})

automerge-js/test/columnar_test.js

@@ -0,0 +1,97 @@
+const assert = require('assert')
+const { checkEncoded } = require('./helpers')
+const Automerge = require('..')
+const { encodeChange, decodeChange } =  Automerge
+
+describe('change encoding', () => {
+  it('should encode text edits', () => {
+    /*
+    const change1 = {actor: 'aaaa', seq: 1, startOp: 1, time: 9, message: '', deps: [], ops: [
+      {action: 'makeText', obj: '_root', key: 'text', insert: false, pred: []},
+      {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'h', pred: []},
+      {action: 'del', obj: '1@aaaa', elemId: '2@aaaa', insert: false, pred: ['2@aaaa']},
+      {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'H', pred: []},
+      {action: 'set', obj: '1@aaaa', elemId: '4@aaaa', insert: true, value: 'i', pred: []}
+    ]}
+    */
+    const change1 = {actor: 'aaaa', seq: 1, startOp: 1, time: 9, message: null, deps: [], ops: [
+      {action: 'makeText', obj: '_root', key: 'text', pred: []},
+      {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'h', pred: []},
+      {action: 'del', obj: '1@aaaa', elemId: '2@aaaa', pred: ['2@aaaa']},
+      {action: 'set', obj: '1@aaaa', elemId: '_head', insert: true, value: 'H', pred: []},
+      {action: 'set', obj: '1@aaaa', elemId: '4@aaaa', insert: true, value: 'i', pred: []}
+    ]}
+    checkEncoded(encodeChange(change1), [
+      0x85, 0x6f, 0x4a, 0x83, // magic bytes
+      0xe2, 0xbd, 0xfb, 0xf5, // checksum
+      1, 94, 0, 2, 0xaa, 0xaa, // chunkType: change, length, deps, actor 'aaaa'
+      1, 1, 9, 0, 0, // seq, startOp, time, message, actor list
+      12, 0x01, 4, 0x02, 4, // column count, objActor, objCtr
+      0x11, 8, 0x13, 7, 0x15, 8, // keyActor, keyCtr, keyStr
+      0x34, 4, 0x42, 6, // insert, action
+      0x56, 6, 0x57, 3, // valLen, valRaw
+      0x70, 6, 0x71, 2, 0x73, 2, // predNum, predActor, predCtr
+      0, 1, 4, 0, // objActor column: null, 0, 0, 0, 0
+      0, 1, 4, 1, // objCtr column: null, 1, 1, 1, 1
+      0, 2, 0x7f, 0, 0, 1, 0x7f, 0, // keyActor column: null, null, 0, null, 0
+      0, 1, 0x7c, 0, 2, 0x7e, 4, // keyCtr column: null, 0, 2, 0, 4
+      0x7f, 4, 0x74, 0x65, 0x78, 0x74, 0, 4, // keyStr column: 'text', null, null, null, null
+      1, 1, 1, 2, // insert column: false, true, false, true, true
+      0x7d, 4, 1, 3, 2, 1, // action column: makeText, set, del, set, set
+      0x7d, 0, 0x16, 0, 2, 0x16, // valLen column: 0, 0x16, 0, 0x16, 0x16
+      0x68, 0x48, 0x69, // valRaw column: 'h', 'H', 'i'
+      2, 0, 0x7f, 1, 2, 0, // predNum column: 0, 0, 1, 0, 0
+      0x7f, 0, // predActor column: 0
+      0x7f, 2 // predCtr column: 2
+    ])
+    const decoded = decodeChange(encodeChange(change1))
+    assert.deepStrictEqual(decoded, Object.assign({hash: decoded.hash}, change1))
+  })
+
+  // FIXME - skipping this b/c it was never implemented in the rust impl and isnt trivial
+/*
+  it.skip('should require strict ordering of preds', () => {
+    const change = new Uint8Array([
+      133, 111, 74, 131, 31, 229, 112, 44, 1, 105, 1, 58, 30, 190, 100, 253, 180, 180, 66, 49, 126,
+      81, 142, 10, 3, 35, 140, 189, 231, 34, 145, 57, 66, 23, 224, 149, 64, 97, 88, 140, 168, 194,
+      229, 4, 244, 209, 58, 138, 67, 140, 1, 152, 236, 250, 2, 0, 1, 4, 55, 234, 66, 242, 8, 21, 11,
+      52, 1, 66, 2, 86, 3, 87, 10, 112, 2, 113, 3, 115, 4, 127, 9, 99, 111, 109, 109, 111, 110, 86,
+      97, 114, 1, 127, 1, 127, 166, 1, 52, 48, 57, 49, 52, 57, 52, 53, 56, 50, 127, 2, 126, 0, 1,
+      126, 139, 1, 0
+    ])
+    assert.throws(() => { decodeChange(change) }, /operation IDs are not in ascending order/)
+  })
+*/
+
+  describe('with trailing bytes', () => {
+    let change = new Uint8Array([
+      0x85, 0x6f, 0x4a, 0x83, // magic bytes
+      0xb2, 0x98, 0x9e, 0xa9, // checksum
+      1, 61, 0, 2, 0x12, 0x34, // chunkType: change, length, deps, actor '1234'
+      1, 1, 252, 250, 220, 255, 5, // seq, startOp, time
+      14, 73, 110, 105, 116, 105, 97, 108, 105, 122, 97, 116, 105, 111, 110, // message: 'Initialization'
+      0, 6, // actor list, column count
+      0x15, 3, 0x34, 1, 0x42, 2, // keyStr, insert, action
+      0x56, 2, 0x57, 1, 0x70, 2, // valLen, valRaw, predNum
+      0x7f, 1, 0x78, // keyStr: 'x'
+      1, // insert: false
+      0x7f, 1, // action: set
+      0x7f, 19, // valLen: 1 byte of type uint
+      1, // valRaw: 1
+      0x7f, 0, // predNum: 0
+      0, 1, 2, 3, 4, 5, 6, 7, 8, 9 // 10 trailing bytes
+    ])
+
+    it('should allow decoding and re-encoding', () => {
+      // NOTE: This calls the JavaScript encoding and decoding functions, even when the WebAssembly
+      // backend is loaded. Should the wasm backend export its own functions for testing?
+      checkEncoded(change, encodeChange(decodeChange(change)))
+    })
+
+    it('should be preserved in document encoding', () => {
+      const [doc] = Automerge.applyChanges(Automerge.init(), [change])
+      const [reconstructed] = Automerge.getAllChanges(Automerge.load(Automerge.save(doc)))
+      checkEncoded(change, reconstructed)
+    })
+  })
+})

automerge-js/test/helpers.js

@@ -1,21 +1,16 @@
-import * as assert from "assert"
-import { Encoder } from "./legacy/encoding"
+const assert = require('assert')
+const { Encoder } = require('../src/encoding')
 
 // Assertion that succeeds if the first argument deepStrictEquals at least one of the
 // subsequent arguments (but we don't care which one)
-export function assertEqualsOneOf(actual, ...expected) {
+function assertEqualsOneOf(actual, ...expected) {
   assert(expected.length > 0)
   for (let i = 0; i < expected.length; i++) {
     try {
       assert.deepStrictEqual(actual, expected[i])
       return // if we get here without an exception, that means success
     } catch (e) {
-      if (e instanceof assert.AssertionError) {
-        if (!e.name.match(/^AssertionError/) || i === expected.length - 1)
-          throw e
-      } else {
-        throw e
-      }
+      if (!e.name.match(/^AssertionError/) || i === expected.length - 1) throw e
     }
   }
 }
@@ -24,13 +19,14 @@ export function assertEqualsOneOf(actual, ...expected) {
  * Asserts that the byte array maintained by `encoder` contains the same byte
  * sequence as the array `bytes`.
  */
-export function checkEncoded(encoder, bytes, detail?) {
-  const encoded = encoder instanceof Encoder ? encoder.buffer : encoder
+function checkEncoded(encoder, bytes, detail) {
+  const encoded = (encoder instanceof Encoder) ? encoder.buffer : encoder
   const expected = new Uint8Array(bytes)
-  const message =
-    (detail ? `${detail}: ` : "") + `${encoded} expected to equal ${expected}`
+  const message = (detail ? `${detail}: ` : '') + `${encoded} expected to equal ${expected}`
   assert(encoded.byteLength === expected.byteLength, message)
   for (let i = 0; i < encoded.byteLength; i++) {
     assert(encoded[i] === expected[i], message)
   }
 }
+
+module.exports = { assertEqualsOneOf, checkEncoded }

automerge-js/test/text_test.js

@@ -0,0 +1,697 @@
+const assert = require('assert')
+const Automerge = require('..')
+const { assertEqualsOneOf } = require('./helpers')
+
+function attributeStateToAttributes(accumulatedAttributes) {
+  const attributes = {}
+  Object.entries(accumulatedAttributes).forEach(([key, values]) => {
+    if (values.length && values[0] !== null) {
+      attributes[key] = values[0]
+    }
+  })
+  return attributes
+}
+
+function isEquivalent(a, b) {
+  const aProps = Object.getOwnPropertyNames(a)
+  const bProps = Object.getOwnPropertyNames(b)
+
+  if (aProps.length != bProps.length) {
+      return false
+  }
+
+  for (let i = 0; i < aProps.length; i++) {
+    const propName = aProps[i]
+      if (a[propName] !== b[propName]) {
+          return false
+      }
+  }
+
+  return true
+}
+
+function isControlMarker(pseudoCharacter) {
+  return typeof pseudoCharacter === 'object' && pseudoCharacter.attributes
+}
+
+function opFrom(text, attributes) {
+  let op = { insert: text }
+  if (Object.keys(attributes).length > 0) {
+      op.attributes = attributes
+  }
+  return op
+}
+
+function accumulateAttributes(span, accumulatedAttributes) {
+  Object.entries(span).forEach(([key, value]) => {
+    if (!accumulatedAttributes[key]) {
+      accumulatedAttributes[key] = []
+    }
+    if (value === null) {
+      if (accumulatedAttributes[key].length === 0 || accumulatedAttributes[key] === null) {
+        accumulatedAttributes[key].unshift(null)
+      } else {
+        accumulatedAttributes[key].shift()
+      }
+    } else {
+      if (accumulatedAttributes[key][0] === null) {
+        accumulatedAttributes[key].shift()
+      } else {
+        accumulatedAttributes[key].unshift(value)
+      }
+    }
+  })
+  return accumulatedAttributes
+}
+
+function automergeTextToDeltaDoc(text) {
+  let ops = []
+  let controlState = {}
+  let currentString = ""
+  let attributes = {}
+  text.toSpans().forEach((span) => {
+    if (isControlMarker(span)) {
+      controlState = accumulateAttributes(span.attributes, controlState)
+    } else {
+      let next = attributeStateToAttributes(controlState)
+
+      // if the next span has the same calculated attributes as the current span
+      // don't bother outputting it as a separate span, just let it ride
+      if (typeof span === 'string' && isEquivalent(next, attributes)) {
+          currentString = currentString + span
+          return
+      }
+
+      if (currentString) {
+        ops.push(opFrom(currentString, attributes))
+      }
+
+      // If we've got a string, we might be able to concatenate it to another
+      // same-attributed-string, so remember it and go to the next iteration.
+      if (typeof span === 'string') {
+        currentString = span
+        attributes = next
+      } else {
+        // otherwise we have an embed "character" and should output it immediately.
+        // embeds are always one-"character" in length.
+        ops.push(opFrom(span, next))
+        currentString = ''
+        attributes = {}
+      }
+    }
+  })
+
+  // at the end, flush any accumulated string out
+  if (currentString) {
+    ops.push(opFrom(currentString, attributes))
+  }
+
+  return ops
+}
+
+function inverseAttributes(attributes) {
+  let invertedAttributes = {}
+  Object.keys(attributes).forEach((key) => {
+    invertedAttributes[key] = null
+  })
+  return invertedAttributes
+}
+
+function applyDeleteOp(text, offset, op) {
+  let length = op.delete
+  while (length > 0) {
+    if (isControlMarker(text.get(offset))) {
+      offset += 1
+    } else {
+      // we need to not delete control characters, but we do delete embed characters
+      text.deleteAt(offset, 1)
+      length -= 1
+    }
+  }
+  return [text, offset]
+}
+
+function applyRetainOp(text, offset, op) {
+  let length = op.retain
+
+  if (op.attributes) {
+    text.insertAt(offset, { attributes: op.attributes })
+    offset += 1
+  }
+
+  while (length > 0) {
+    const char = text.get(offset)
+    offset += 1
+    if (!isControlMarker(char)) {
+      length -= 1
+    }
+  }
+
+  if (op.attributes) {
+    text.insertAt(offset, { attributes: inverseAttributes(op.attributes) })
+    offset += 1
+  }
+
+  return [text, offset]
+}
+
+
+function applyInsertOp(text, offset, op) {
+  let originalOffset = offset
+
+  if (typeof op.insert === 'string') {
+    text.insertAt(offset, ...op.insert.split(''))
+    offset += op.insert.length
+  } else {
+    // we have an embed or something similar
+    text.insertAt(offset, op.insert)
+    offset += 1
+  }
+
+  if (op.attributes) {
+    text.insertAt(originalOffset, { attributes: op.attributes })
+    offset += 1
+  }
+  if (op.attributes) {
+    text.insertAt(offset, { attributes: inverseAttributes(op.attributes) })
+    offset += 1
+  }
+  return [text, offset]
+}
+
+// XXX: uhhhhh, why can't I pass in text?
+function applyDeltaDocToAutomergeText(delta, doc) {
+  let offset = 0
+
+  delta.forEach(op => {
+    if (op.retain) {
+      [, offset] = applyRetainOp(doc.text, offset, op)
+    } else if (op.delete) {
+      [, offset] = applyDeleteOp(doc.text, offset, op)
+    } else if (op.insert) {
+      [, offset] = applyInsertOp(doc.text, offset, op)
+    }
+  })
+}
+
+describe('Automerge.Text', () => {
+  let s1, s2
+  beforeEach(() => {
+    s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text())
+    s2 = Automerge.merge(Automerge.init(), s1)
+  })
+
+  it('should support insertion', () => {
+    s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a'))
+    assert.strictEqual(s1.text.length, 1)
+    assert.strictEqual(s1.text.get(0), 'a')
+    assert.strictEqual(s1.text.toString(), 'a')
+    //assert.strictEqual(s1.text.getElemId(0), `2@${Automerge.getActorId(s1)}`)
+  })
+
+  it('should support deletion', () => {
+    s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', 'b', 'c'))
+    s1 = Automerge.change(s1, doc => doc.text.deleteAt(1, 1))
+    assert.strictEqual(s1.text.length, 2)
+    assert.strictEqual(s1.text.get(0), 'a')
+    assert.strictEqual(s1.text.get(1), 'c')
+    assert.strictEqual(s1.text.toString(), 'ac')
+  })
+
+  it("should support implicit and explicit deletion", () => {
+    s1 = Automerge.change(s1, doc => doc.text.insertAt(0, "a", "b", "c"))
+    s1 = Automerge.change(s1, doc => doc.text.deleteAt(1))
+    s1 = Automerge.change(s1, doc => doc.text.deleteAt(1, 0))
+    assert.strictEqual(s1.text.length, 2)
+    assert.strictEqual(s1.text.get(0), "a")
+    assert.strictEqual(s1.text.get(1), "c")
+    assert.strictEqual(s1.text.toString(), "ac")
+  })
+
+  it('should handle concurrent insertion', () => {
+    s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', 'b', 'c'))
+    s2 = Automerge.change(s2, doc => doc.text.insertAt(0, 'x', 'y', 'z'))
+    s1 = Automerge.merge(s1, s2)
+    assert.strictEqual(s1.text.length, 6)
+    assertEqualsOneOf(s1.text.toString(), 'abcxyz', 'xyzabc')
+    assertEqualsOneOf(s1.text.join(''), 'abcxyz', 'xyzabc')
+  })
+
+  it('should handle text and other ops in the same change', () => {
+    s1 = Automerge.change(s1, doc => {
+      doc.foo = 'bar'
+      doc.text.insertAt(0, 'a')
+    })
+    assert.strictEqual(s1.foo, 'bar')
+    assert.strictEqual(s1.text.toString(), 'a')
+    assert.strictEqual(s1.text.join(''), 'a')
+  })
+
+  it('should serialize to JSON as a simple string', () => {
+    s1 = Automerge.change(s1, doc => doc.text.insertAt(0, 'a', '"', 'b'))
+    assert.strictEqual(JSON.stringify(s1), '{"text":"a\\"b"}')
+  })
+
+  it('should allow modification before an object is assigned to a document', () => {
+    s1 = Automerge.change(Automerge.init(), doc => {
+      const text = new Automerge.Text()
+      text.insertAt(0, 'a', 'b', 'c', 'd')
+      text.deleteAt(2)
+      doc.text = text
+      assert.strictEqual(doc.text.toString(), 'abd')
+      assert.strictEqual(doc.text.join(''), 'abd')
+    })
+    assert.strictEqual(s1.text.toString(), 'abd')
+    assert.strictEqual(s1.text.join(''), 'abd')
+  })
+
+  it('should allow modification after an object is assigned to a document', () => {
+    s1 = Automerge.change(Automerge.init(), doc => {
+      const text = new Automerge.Text()
+      doc.text = text
+      doc.text.insertAt(0, 'a', 'b', 'c', 'd')
+      doc.text.deleteAt(2)
+      assert.strictEqual(doc.text.toString(), 'abd')
+      assert.strictEqual(doc.text.join(''), 'abd')
+    })
+    assert.strictEqual(s1.text.join(''), 'abd')
+  })
+
+  it('should not allow modification outside of a change callback', () => {
+    assert.throws(() => s1.text.insertAt(0, 'a'), /object cannot be modified outside of a change block/)
+  })
+
+  describe('with initial value', () => {
+    it('should accept a string as initial value', () => {
+      let s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text('init'))
+      assert.strictEqual(s1.text.length, 4)
+      assert.strictEqual(s1.text.get(0), 'i')
+      assert.strictEqual(s1.text.get(1), 'n')
+      assert.strictEqual(s1.text.get(2), 'i')
+      assert.strictEqual(s1.text.get(3), 't')
+      assert.strictEqual(s1.text.toString(), 'init')
+    })
+
+    it('should accept an array as initial value', () => {
+      let s1 = Automerge.change(Automerge.init(), doc => doc.text = new Automerge.Text(['i', 'n', 'i', 't']))
+      assert.strictEqual(s1.text.length, 4)
+      assert.strictEqual(s1.text.get(0), 'i')
+      assert.strictEqual(s1.text.get(1), 'n')
+      assert.strictEqual(s1.text.get(2), 'i')
+      assert.strictEqual(s1.text.get(3), 't')
+      assert.strictEqual(s1.text.toString(), 'init')
+    })
+
+    it('should initialize text in Automerge.from()', () => {
+      let s1 = Automerge.from({text: new Automerge.Text('init')})
+      assert.strictEqual(s1.text.length, 4)
+      assert.strictEqual(s1.text.get(0), 'i')
+      assert.strictEqual(s1.text.get(1), 'n')
+      assert.strictEqual(s1.text.get(2), 'i')
+      assert.strictEqual(s1.text.get(3), 't')
+      assert.strictEqual(s1.text.toString(), 'init')
+    })
+
+    it('should encode the initial value as a change', () => {
+      const s1 = Automerge.from({text: new Automerge.Text('init')})
+      const changes = Automerge.getAllChanges(s1)
+      assert.strictEqual(changes.length, 1)
+      const [s2] = Automerge.applyChanges(Automerge.init(), changes)
+      assert.strictEqual(s2.text instanceof Automerge.Text, true)
+      assert.strictEqual(s2.text.toString(), 'init')
+      assert.strictEqual(s2.text.join(''), 'init')
+    })
+
+    it('should allow immediate access to the value', () => {
+      Automerge.change(Automerge.init(), doc => {
+        const text = new Automerge.Text('init')
+        assert.strictEqual(text.length, 4)
+        assert.strictEqual(text.get(0), 'i')
+        assert.strictEqual(text.toString(), 'init')
+        doc.text = text
+        assert.strictEqual(doc.text.length, 4)
+        assert.strictEqual(doc.text.get(0), 'i')
+        assert.strictEqual(doc.text.toString(), 'init')
+      })
+    })
+
+    it('should allow pre-assignment modification of the initial value', () => {
+      let s1 = Automerge.change(Automerge.init(), doc => {
+        const text = new Automerge.Text('init')
+        text.deleteAt(3)
+        assert.strictEqual(text.join(''), 'ini')
+        doc.text = text
+        assert.strictEqual(doc.text.join(''), 'ini')
+        assert.strictEqual(doc.text.toString(), 'ini')
+      })
+      assert.strictEqual(s1.text.toString(), 'ini')
+      assert.strictEqual(s1.text.join(''), 'ini')
+    })
+
+    it('should allow post-assignment modification of the initial value', () => {
+      let s1 = Automerge.change(Automerge.init(), doc => {
+        const text = new Automerge.Text('init')
+        doc.text = text
+        doc.text.deleteAt(0)
+        doc.text.insertAt(0, 'I')
+        assert.strictEqual(doc.text.join(''), 'Init')
+        assert.strictEqual(doc.text.toString(), 'Init')
+      })
+      assert.strictEqual(s1.text.join(''), 'Init')
+      assert.strictEqual(s1.text.toString(), 'Init')
+    })
+  })
+
+  describe('non-textual control characters', () => {
+    let s1
+    beforeEach(() => {
+      s1 = Automerge.change(Automerge.init(), doc => {
+        doc.text = new Automerge.Text()
+        doc.text.insertAt(0, 'a')
+        doc.text.insertAt(1, { attribute: 'bold' })
+      })
+    })
+
+    it('should allow fetching non-textual characters', () => {
+      assert.deepEqual(s1.text.get(1), { attribute: 'bold' })
+      //assert.strictEqual(s1.text.getElemId(1), `3@${Automerge.getActorId(s1)}`)
+    })
+
+    it('should include control characters in string length', () => {
+      assert.strictEqual(s1.text.length, 2)
+      assert.strictEqual(s1.text.get(0), 'a')
+    })
+
+    it('should exclude control characters from toString()', () => {
+      assert.strictEqual(s1.text.toString(), 'a')
+    })
+
+    it('should allow control characters to be updated', () => {
+      const s2 = Automerge.change(s1, doc => doc.text.get(1).attribute = 'italic')
+      const s3 = Automerge.load(Automerge.save(s2))
+      assert.strictEqual(s1.text.get(1).attribute, 'bold')
+      assert.strictEqual(s2.text.get(1).attribute, 'italic')
+      assert.strictEqual(s3.text.get(1).attribute, 'italic')
+    })
+
+    describe('spans interface to Text', () => {
+      it('should return a simple string as a single span', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('hello world')
+        })
+        assert.deepEqual(s1.text.toSpans(), ['hello world'])
+      })
+      it('should return an empty string as an empty array', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text()
+        })
+        assert.deepEqual(s1.text.toSpans(), [])
+      })
+      it('should split a span at a control character', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('hello world')
+          doc.text.insertAt(5, { attributes: { bold: true } })
+        })
+        assert.deepEqual(s1.text.toSpans(),
+          ['hello', { attributes: { bold: true } }, ' world'])
+      })
+      it('should allow consecutive control characters', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('hello world')
+          doc.text.insertAt(5, { attributes: { bold: true } })
+          doc.text.insertAt(6, { attributes: { italic: true } })
+        })
+        assert.deepEqual(s1.text.toSpans(),
+          ['hello',
+           { attributes: { bold: true } },
+           { attributes: { italic: true } },
+           ' world'
+          ])
+      })
+      it('should allow non-consecutive control characters', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('hello world')
+          doc.text.insertAt(5, { attributes: { bold: true } })
+          doc.text.insertAt(12, { attributes: { italic: true } })
+        })
+        assert.deepEqual(s1.text.toSpans(),
+          ['hello',
+           { attributes: { bold: true } },
+           ' world',
+           { attributes: { italic: true } }
+          ])
+      })
+
+      it('should be convertable into a Quill delta', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Gandalf the Grey')
+          doc.text.insertAt(0,  { attributes: { bold: true } })
+          doc.text.insertAt(7 + 1, { attributes: { bold: null } })
+          doc.text.insertAt(12 + 2, { attributes: { color: '#cccccc' } })
+        })
+
+        let deltaDoc = automergeTextToDeltaDoc(s1.text)
+
+        // From https://quilljs.com/docs/delta/
+        let expectedDoc = [
+          { insert: 'Gandalf', attributes: { bold: true } },
+          { insert: ' the ' },
+          { insert: 'Grey', attributes: { color: '#cccccc' } }
+        ]
+
+        assert.deepEqual(deltaDoc, expectedDoc)
+      })
+
+      it('should support embeds', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('')
+          doc.text.insertAt(0, { attributes: { link: 'https://quilljs.com' } })
+          doc.text.insertAt(1, {
+            image: 'https://quilljs.com/assets/images/icon.png'
+          })
+          doc.text.insertAt(2, { attributes: { link: null } })
+        })
+
+        let deltaDoc = automergeTextToDeltaDoc(s1.text)
+
+        // From https://quilljs.com/docs/delta/
+        let expectedDoc = [{
+          // An image link
+          insert: {
+            image: 'https://quilljs.com/assets/images/icon.png'
+          },
+          attributes: {
+            link: 'https://quilljs.com'
+          }
+        }]
+
+        assert.deepEqual(deltaDoc, expectedDoc)
+      })
+
+      it('should handle concurrent overlapping spans', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Gandalf the Grey')
+        })
+
+        let s2 = Automerge.merge(Automerge.init(), s1)
+
+        let s3 = Automerge.change(s1, doc => {
+          doc.text.insertAt(8,  { attributes: { bold: true } })
+          doc.text.insertAt(16 + 1, { attributes: { bold: null } })
+        })
+
+        let s4 = Automerge.change(s2, doc => {
+          doc.text.insertAt(0,  { attributes: { bold: true } })
+          doc.text.insertAt(11 + 1, { attributes: { bold: null } })
+        })
+
+        let merged = Automerge.merge(s3, s4)
+
+        let deltaDoc = automergeTextToDeltaDoc(merged.text)
+
+        // From https://quilljs.com/docs/delta/
+        let expectedDoc = [
+          { insert: 'Gandalf the Grey', attributes: { bold: true } },
+        ]
+
+        assert.deepEqual(deltaDoc, expectedDoc)
+      })
+
+      it('should handle debolding spans', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Gandalf the Grey')
+        })
+
+        let s2 = Automerge.merge(Automerge.init(), s1)
+
+        let s3 = Automerge.change(s1, doc => {
+          doc.text.insertAt(0,  { attributes: { bold: true } })
+          doc.text.insertAt(16 + 1, { attributes: { bold: null } })
+        })
+
+        let s4 = Automerge.change(s2, doc => {
+          doc.text.insertAt(8,  { attributes: { bold: null } })
+          doc.text.insertAt(11 + 1, { attributes: { bold: true } })
+        })
+
+
+        let merged = Automerge.merge(s3, s4)
+
+        let deltaDoc = automergeTextToDeltaDoc(merged.text)
+
+        // From https://quilljs.com/docs/delta/
+        let expectedDoc = [
+          { insert: 'Gandalf ', attributes: { bold: true } },
+          { insert: 'the' },
+          { insert: ' Grey', attributes: { bold: true } },
+        ]
+
+        assert.deepEqual(deltaDoc, expectedDoc)
+      })
+
+      // xxx: how would this work for colors?
+      it('should handle destyling across destyled spans', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Gandalf the Grey')
+        })
+
+        let s2 = Automerge.merge(Automerge.init(), s1)
+
+        let s3 = Automerge.change(s1, doc => {
+          doc.text.insertAt(0,  { attributes: { bold: true } })
+          doc.text.insertAt(16 + 1, { attributes: { bold: null } })
+        })
+
+        let s4 = Automerge.change(s2, doc => {
+          doc.text.insertAt(8,  { attributes: { bold: null } })
+          doc.text.insertAt(11 + 1, { attributes: { bold: true } })
+        })
+
+        let merged = Automerge.merge(s3, s4)
+
+        let final = Automerge.change(merged, doc => {
+          doc.text.insertAt(3 + 1, { attributes: { bold: null } })
+          doc.text.insertAt(doc.text.length, { attributes: { bold: true } })
+        })
+
+        let deltaDoc = automergeTextToDeltaDoc(final.text)
+
+        // From https://quilljs.com/docs/delta/
+        let expectedDoc = [
+          { insert: 'Gan', attributes: { bold: true } },
+          { insert: 'dalf the Grey' },
+        ]
+
+        assert.deepEqual(deltaDoc, expectedDoc)
+      })
+
+      it('should apply an insert', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Hello world')
+        })
+
+        const delta = [
+          { retain: 6 },
+          { insert: 'reader' },
+          { delete: 5 }
+        ]
+
+        let s2 = Automerge.change(s1, doc => {
+          applyDeltaDocToAutomergeText(delta, doc)
+        })
+
+        assert.strictEqual(s2.text.join(''), 'Hello reader')
+      })
+
+      it('should apply an insert with control characters', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Hello world')
+        })
+
+        const delta = [
+          { retain: 6 },
+          { insert: 'reader', attributes: { bold: true } },
+          { delete: 5 },
+          { insert: '!' }
+        ]
+
+        let s2 = Automerge.change(s1, doc => {
+          applyDeltaDocToAutomergeText(delta, doc)
+        })
+
+        assert.strictEqual(s2.text.toString(), 'Hello reader!')
+        assert.deepEqual(s2.text.toSpans(), [
+          "Hello ",
+          { attributes: { bold: true } },
+          "reader",
+          { attributes: { bold: null } },
+          "!"
+        ])
+      })
+
+      it('should account for control characters in retain/delete lengths', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('Hello world')
+          doc.text.insertAt(4, { attributes: { color: '#ccc' } })
+          doc.text.insertAt(10, { attributes: { color: '#f00' } })
+        })
+
+        const delta = [
+          { retain: 6 },
+          { insert: 'reader', attributes: { bold: true } },
+          { delete: 5 },
+          { insert: '!' }
+        ]
+
+        let s2 = Automerge.change(s1, doc => {
+          applyDeltaDocToAutomergeText(delta, doc)
+        })
+
+        assert.strictEqual(s2.text.toString(), 'Hello reader!')
+        assert.deepEqual(s2.text.toSpans(), [
+          "Hell",
+          { attributes: { color: '#ccc'} },
+          "o ",
+          { attributes: { bold: true } },
+          "reader",
+          { attributes: { bold: null } },
+          { attributes: { color: '#f00'} },
+          "!"
+        ])
+      })
+
+      it('should support embeds', () => {
+        let s1 = Automerge.change(Automerge.init(), doc => {
+          doc.text = new Automerge.Text('')
+        })
+
+        let deltaDoc = [{
+          // An image link
+          insert: {
+            image: 'https://quilljs.com/assets/images/icon.png'
+          },
+          attributes: {
+            link: 'https://quilljs.com'
+          }
+        }]
+
+        let s2 = Automerge.change(s1, doc => {
+          applyDeltaDocToAutomergeText(deltaDoc, doc)
+        })
+
+        assert.deepEqual(s2.text.toSpans(), [
+          { attributes: { link: 'https://quilljs.com' } },
+          { image: 'https://quilljs.com/assets/images/icon.png'},
+          { attributes: { link: null } },
+        ])
+      })
+    })
+  })
+
+  it('should support unicode when creating text', () => {
+    s1 = Automerge.from({
+      text: new Automerge.Text('🐦')
+    })
+    assert.strictEqual(s1.text.get(0), '🐦')
+  })
+})

automerge-js/test/uuid_test.js

@@ -0,0 +1,32 @@
+const assert = require('assert')
+const Automerge = require('..')
+
+const uuid = Automerge.uuid
+
+describe('uuid', () => {
+  afterEach(() => {
+    uuid.reset()
+  })
+
+  describe('default implementation', () => {
+    it('generates unique values', () => {
+      assert.notEqual(uuid(), uuid())
+    })
+  })
+
+  describe('custom implementation', () => {
+    let counter
+
+    function customUuid() {
+      return `custom-uuid-${counter++}`
+    }
+
+    before(() => uuid.setFactory(customUuid))
+    beforeEach(() => counter = 0)
+
+    it('invokes the custom factory', () => {
+      assert.equal(uuid(), 'custom-uuid-0')
+      assert.equal(uuid(), 'custom-uuid-1')
+    })
+  })
+})

automerge-wasm/.gitignore

@@ -1,6 +1,5 @@
 /node_modules
-/bundler
-/nodejs
-/deno
+/dev
+/target
 Cargo.lock
 yarn.lock

automerge-wasm/Cargo.toml

@@ -2,14 +2,13 @@
 [package]
 name = "automerge-wasm"
 description = "An js/wasm wrapper for the rust implementation of automerge-backend"
-repository = "https://github.com/automerge/automerge-rs"
+# repository = "https://github.com/automerge/automerge-rs"
 version = "0.1.0"
 authors = ["Alex Good <alex@memoryandthought.me>","Orion Henry <orion@inkandswitch.com>", "Martin Kleppmann"]
 categories = ["wasm"]
 readme = "README.md"
 edition = "2021"
 license = "MIT"
-rust-version = "1.57.0"
 
 [lib]
 crate-type = ["cdylib","rlib"]
@@ -28,16 +27,15 @@ serde = "^1.0"
 serde_json = "^1.0"
 rand = { version = "^0.8.4" }
 getrandom = { version = "^0.2.2", features=["js"] }
-uuid = { version = "^1.2.1", features=["v4", "js", "serde"] }
-serde-wasm-bindgen = "0.4.3"
+uuid = { version = "^0.8.2", features=["v4", "wasm-bindgen", "serde"] }
+serde-wasm-bindgen = "0.1.3"
 serde_bytes = "0.11.5"
+unicode-segmentation = "1.7.1"
 hex = "^0.4.3"
 regex = "^1.5"
-itertools = "^0.10.3"
-thiserror = "^1.0.16"
 
 [dependencies.wasm-bindgen]
-version = "^0.2.83"
+version = "^0.2"
 #features = ["std"]
 features = ["serde-serialize", "std"]
 
@@ -57,6 +55,5 @@ features = ["console"]
 
 [dev-dependencies]
 futures = "^0.1"
-proptest = { version = "^1.0.0", default-features = false, features = ["std"] }
 wasm-bindgen-futures = "^0.4"
 wasm-bindgen-test = "^0.3"

automerge-wasm/README.md

@@ -0,0 +1,696 @@
+## Automerge WASM Low Level Interface
+
+This is a low level automerge library written in rust exporting a javascript API via WASM. This low level api is the underpinning to the `automerge-js` library that reimplements the Automerge API via these low level functions.
+
+### Static Functions
+
+### Methods
+
+`doc.clone(actor?: string)` : Make a complete
+
+`doc.free()` : deallocate WASM memory associated with a document
+
+```rust
+    #[wasm_bindgen]
+    pub fn free(self) {}
+
+    #[wasm_bindgen(js_name = pendingOps)]
+    pub fn pending_ops(&self) -> JsValue {
+        (self.0.pending_ops() as u32).into()
+    }
+
+    pub fn commit(&mut self, message: Option<String>, time: Option<f64>) -> Array {
+        let heads = self.0.commit(message, time.map(|n| n as i64));
+        let heads: Array = heads
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        heads
+    }
+
+    pub fn rollback(&mut self) -> f64 {
+        self.0.rollback() as f64
+    }
+
+    pub fn keys(&mut self, obj: String, heads: Option<Array>) -> Result<Array, JsValue> {
+        let obj = self.import(obj)?;
+        let result = if let Some(heads) = get_heads(heads) {
+            self.0.keys_at(&obj, &heads)
+        } else {
+            self.0.keys(&obj)
+        }
+        .iter()
+        .map(|s| JsValue::from_str(s))
+        .collect();
+        Ok(result)
+    }
+
+    pub fn text(&mut self, obj: String, heads: Option<Array>) -> Result<String, JsValue> {
+        let obj = self.import(obj)?;
+        if let Some(heads) = get_heads(heads) {
+            self.0.text_at(&obj, &heads)
+        } else {
+            self.0.text(&obj)
+        }
+        .map_err(to_js_err)
+    }
+
+    pub fn splice(
+        &mut self,
+        obj: String,
+        start: f64,
+        delete_count: f64,
+        text: JsValue,
+    ) -> Result<Option<Array>, JsValue> {
+        let obj = self.import(obj)?;
+        let start = start as usize;
+        let delete_count = delete_count as usize;
+        let mut vals = vec![];
+        if let Some(t) = text.as_string() {
+            self.0
+                .splice_text(&obj, start, delete_count, &t)
+                .map_err(to_js_err)?;
+            Ok(None)
+        } else {
+            if let Ok(array) = text.dyn_into::<Array>() {
+                for i in array.iter() {
+                    if let Ok(array) = i.clone().dyn_into::<Array>() {
+                        let value = array.get(1);
+                        let datatype = array.get(2);
+                        let value = self.import_value(value, datatype.as_string())?;
+                        vals.push(value);
+                    } else {
+                        let value = self.import_value(i, None)?;
+                        vals.push(value);
+                    }
+                }
+            }
+            let result = self
+                .0
+                .splice(&obj, start, delete_count, vals)
+                .map_err(to_js_err)?;
+            if result.is_empty() {
+                Ok(None)
+            } else {
+                let result: Array = result
+                    .iter()
+                    .map(|r| JsValue::from(r.to_string()))
+                    .collect();
+                Ok(result.into())
+            }
+        }
+    }
+
+    pub fn push(
+        &mut self,
+        obj: String,
+        value: JsValue,
+        datatype: Option<String>,
+    ) -> Result<Option<String>, JsValue> {
+        let obj = self.import(obj)?;
+        let value = self.import_value(value, datatype)?;
+        let index = self.0.length(&obj);
+        let opid = self.0.insert(&obj, index, value).map_err(to_js_err)?;
+        Ok(opid.map(|id| id.to_string()))
+    }
+
+    pub fn insert(
+        &mut self,
+        obj: String,
+        index: f64,
+        value: JsValue,
+        datatype: Option<String>,
+    ) -> Result<Option<String>, JsValue> {
+        let obj = self.import(obj)?;
+        let index = index as f64;
+        let value = self.import_value(value, datatype)?;
+        let opid = self
+            .0
+            .insert(&obj, index as usize, value)
+            .map_err(to_js_err)?;
+        Ok(opid.map(|id| id.to_string()))
+    }
+
+    pub fn set(
+        &mut self,
+        obj: String,
+        prop: JsValue,
+        value: JsValue,
+        datatype: Option<String>,
+    ) -> Result<Option<String>, JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let value = self.import_value(value, datatype)?;
+        let opid = self.0.set(&obj, prop, value).map_err(to_js_err)?;
+        Ok(opid.map(|id| id.to_string()))
+    }
+
+    pub fn make(
+        &mut self,
+        obj: String,
+        prop: JsValue,
+        value: JsValue,
+    ) -> Result<String, JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let value = self.import_value(value, None)?;
+        if value.is_object() {
+          let opid = self.0.set(&obj, prop, value).map_err(to_js_err)?;
+          Ok(opid.unwrap().to_string())
+        } else {
+          Err("invalid object type".into())
+        }
+    }
+
+    pub fn inc(&mut self, obj: String, prop: JsValue, value: JsValue) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let value: f64 = value
+            .as_f64()
+            .ok_or("inc needs a numberic value")
+            .map_err(to_js_err)?;
+        self.0.inc(&obj, prop, value as i64).map_err(to_js_err)?;
+        Ok(())
+    }
+
+    pub fn value(
+        &mut self,
+        obj: String,
+        prop: JsValue,
+        heads: Option<Array>,
+    ) -> Result<Array, JsValue> {
+        let obj = self.import(obj)?;
+        let result = Array::new();
+        let prop = to_prop(prop);
+        let heads = get_heads(heads);
+        if let Ok(prop) = prop {
+            let value = if let Some(h) = heads {
+                self.0.value_at(&obj, prop, &h)
+            } else {
+                self.0.value(&obj, prop)
+            }
+            .map_err(to_js_err)?;
+            match value {
+                Some((Value::Object(obj_type), obj_id)) => {
+                    result.push(&obj_type.to_string().into());
+                    result.push(&obj_id.to_string().into());
+                }
+                Some((Value::Scalar(value), _)) => {
+                    result.push(&datatype(&value).into());
+                    result.push(&ScalarValue(value).into());
+                }
+                None => {}
+            }
+        }
+        Ok(result)
+    }
+
+    pub fn values(
+        &mut self,
+        obj: String,
+        arg: JsValue,
+        heads: Option<Array>,
+    ) -> Result<Array, JsValue> {
+        let obj = self.import(obj)?;
+        let result = Array::new();
+        let prop = to_prop(arg);
+        if let Ok(prop) = prop {
+            let values = if let Some(heads) = get_heads(heads) {
+                self.0.values_at(&obj, prop, &heads)
+            } else {
+                self.0.values(&obj, prop)
+            }
+            .map_err(to_js_err)?;
+            for value in values {
+                match value {
+                    (Value::Object(obj_type), obj_id) => {
+                        let sub = Array::new();
+                        sub.push(&obj_type.to_string().into());
+                        sub.push(&obj_id.to_string().into());
+                        result.push(&sub.into());
+                    }
+                    (Value::Scalar(value), id) => {
+                        let sub = Array::new();
+                        sub.push(&datatype(&value).into());
+                        sub.push(&ScalarValue(value).into());
+                        sub.push(&id.to_string().into());
+                        result.push(&sub.into());
+                    }
+                }
+            }
+        }
+        Ok(result)
+    }
+
+    pub fn length(&mut self, obj: String, heads: Option<Array>) -> Result<f64, JsValue> {
+        let obj = self.import(obj)?;
+        if let Some(heads) = get_heads(heads) {
+            Ok(self.0.length_at(&obj, &heads) as f64)
+        } else {
+            Ok(self.0.length(&obj) as f64)
+        }
+    }
+
+    pub fn del(&mut self, obj: String, prop: JsValue) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let prop = to_prop(prop)?;
+        self.0.del(&obj, prop).map_err(to_js_err)?;
+        Ok(())
+    }
+
+    pub fn mark(
+        &mut self,
+        obj: JsValue,
+        range: JsValue,
+        name: JsValue,
+        value: JsValue,
+        datatype: JsValue,
+    ) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let re = Regex::new(r"([\[\(])(\d+)\.\.(\d+)([\)\]])").unwrap();
+        let range = range.as_string().ok_or("range must be a string")?;
+        let cap = re.captures_iter(&range).next().ok_or("range must be in the form of (start..end] or [start..end) etc... () for sticky, [] for normal")?;
+        let start: usize = cap[2].parse().map_err(|_| to_js_err("invalid start"))?;
+        let end: usize = cap[3].parse().map_err(|_| to_js_err("invalid end"))?;
+        let start_sticky = &cap[1] == "(";
+        let end_sticky = &cap[4] == ")";
+        let name = name
+            .as_string()
+            .ok_or("invalid mark name")
+            .map_err(to_js_err)?;
+        let value = self.import_scalar(&value, datatype.as_string())?;
+        self.0
+            .mark(&obj, start, start_sticky, end, end_sticky, &name, value)
+            .map_err(to_js_err)?;
+        Ok(())
+    }
+
+    pub fn spans(&mut self, obj: JsValue) -> Result<JsValue, JsValue> {
+        let obj = self.import(obj)?;
+        let text = self.0.text(&obj).map_err(to_js_err)?;
+        let spans = self.0.spans(&obj).map_err(to_js_err)?;
+        let mut last_pos = 0;
+        let result = Array::new();
+        for s in spans {
+            let marks = Array::new();
+            for m in s.marks {
+                let mark = Array::new();
+                mark.push(&m.0.into());
+                mark.push(&datatype(&m.1).into());
+                mark.push(&ScalarValue(m.1).into());
+                marks.push(&mark.into());
+            }
+            let text_span = &text[last_pos..s.pos]; //.slice(last_pos, s.pos);
+            if text_span.len() > 0 {
+                result.push(&text_span.into());
+            }
+            result.push(&marks);
+            last_pos = s.pos;
+            //let obj = Object::new().into();
+            //js_set(&obj, "pos", s.pos as i32)?;
+            //js_set(&obj, "marks", marks)?;
+            //result.push(&obj.into());
+        }
+        let text_span = &text[last_pos..];
+        if text_span.len() > 0 {
+            result.push(&text_span.into());
+        }
+        Ok(result.into())
+    }
+
+    pub fn save(&mut self) -> Result<Uint8Array, JsValue> {
+        self.0
+            .save()
+            .map(|v| Uint8Array::from(v.as_slice()))
+            .map_err(to_js_err)
+    }
+
+    #[wasm_bindgen(js_name = saveIncremental)]
+    pub fn save_incremental(&mut self) -> Uint8Array {
+        let bytes = self.0.save_incremental();
+        Uint8Array::from(bytes.as_slice())
+    }
+
+    #[wasm_bindgen(js_name = loadIncremental)]
+    pub fn load_incremental(&mut self, data: Uint8Array) -> Result<f64, JsValue> {
+        let data = data.to_vec();
+        let len = self.0.load_incremental(&data).map_err(to_js_err)?;
+        Ok(len as f64)
+    }
+
+    #[wasm_bindgen(js_name = applyChanges)]
+    pub fn apply_changes(&mut self, changes: JsValue) -> Result<(), JsValue> {
+        let changes: Vec<_> = JS(changes).try_into()?;
+        self.0.apply_changes(&changes).map_err(to_js_err)?;
+        Ok(())
+    }
+
+    #[wasm_bindgen(js_name = getChanges)]
+    pub fn get_changes(&mut self, have_deps: JsValue) -> Result<Array, JsValue> {
+        let deps: Vec<_> = JS(have_deps).try_into()?;
+        let changes = self.0.get_changes(&deps);
+        let changes: Array = changes
+            .iter()
+            .map(|c| Uint8Array::from(c.raw_bytes()))
+            .collect();
+        Ok(changes)
+    }
+
+    #[wasm_bindgen(js_name = getChangesAdded)]
+    pub fn get_changes_added(&mut self, other: &Automerge) -> Result<Array, JsValue> {
+        let changes = self.0.get_changes_added(&other.0);
+        let changes: Array = changes
+            .iter()
+            .map(|c| Uint8Array::from(c.raw_bytes()))
+            .collect();
+        Ok(changes)
+    }
+
+    #[wasm_bindgen(js_name = getHeads)]
+    pub fn get_heads(&mut self) -> Array {
+        let heads = self.0.get_heads();
+        let heads: Array = heads
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        heads
+    }
+
+    #[wasm_bindgen(js_name = getActorId)]
+    pub fn get_actor_id(&mut self) -> String {
+        let actor = self.0.get_actor();
+        actor.to_string()
+    }
+
+    #[wasm_bindgen(js_name = getLastLocalChange)]
+    pub fn get_last_local_change(&mut self) -> Result<Option<Uint8Array>, JsValue> {
+        if let Some(change) = self.0.get_last_local_change() {
+            Ok(Some(Uint8Array::from(change.raw_bytes())))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn dump(&self) {
+        self.0.dump()
+    }
+
+    #[wasm_bindgen(js_name = getMissingDeps)]
+    pub fn get_missing_deps(&mut self, heads: Option<Array>) -> Result<Array, JsValue> {
+        let heads = get_heads(heads).unwrap_or_default();
+        let deps = self.0.get_missing_deps(&heads);
+        let deps: Array = deps
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        Ok(deps)
+    }
+
+    #[wasm_bindgen(js_name = receiveSyncMessage)]
+    pub fn receive_sync_message(
+        &mut self,
+        state: &mut SyncState,
+        message: Uint8Array,
+    ) -> Result<(), JsValue> {
+        let message = message.to_vec();
+        let message = am::SyncMessage::decode(message.as_slice()).map_err(to_js_err)?;
+        self.0
+            .receive_sync_message(&mut state.0, message)
+            .map_err(to_js_err)?;
+        Ok(())
+    }
+
+    #[wasm_bindgen(js_name = generateSyncMessage)]
+    pub fn generate_sync_message(&mut self, state: &mut SyncState) -> Result<JsValue, JsValue> {
+        if let Some(message) = self.0.generate_sync_message(&mut state.0) {
+            Ok(Uint8Array::from(message.encode().map_err(to_js_err)?.as_slice()).into())
+        } else {
+            Ok(JsValue::null())
+        }
+    }
+
+    #[wasm_bindgen(js_name = toJS)]
+    pub fn to_js(&self) -> JsValue {
+        map_to_js(&self.0, ROOT)
+    }
+
+    fn import(&self, id: String) -> Result<ObjId, JsValue> {
+        self.0.import(&id).map_err(to_js_err)
+    }
+
+    fn import_prop(&mut self, prop: JsValue) -> Result<Prop, JsValue> {
+        if let Some(s) = prop.as_string() {
+            Ok(s.into())
+        } else if let Some(n) = prop.as_f64() {
+            Ok((n as usize).into())
+        } else {
+            Err(format!("invalid prop {:?}", prop).into())
+        }
+    }
+
+    fn import_scalar(
+        &mut self,
+        value: &JsValue,
+        datatype: Option<String>,
+    ) -> Result<am::ScalarValue, JsValue> {
+        match datatype.as_deref() {
+            Some("boolean") => value
+                .as_bool()
+                .ok_or_else(|| "value must be a bool".into())
+                .map(am::ScalarValue::Boolean),
+            Some("int") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Int(v as i64)),
+            Some("uint") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Uint(v as u64)),
+            Some("f64") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(am::ScalarValue::F64),
+            Some("bytes") => Ok(am::ScalarValue::Bytes(
+                value.clone().dyn_into::<Uint8Array>().unwrap().to_vec(),
+            )),
+            Some("counter") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::counter(v as i64)),
+            Some("timestamp") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Timestamp(v as i64)),
+            /*
+            Some("bytes") => unimplemented!(),
+            Some("cursor") => unimplemented!(),
+            */
+            Some("null") => Ok(am::ScalarValue::Null),
+            Some(_) => Err(format!("unknown datatype {:?}", datatype).into()),
+            None => {
+                if value.is_null() {
+                    Ok(am::ScalarValue::Null)
+                } else if let Some(b) = value.as_bool() {
+                    Ok(am::ScalarValue::Boolean(b))
+                } else if let Some(s) = value.as_string() {
+                    // FIXME - we need to detect str vs int vs float vs bool here :/
+                    Ok(am::ScalarValue::Str(s.into()))
+                } else if let Some(n) = value.as_f64() {
+                    if (n.round() - n).abs() < f64::EPSILON {
+                        Ok(am::ScalarValue::Int(n as i64))
+                    } else {
+                        Ok(am::ScalarValue::F64(n))
+                    }
+                //                } else if let Some(o) = to_objtype(&value) {
+                //                    Ok(o.into())
+                } else if let Ok(d) = value.clone().dyn_into::<js_sys::Date>() {
+                    Ok(am::ScalarValue::Timestamp(d.get_time() as i64))
+                } else if let Ok(o) = &value.clone().dyn_into::<Uint8Array>() {
+                    Ok(am::ScalarValue::Bytes(o.to_vec()))
+                } else {
+                    Err("value is invalid".into())
+                }
+            }
+        }
+    }
+
+    fn import_value(&mut self, value: JsValue, datatype: Option<String>) -> Result<Value, JsValue> {
+        match self.import_scalar(&value, datatype) {
+            Ok(val) => Ok(val.into()),
+            Err(err) => {
+                if let Some(o) = to_objtype(&value) {
+                    Ok(o.into())
+                } else {
+                    Err(err)
+                }
+            }
+        }
+        /*
+        match datatype.as_deref() {
+            Some("boolean") => value
+                .as_bool()
+                .ok_or_else(|| "value must be a bool".into())
+                .map(|v| am::ScalarValue::Boolean(v).into()),
+            Some("int") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Int(v as i64).into()),
+            Some("uint") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Uint(v as u64).into()),
+            Some("f64") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|n| am::ScalarValue::F64(n).into()),
+            Some("bytes") => {
+                Ok(am::ScalarValue::Bytes(value.dyn_into::<Uint8Array>().unwrap().to_vec()).into())
+            }
+            Some("counter") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::counter(v as i64).into()),
+            Some("timestamp") => value
+                .as_f64()
+                .ok_or_else(|| "value must be a number".into())
+                .map(|v| am::ScalarValue::Timestamp(v as i64).into()),
+            Some("null") => Ok(am::ScalarValue::Null.into()),
+            Some(_) => Err(format!("unknown datatype {:?}", datatype).into()),
+            None => {
+                if value.is_null() {
+                    Ok(am::ScalarValue::Null.into())
+                } else if let Some(b) = value.as_bool() {
+                    Ok(am::ScalarValue::Boolean(b).into())
+                } else if let Some(s) = value.as_string() {
+                    // FIXME - we need to detect str vs int vs float vs bool here :/
+                    Ok(am::ScalarValue::Str(s.into()).into())
+                } else if let Some(n) = value.as_f64() {
+                    if (n.round() - n).abs() < f64::EPSILON {
+                        Ok(am::ScalarValue::Int(n as i64).into())
+                    } else {
+                        Ok(am::ScalarValue::F64(n).into())
+                    }
+                } else if let Some(o) = to_objtype(&value) {
+                    Ok(o.into())
+                } else if let Ok(d) = value.clone().dyn_into::<js_sys::Date>() {
+                    Ok(am::ScalarValue::Timestamp(d.get_time() as i64).into())
+                } else if let Ok(o) = &value.dyn_into::<Uint8Array>() {
+                    Ok(am::ScalarValue::Bytes(o.to_vec()).into())
+                } else {
+                    Err("value is invalid".into())
+                }
+            }
+        }
+        */
+    }
+
+}
+
+#[wasm_bindgen(js_name = create)]
+pub fn init(actor: Option<String>) -> Result<Automerge, JsValue> {
+    console_error_panic_hook::set_once();
+    Automerge::new(actor)
+}
+
+#[wasm_bindgen(js_name = loadDoc)]
+pub fn load(data: Uint8Array, actor: Option<String>) -> Result<Automerge, JsValue> {
+    let data = data.to_vec();
+    let mut automerge = am::Automerge::load(&data).map_err(to_js_err)?;
+    if let Some(s) = actor {
+        let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
+        automerge.set_actor(actor)
+    }
+    Ok(Automerge(automerge))
+}
+
+#[wasm_bindgen(js_name = encodeChange)]
+pub fn encode_change(change: JsValue) -> Result<Uint8Array, JsValue> {
+    let change: am::ExpandedChange = change.into_serde().map_err(to_js_err)?;
+    let change: Change = change.into();
+    Ok(Uint8Array::from(change.raw_bytes()))
+}
+
+#[wasm_bindgen(js_name = decodeChange)]
+pub fn decode_change(change: Uint8Array) -> Result<JsValue, JsValue> {
+    let change = Change::from_bytes(change.to_vec()).map_err(to_js_err)?;
+    let change: am::ExpandedChange = change.decode();
+    JsValue::from_serde(&change).map_err(to_js_err)
+}
+
+#[wasm_bindgen(js_name = initSyncState)]
+pub fn init_sync_state() -> SyncState {
+    SyncState(am::SyncState::new())
+}
+
+// this is needed to be compatible with the automerge-js api
+#[wasm_bindgen(js_name = importSyncState)]
+pub fn import_sync_state(state: JsValue) -> Result<SyncState, JsValue> {
+    Ok(SyncState(JS(state).try_into()?))
+}
+
+// this is needed to be compatible with the automerge-js api
+#[wasm_bindgen(js_name = exportSyncState)]
+pub fn export_sync_state(state: SyncState) -> JsValue {
+    JS::from(state.0).into()
+}
+
+#[wasm_bindgen(js_name = encodeSyncMessage)]
+pub fn encode_sync_message(message: JsValue) -> Result<Uint8Array, JsValue> {
+    let heads = js_get(&message, "heads")?.try_into()?;
+    let need = js_get(&message, "need")?.try_into()?;
+    let changes = js_get(&message, "changes")?.try_into()?;
+    let have = js_get(&message, "have")?.try_into()?;
+    Ok(Uint8Array::from(
+        am::SyncMessage {
+            heads,
+            need,
+            have,
+            changes,
+        }
+        .encode()
+        .unwrap()
+        .as_slice(),
+    ))
+}
+
+#[wasm_bindgen(js_name = decodeSyncMessage)]
+pub fn decode_sync_message(msg: Uint8Array) -> Result<JsValue, JsValue> {
+    let data = msg.to_vec();
+    let msg = am::SyncMessage::decode(&data).map_err(to_js_err)?;
+    let heads = AR::from(msg.heads.as_slice());
+    let need = AR::from(msg.need.as_slice());
+    let changes = AR::from(msg.changes.as_slice());
+    let have = AR::from(msg.have.as_slice());
+    let obj = Object::new().into();
+    js_set(&obj, "heads", heads)?;
+    js_set(&obj, "need", need)?;
+    js_set(&obj, "have", have)?;
+    js_set(&obj, "changes", changes)?;
+    Ok(obj)
+}
+
+#[wasm_bindgen(js_name = encodeSyncState)]
+pub fn encode_sync_state(state: SyncState) -> Result<Uint8Array, JsValue> {
+    let state = state.0;
+    Ok(Uint8Array::from(
+        state.encode().map_err(to_js_err)?.as_slice(),
+    ))
+}
+
+#[wasm_bindgen(js_name = decodeSyncState)]
+pub fn decode_sync_state(data: Uint8Array) -> Result<SyncState, JsValue> {
+    SyncState::decode(data)
+}
+
+#[wasm_bindgen(js_name = MAP)]
+pub struct Map {}
+
+#[wasm_bindgen(js_name = LIST)]
+pub struct List {}
+
+#[wasm_bindgen(js_name = TEXT)]
+pub struct Text {}
+
+#[wasm_bindgen(js_name = TABLE)]
+pub struct Table {}
+```

automerge-wasm/index.d.ts

@@ -0,0 +1,222 @@
+
+export type Actor = string;
+export type ObjID = string;
+export type Change = Uint8Array;
+export type SyncMessage = Uint8Array;
+export type Prop = string | number;
+export type Hash = string;
+export type Heads = Hash[];
+export type Value = string | number | boolean | null | Date | Uint8Array
+export type ObjType = string | Array | Object
+export type FullValue =
+  ["str", string] |
+  ["int", number] |
+  ["uint", number] |
+  ["f64", number] |
+  ["boolean", boolean] |
+  ["timestamp", Date] |
+  ["counter", number] |
+  ["bytes", Uint8Array] |
+  ["null", Uint8Array] |
+  ["map", ObjID] |
+  ["list", ObjID] |
+  ["text", ObjID] |
+  ["table", ObjID]
+
+export enum ObjTypeName {
+  list = "list",
+  map = "map",
+  table = "table",
+  text = "text",
+}
+
+export type Datatype =
+  "boolean" |
+  "str" |
+  "int" |
+  "uint" |
+  "f64" |
+  "null" |
+  "timestamp" |
+  "counter" |
+  "bytes" |
+  "map" |
+  "text" |
+  "list";
+
+export type DecodedSyncMessage = {
+  heads: Heads,
+  need: Heads,
+  have: any[]
+  changes: Change[]
+}
+
+export type DecodedChange = {
+  actor: Actor,
+  seq: number
+  startOp: number,
+  time: number,
+  message: string | null,
+  deps: Heads,
+  hash: Hash,
+  ops: Op[]
+}
+
+export type Op = {
+  action: string,
+  obj: ObjID,
+  key: string,
+  value?: string | number | boolean,
+  datatype?: string,
+  pred: string[],
+}
+
+export function create(actor?: Actor): Automerge;
+export function loadDoc(data: Uint8Array, actor?: Actor): Automerge;
+export function encodeChange(change: DecodedChange): Change;
+export function decodeChange(change: Change): DecodedChange;
+export function initSyncState(): SyncState;
+export function encodeSyncMessage(message: DecodedSyncMessage): SyncMessage;
+export function decodeSyncMessage(msg: SyncMessage): DecodedSyncMessage;
+export function encodeSyncState(state: SyncState): Uint8Array;
+export function decodeSyncState(data: Uint8Array): SyncState;
+
+export class Automerge {
+  // change state
+  set(obj: ObjID, prop: Prop, value: Value, datatype?: Datatype): undefined;
+  set_object(obj: ObjID, prop: Prop, value: ObjType): ObjID;
+  insert(obj: ObjID, index: number, value: Value, datatype?: Datatype): undefined;
+  insert_object(obj: ObjID, index: number, value: ObjType): ObjID;
+  push(obj: ObjID, value: Value, datatype?: Datatype): undefined;
+  push_object(obj: ObjID, value: ObjType): ObjID;
+  splice(obj: ObjID, start: number, delete_count: number, text?: string | Array<Value>): ObjID[] | undefined;
+  inc(obj: ObjID, prop: Prop, value: number): void;
+  del(obj: ObjID, prop: Prop): void;
+
+  // returns a single value - if there is a conflict return the winner
+  value(obj: ObjID, prop: any, heads?: Heads): FullValue | null;
+  // return all values in case of a conflict
+  values(obj: ObjID, arg: any, heads?: Heads): FullValue[];
+  keys(obj: ObjID, heads?: Heads): string[];
+  text(obj: ObjID, heads?: Heads): string;
+  length(obj: ObjID, heads?: Heads): number;
+  materialize(obj?: ObjID): any;
+
+  // transactions
+  commit(message?: string, time?: number): Heads;
+  merge(other: Automerge): Heads;
+  getActorId(): Actor;
+  pendingOps(): number;
+  rollback(): number;
+
+  // save and load to local store
+  save(): Uint8Array;
+  saveIncremental(): Uint8Array;
+  loadIncremental(data: Uint8Array): number;
+
+  // sync over network
+  receiveSyncMessage(state: SyncState, message: SyncMessage): void;
+  generateSyncMessage(state: SyncState): SyncMessage | null;
+
+  // low level change functions
+  applyChanges(changes: Change[]): void;
+  getChanges(have_deps: Heads): Change[];
+  getChangesAdded(other: Automerge): Change[];
+  getHeads(): Heads;
+  getLastLocalChange(): Change;
+  getMissingDeps(heads?: Heads): Heads;
+
+  // memory management
+  free(): void;
+  clone(actor?: string): Automerge;
+  fork(actor?: string): Automerge;
+
+  // dump internal state to console.log
+  dump(): void;
+
+  // dump internal state to a JS object
+  toJS(): any;
+}
+
+export class SyncState {
+  free(): void;
+  clone(): SyncState;
+  lastSentHeads: any;
+  sentHashes: any;
+  readonly sharedHeads: any;
+}
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+  readonly memory: WebAssembly.Memory;
+  readonly __wbg_automerge_free: (a: number) => void;
+  readonly automerge_new: (a: number, b: number, c: number) => void;
+  readonly automerge_clone: (a: number, b: number, c: number, d: number) => void;
+  readonly automerge_free: (a: number) => void;
+  readonly automerge_pendingOps: (a: number) => number;
+  readonly automerge_commit: (a: number, b: number, c: number, d: number, e: number) => number;
+  readonly automerge_rollback: (a: number) => number;
+  readonly automerge_keys: (a: number, b: number, c: number, d: number, e: number) => void;
+  readonly automerge_text: (a: number, b: number, c: number, d: number, e: number) => void;
+  readonly automerge_splice: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
+  readonly automerge_push: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
+  readonly automerge_insert: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => void;
+  readonly automerge_set: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => void;
+  readonly automerge_inc: (a: number, b: number, c: number, d: number, e: number, f: number) => void;
+  readonly automerge_value: (a: number, b: number, c: number, d: number, e: number, f: number) => void;
+  readonly automerge_values: (a: number, b: number, c: number, d: number, e: number, f: number) => void;
+  readonly automerge_length: (a: number, b: number, c: number, d: number, e: number) => void;
+  readonly automerge_del: (a: number, b: number, c: number, d: number, e: number) => void;
+  readonly automerge_save: (a: number, b: number) => void;
+  readonly automerge_saveIncremental: (a: number) => number;
+  readonly automerge_loadIncremental: (a: number, b: number, c: number) => void;
+  readonly automerge_applyChanges: (a: number, b: number, c: number) => void;
+  readonly automerge_getChanges: (a: number, b: number, c: number) => void;
+  readonly automerge_getChangesAdded: (a: number, b: number, c: number) => void;
+  readonly automerge_getHeads: (a: number) => number;
+  readonly automerge_getActorId: (a: number, b: number) => void;
+  readonly automerge_getLastLocalChange: (a: number, b: number) => void;
+  readonly automerge_dump: (a: number) => void;
+  readonly automerge_getMissingDeps: (a: number, b: number, c: number) => void;
+  readonly automerge_receiveSyncMessage: (a: number, b: number, c: number, d: number) => void;
+  readonly automerge_generateSyncMessage: (a: number, b: number, c: number) => void;
+  readonly automerge_toJS: (a: number) => number;
+  readonly create: (a: number, b: number, c: number) => void;
+  readonly loadDoc: (a: number, b: number, c: number, d: number) => void;
+  readonly encodeChange: (a: number, b: number) => void;
+  readonly decodeChange: (a: number, b: number) => void;
+  readonly initSyncState: () => number;
+  readonly importSyncState: (a: number, b: number) => void;
+  readonly exportSyncState: (a: number) => number;
+  readonly encodeSyncMessage: (a: number, b: number) => void;
+  readonly decodeSyncMessage: (a: number, b: number) => void;
+  readonly encodeSyncState: (a: number, b: number) => void;
+  readonly decodeSyncState: (a: number, b: number) => void;
+  readonly __wbg_list_free: (a: number) => void;
+  readonly __wbg_map_free: (a: number) => void;
+  readonly __wbg_text_free: (a: number) => void;
+  readonly __wbg_table_free: (a: number) => void;
+  readonly __wbg_syncstate_free: (a: number) => void;
+  readonly syncstate_sharedHeads: (a: number) => number;
+  readonly syncstate_lastSentHeads: (a: number) => number;
+  readonly syncstate_set_lastSentHeads: (a: number, b: number, c: number) => void;
+  readonly syncstate_set_sentHashes: (a: number, b: number, c: number) => void;
+  readonly syncstate_clone: (a: number) => number;
+  readonly __wbindgen_malloc: (a: number) => number;
+  readonly __wbindgen_realloc: (a: number, b: number, c: number) => number;
+  readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
+  readonly __wbindgen_free: (a: number, b: number) => void;
+  readonly __wbindgen_exn_store: (a: number) => void;
+}
+
+/**
+* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+* for everything else, calls `WebAssembly.instantiate` directly.
+*
+* @param {InitInput | Promise<InitInput>} module_or_path
+*
+* @returns {Promise<InitOutput>}
+*/
+
+export default function init (module_or_path?: InitInput | Promise<InitInput>): Promise<InitOutput>;

automerge-wasm/package.json

@@ -0,0 +1,40 @@
+{
+  "collaborators": [
+    "Orion Henry <orion@inkandswitch.com>",
+    "Alex Good <alex@memoryandthought.me>",
+    "Martin Kleppmann"
+  ],
+  "name": "automerge-wasm",
+  "description": "wasm-bindgen bindings to the automerge rust implementation",
+  "version": "0.0.1",
+  "license": "MIT",
+  "files": [
+    "README.md",
+    "LICENSE",
+    "package.json",
+    "automerge_wasm_bg.wasm",
+    "automerge_wasm.js"
+  ],
+  "module": "./pkg/index.js",
+  "main": "./dev/index.js",
+  "scripts": {
+    "build": "rimraf ./dev && wasm-pack build --target nodejs --dev --out-name index -d dev && cp index.d.ts dev",
+    "release": "rimraf ./dev && wasm-pack build --target nodejs --release --out-name index -d dev && cp index.d.ts dev",
+    "pkg": "rimraf ./pkg && wasm-pack build --target web --release --out-name index -d pkg && cp index.d.ts pkg && cd pkg && yarn pack && mv automerge-wasm*tgz ..",
+    "prof": "rimraf ./dev && wasm-pack build --target nodejs --profiling --out-name index -d dev",
+    "test": "yarn build && ts-mocha -p tsconfig.json --type-check --bail --full-trace test/*.ts"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@types/expect": "^24.3.0",
+    "@types/jest": "^27.4.0",
+    "@types/mocha": "^9.1.0",
+    "@types/node": "^17.0.13",
+    "fast-sha256": "^1.3.0",
+    "mocha": "^9.1.3",
+    "pako": "^2.0.4",
+    "rimraf": "^3.0.2",
+    "ts-mocha": "^9.0.2",
+    "typescript": "^4.5.5"
+  }
+}

automerge-wasm/src/interop.rs

@@ -0,0 +1,379 @@
+use automerge as am;
+use automerge::transaction::Transactable;
+use automerge::{Change, ChangeHash, Prop};
+use js_sys::{Array, Object, Reflect, Uint8Array};
+use std::collections::HashSet;
+use std::fmt::Display;
+use unicode_segmentation::UnicodeSegmentation;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsCast;
+
+use crate::{ObjId, ScalarValue, Value};
+
+pub(crate) struct JS(pub JsValue);
+pub(crate) struct AR(pub Array);
+
+impl From<AR> for JsValue {
+    fn from(ar: AR) -> Self {
+        ar.0.into()
+    }
+}
+
+impl From<JS> for JsValue {
+    fn from(js: JS) -> Self {
+        js.0
+    }
+}
+
+impl From<am::sync::State> for JS {
+    fn from(state: am::sync::State) -> Self {
+        let shared_heads: JS = state.shared_heads.into();
+        let last_sent_heads: JS = state.last_sent_heads.into();
+        let their_heads: JS = state.their_heads.into();
+        let their_need: JS = state.their_need.into();
+        let sent_hashes: JS = state.sent_hashes.into();
+        let their_have = if let Some(have) = &state.their_have {
+            JsValue::from(AR::from(have.as_slice()).0)
+        } else {
+            JsValue::null()
+        };
+        let result: JsValue = Object::new().into();
+        // we can unwrap here b/c we made the object and know its not frozen
+        Reflect::set(&result, &"sharedHeads".into(), &shared_heads.0).unwrap();
+        Reflect::set(&result, &"lastSentHeads".into(), &last_sent_heads.0).unwrap();
+        Reflect::set(&result, &"theirHeads".into(), &their_heads.0).unwrap();
+        Reflect::set(&result, &"theirNeed".into(), &their_need.0).unwrap();
+        Reflect::set(&result, &"theirHave".into(), &their_have).unwrap();
+        Reflect::set(&result, &"sentHashes".into(), &sent_hashes.0).unwrap();
+        JS(result)
+    }
+}
+
+impl From<Vec<ChangeHash>> for JS {
+    fn from(heads: Vec<ChangeHash>) -> Self {
+        let heads: Array = heads
+            .iter()
+            .map(|h| JsValue::from_str(&h.to_string()))
+            .collect();
+        JS(heads.into())
+    }
+}
+
+impl From<HashSet<ChangeHash>> for JS {
+    fn from(heads: HashSet<ChangeHash>) -> Self {
+        let result: JsValue = Object::new().into();
+        for key in &heads {
+            Reflect::set(&result, &key.to_string().into(), &true.into()).unwrap();
+        }
+        JS(result)
+    }
+}
+
+impl From<Option<Vec<ChangeHash>>> for JS {
+    fn from(heads: Option<Vec<ChangeHash>>) -> Self {
+        if let Some(v) = heads {
+            let v: Array = v
+                .iter()
+                .map(|h| JsValue::from_str(&h.to_string()))
+                .collect();
+            JS(v.into())
+        } else {
+            JS(JsValue::null())
+        }
+    }
+}
+
+impl TryFrom<JS> for HashSet<ChangeHash> {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let mut result = HashSet::new();
+        for key in Reflect::own_keys(&value.0)?.iter() {
+            if let Some(true) = Reflect::get(&value.0, &key)?.as_bool() {
+                result.insert(key.into_serde().map_err(to_js_err)?);
+            }
+        }
+        Ok(result)
+    }
+}
+
+impl TryFrom<JS> for Vec<ChangeHash> {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let value = value.0.dyn_into::<Array>()?;
+        let value: Result<Vec<ChangeHash>, _> = value.iter().map(|j| j.into_serde()).collect();
+        let value = value.map_err(to_js_err)?;
+        Ok(value)
+    }
+}
+
+impl From<JS> for Option<Vec<ChangeHash>> {
+    fn from(value: JS) -> Self {
+        let value = value.0.dyn_into::<Array>().ok()?;
+        let value: Result<Vec<ChangeHash>, _> = value.iter().map(|j| j.into_serde()).collect();
+        let value = value.ok()?;
+        Some(value)
+    }
+}
+
+impl TryFrom<JS> for Vec<Change> {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let value = value.0.dyn_into::<Array>()?;
+        let changes: Result<Vec<Uint8Array>, _> = value.iter().map(|j| j.dyn_into()).collect();
+        let changes = changes?;
+        let changes: Result<Vec<Change>, _> = changes
+            .iter()
+            .map(|a| Change::try_from(a.to_vec()))
+            .collect();
+        let changes = changes.map_err(to_js_err)?;
+        Ok(changes)
+    }
+}
+
+impl TryFrom<JS> for am::sync::State {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let value = value.0;
+        let shared_heads = js_get(&value, "sharedHeads")?.try_into()?;
+        let last_sent_heads = js_get(&value, "lastSentHeads")?.try_into()?;
+        let their_heads = js_get(&value, "theirHeads")?.into();
+        let their_need = js_get(&value, "theirNeed")?.into();
+        let their_have = js_get(&value, "theirHave")?.try_into()?;
+        let sent_hashes = js_get(&value, "sentHashes")?.try_into()?;
+        Ok(am::sync::State {
+            shared_heads,
+            last_sent_heads,
+            their_heads,
+            their_need,
+            their_have,
+            sent_hashes,
+        })
+    }
+}
+
+impl TryFrom<JS> for Option<Vec<am::sync::Have>> {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        if value.0.is_null() {
+            Ok(None)
+        } else {
+            Ok(Some(value.try_into()?))
+        }
+    }
+}
+
+impl TryFrom<JS> for Vec<am::sync::Have> {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let value = value.0.dyn_into::<Array>()?;
+        let have: Result<Vec<am::sync::Have>, JsValue> = value
+            .iter()
+            .map(|s| {
+                let last_sync = js_get(&s, "lastSync")?.try_into()?;
+                let bloom = js_get(&s, "bloom")?.try_into()?;
+                Ok(am::sync::Have { last_sync, bloom })
+            })
+            .collect();
+        let have = have?;
+        Ok(have)
+    }
+}
+
+impl TryFrom<JS> for am::sync::BloomFilter {
+    type Error = JsValue;
+
+    fn try_from(value: JS) -> Result<Self, Self::Error> {
+        let value: Uint8Array = value.0.dyn_into()?;
+        let value = value.to_vec();
+        let value = value.as_slice().try_into().map_err(to_js_err)?;
+        Ok(value)
+    }
+}
+
+impl From<&[ChangeHash]> for AR {
+    fn from(value: &[ChangeHash]) -> Self {
+        AR(value
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect())
+    }
+}
+
+impl From<&[Change]> for AR {
+    fn from(value: &[Change]) -> Self {
+        let changes: Array = value
+            .iter()
+            .map(|c| Uint8Array::from(c.raw_bytes()))
+            .collect();
+        AR(changes)
+    }
+}
+
+impl From<&[am::sync::Have]> for AR {
+    fn from(value: &[am::sync::Have]) -> Self {
+        AR(value
+            .iter()
+            .map(|have| {
+                let last_sync: Array = have
+                    .last_sync
+                    .iter()
+                    .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+                    .collect();
+                // FIXME - the clone and the unwrap here shouldnt be needed - look at into_bytes()
+                let bloom = Uint8Array::from(have.bloom.to_bytes().as_slice());
+                let obj: JsValue = Object::new().into();
+                // we can unwrap here b/c we created the object and know its not frozen
+                Reflect::set(&obj, &"lastSync".into(), &last_sync.into()).unwrap();
+                Reflect::set(&obj, &"bloom".into(), &bloom.into()).unwrap();
+                obj
+            })
+            .collect())
+    }
+}
+
+pub(crate) fn to_js_err<T: Display>(err: T) -> JsValue {
+    js_sys::Error::new(&std::format!("{}", err)).into()
+}
+
+pub(crate) fn js_get<J: Into<JsValue>>(obj: J, prop: &str) -> Result<JS, JsValue> {
+    Ok(JS(Reflect::get(&obj.into(), &prop.into())?))
+}
+
+pub(crate) fn js_set<V: Into<JsValue>>(obj: &JsValue, prop: &str, val: V) -> Result<bool, JsValue> {
+    Reflect::set(obj, &prop.into(), &val.into())
+}
+
+pub(crate) fn to_prop(p: JsValue) -> Result<Prop, JsValue> {
+    if let Some(s) = p.as_string() {
+        Ok(Prop::Map(s))
+    } else if let Some(n) = p.as_f64() {
+        Ok(Prop::Seq(n as usize))
+    } else {
+        Err(to_js_err("prop must me a string or number"))
+    }
+}
+
+pub(crate) fn to_objtype(
+    value: &JsValue,
+    datatype: &Option<String>,
+) -> Option<(am::ObjType, Vec<(Prop, JsValue)>)> {
+    match datatype.as_deref() {
+        Some("map") => {
+            let map = value.clone().dyn_into::<js_sys::Object>().ok()?;
+            // FIXME unwrap
+            let map = js_sys::Object::keys(&map)
+                .iter()
+                .zip(js_sys::Object::values(&map).iter())
+                .map(|(key, val)| (key.as_string().unwrap().into(), val))
+                .collect();
+            Some((am::ObjType::Map, map))
+        }
+        Some("list") => {
+            let list = value.clone().dyn_into::<js_sys::Array>().ok()?;
+            let list = list
+                .iter()
+                .enumerate()
+                .map(|(i, e)| (i.into(), e))
+                .collect();
+            Some((am::ObjType::List, list))
+        }
+        Some("text") => {
+            let text = value.as_string()?;
+            let text = text
+                .graphemes(true)
+                .enumerate()
+                .map(|(i, ch)| (i.into(), ch.into()))
+                .collect();
+            Some((am::ObjType::Text, text))
+        }
+        Some(_) => None,
+        None => {
+            if let Ok(list) = value.clone().dyn_into::<js_sys::Array>() {
+                let list = list
+                    .iter()
+                    .enumerate()
+                    .map(|(i, e)| (i.into(), e))
+                    .collect();
+                Some((am::ObjType::List, list))
+            } else if let Ok(map) = value.clone().dyn_into::<js_sys::Object>() {
+                // FIXME unwrap
+                let map = js_sys::Object::keys(&map)
+                    .iter()
+                    .zip(js_sys::Object::values(&map).iter())
+                    .map(|(key, val)| (key.as_string().unwrap().into(), val))
+                    .collect();
+                Some((am::ObjType::Map, map))
+            } else if let Some(text) = value.as_string() {
+                let text = text
+                    .graphemes(true)
+                    .enumerate()
+                    .map(|(i, ch)| (i.into(), ch.into()))
+                    .collect();
+                Some((am::ObjType::Text, text))
+            } else {
+                None
+            }
+        }
+    }
+}
+
+pub(crate) fn get_heads(heads: Option<Array>) -> Option<Vec<ChangeHash>> {
+    let heads = heads?;
+    let heads: Result<Vec<ChangeHash>, _> = heads.iter().map(|j| j.into_serde()).collect();
+    heads.ok()
+}
+
+pub(crate) fn map_to_js(doc: &am::AutoCommit, obj: &ObjId) -> JsValue {
+    let keys = doc.keys(obj);
+    let map = Object::new();
+    for k in keys {
+        let val = doc.value(obj, &k);
+        match val {
+            Ok(Some((Value::Object(o), exid)))
+                if o == am::ObjType::Map || o == am::ObjType::Table =>
+            {
+                Reflect::set(&map, &k.into(), &map_to_js(doc, &exid)).unwrap();
+            }
+            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::List => {
+                Reflect::set(&map, &k.into(), &list_to_js(doc, &exid)).unwrap();
+            }
+            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::Text => {
+                Reflect::set(&map, &k.into(), &doc.text(&exid).unwrap().into()).unwrap();
+            }
+            Ok(Some((Value::Scalar(v), _))) => {
+                Reflect::set(&map, &k.into(), &ScalarValue(v).into()).unwrap();
+            }
+            _ => (),
+        };
+    }
+    map.into()
+}
+
+pub(crate) fn list_to_js(doc: &am::AutoCommit, obj: &ObjId) -> JsValue {
+    let len = doc.length(obj);
+    let array = Array::new();
+    for i in 0..len {
+        let val = doc.value(obj, i as usize);
+        match val {
+            Ok(Some((Value::Object(o), exid)))
+                if o == am::ObjType::Map || o == am::ObjType::Table =>
+            {
+                array.push(&map_to_js(doc, &exid));
+            }
+            Ok(Some((Value::Object(_), exid))) => {
+                array.push(&list_to_js(doc, &exid));
+            }
+            Ok(Some((Value::Scalar(v), _))) => {
+                array.push(&ScalarValue(v).into());
+            }
+            _ => (),
+        };
+    }
+    array.into()
+}

automerge-wasm/src/lib.rs

@@ -0,0 +1,683 @@
+#![allow(clippy::unused_unit)]
+use am::transaction::CommitOptions;
+use am::transaction::Transactable;
+use automerge as am;
+use automerge::{Change, ObjId, Prop, Value, ROOT};
+use js_sys::{Array, Object, Uint8Array};
+use std::convert::TryInto;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsCast;
+
+mod interop;
+mod sync;
+mod value;
+
+use interop::{
+    get_heads, js_get, js_set, list_to_js, map_to_js, to_js_err, to_objtype, to_prop, AR, JS,
+};
+use sync::SyncState;
+use value::{datatype, ScalarValue};
+
+#[allow(unused_macros)]
+macro_rules! log {
+    ( $( $t:tt )* ) => {
+          web_sys::console::log_1(&format!( $( $t )* ).into());
+    };
+}
+
+#[cfg(feature = "wee_alloc")]
+#[global_allocator]
+static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
+
+#[wasm_bindgen]
+#[derive(Debug)]
+pub struct Automerge(automerge::AutoCommit);
+
+#[wasm_bindgen]
+impl Automerge {
+    pub fn new(actor: Option<String>) -> Result<Automerge, JsValue> {
+        let mut automerge = automerge::AutoCommit::new();
+        if let Some(a) = actor {
+            let a = automerge::ActorId::from(hex::decode(a).map_err(to_js_err)?.to_vec());
+            automerge.set_actor(a);
+        }
+        Ok(Automerge(automerge))
+    }
+
+    #[allow(clippy::should_implement_trait)]
+    pub fn clone(&mut self, actor: Option<String>) -> Result<Automerge, JsValue> {
+        if self.0.pending_ops() > 0 {
+            self.0.commit();
+        }
+        let mut automerge = Automerge(self.0.clone());
+        if let Some(s) = actor {
+            let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
+            automerge.0.set_actor(actor);
+        }
+        Ok(automerge)
+    }
+
+    #[allow(clippy::should_implement_trait)]
+    pub fn fork(&mut self, actor: Option<String>) -> Result<Automerge, JsValue> {
+        let mut automerge = Automerge(self.0.fork());
+        if let Some(s) = actor {
+            let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
+            automerge.0.set_actor(actor);
+        }
+        Ok(automerge)
+    }
+
+    pub fn free(self) {}
+
+    #[wasm_bindgen(js_name = pendingOps)]
+    pub fn pending_ops(&self) -> JsValue {
+        (self.0.pending_ops() as u32).into()
+    }
+
+    pub fn commit(&mut self, message: Option<String>, time: Option<f64>) -> JsValue {
+        let mut commit_opts = CommitOptions::default();
+        if let Some(message) = message {
+            commit_opts.set_message(message);
+        }
+        if let Some(time) = time {
+            commit_opts.set_time(time as i64);
+        }
+        let hash = self.0.commit_with(commit_opts);
+        let result = Array::new();
+        result.push(&JsValue::from_str(&hex::encode(&hash.0)));
+        result.into()
+    }
+
+    pub fn merge(&mut self, other: &mut Automerge) -> Result<Array, JsValue> {
+        let heads = self.0.merge(&mut other.0)?;
+        let heads: Array = heads
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        Ok(heads)
+    }
+
+    pub fn rollback(&mut self) -> f64 {
+        self.0.rollback() as f64
+    }
+
+    pub fn keys(&mut self, obj: JsValue, heads: Option<Array>) -> Result<Array, JsValue> {
+        let obj = self.import(obj)?;
+        let result = if let Some(heads) = get_heads(heads) {
+            self.0
+                .keys_at(&obj, &heads)
+                .map(|s| JsValue::from_str(&s))
+                .collect()
+        } else {
+            self.0.keys(&obj).map(|s| JsValue::from_str(&s)).collect()
+        };
+        Ok(result)
+    }
+
+    pub fn text(&mut self, obj: JsValue, heads: Option<Array>) -> Result<String, JsValue> {
+        let obj = self.import(obj)?;
+        if let Some(heads) = get_heads(heads) {
+            Ok(self.0.text_at(&obj, &heads)?)
+        } else {
+            Ok(self.0.text(&obj)?)
+        }
+    }
+
+    pub fn splice(
+        &mut self,
+        obj: JsValue,
+        start: f64,
+        delete_count: f64,
+        text: JsValue,
+    ) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let start = start as usize;
+        let delete_count = delete_count as usize;
+        let mut vals = vec![];
+        if let Some(t) = text.as_string() {
+            self.0.splice_text(&obj, start, delete_count, &t)?;
+        } else {
+            if let Ok(array) = text.dyn_into::<Array>() {
+                for i in array.iter() {
+                    let value = self
+                        .import_scalar(&i, &None)
+                        .ok_or_else(|| to_js_err("expected scalar"))?;
+                    vals.push(value);
+                }
+            }
+            self.0.splice(&obj, start, delete_count, vals.into_iter())?;
+        }
+        Ok(())
+    }
+
+    pub fn push(&mut self, obj: JsValue, value: JsValue, datatype: JsValue) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let value = self
+            .import_scalar(&value, &datatype.as_string())
+            .ok_or_else(|| to_js_err("invalid scalar value"))?;
+        let index = self.0.length(&obj);
+        self.0.insert(&obj, index, value)?;
+        Ok(())
+    }
+
+    pub fn push_object(&mut self, obj: JsValue, value: JsValue) -> Result<Option<String>, JsValue> {
+        let obj = self.import(obj)?;
+        let (value, subvals) =
+            to_objtype(&value, &None).ok_or_else(|| to_js_err("expected object"))?;
+        let index = self.0.length(&obj);
+        let opid = self.0.insert_object(&obj, index, value)?;
+        self.subset(&opid, subvals)?;
+        Ok(opid.to_string().into())
+    }
+
+    pub fn insert(
+        &mut self,
+        obj: JsValue,
+        index: f64,
+        value: JsValue,
+        datatype: JsValue,
+    ) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let index = index as f64;
+        let value = self
+            .import_scalar(&value, &datatype.as_string())
+            .ok_or_else(|| to_js_err("expected scalar value"))?;
+        self.0.insert(&obj, index as usize, value)?;
+        Ok(())
+    }
+
+    pub fn insert_object(
+        &mut self,
+        obj: JsValue,
+        index: f64,
+        value: JsValue,
+    ) -> Result<Option<String>, JsValue> {
+        let obj = self.import(obj)?;
+        let index = index as f64;
+        let (value, subvals) =
+            to_objtype(&value, &None).ok_or_else(|| to_js_err("expected object"))?;
+        let opid = self.0.insert_object(&obj, index as usize, value)?;
+        self.subset(&opid, subvals)?;
+        Ok(opid.to_string().into())
+    }
+
+    pub fn set(
+        &mut self,
+        obj: JsValue,
+        prop: JsValue,
+        value: JsValue,
+        datatype: JsValue,
+    ) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let value = self
+            .import_scalar(&value, &datatype.as_string())
+            .ok_or_else(|| to_js_err("expected scalar value"))?;
+        self.0.set(&obj, prop, value)?;
+        Ok(())
+    }
+
+    pub fn set_object(
+        &mut self,
+        obj: JsValue,
+        prop: JsValue,
+        value: JsValue,
+    ) -> Result<JsValue, JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let (value, subvals) =
+            to_objtype(&value, &None).ok_or_else(|| to_js_err("expected object"))?;
+        let opid = self.0.set_object(&obj, prop, value)?;
+        self.subset(&opid, subvals)?;
+        Ok(opid.to_string().into())
+    }
+
+    fn subset(&mut self, obj: &am::ObjId, vals: Vec<(am::Prop, JsValue)>) -> Result<(), JsValue> {
+        for (p, v) in vals {
+            let (value, subvals) = self.import_value(&v, None)?;
+            //let opid = self.0.set(id, p, value)?;
+            let opid = match (p, value) {
+                (Prop::Map(s), Value::Object(objtype)) => Some(self.0.set_object(obj, s, objtype)?),
+                (Prop::Map(s), Value::Scalar(scalar)) => {
+                    self.0.set(obj, s, scalar)?;
+                    None
+                }
+                (Prop::Seq(i), Value::Object(objtype)) => {
+                    Some(self.0.insert_object(obj, i, objtype)?)
+                }
+                (Prop::Seq(i), Value::Scalar(scalar)) => {
+                    self.0.insert(obj, i, scalar)?;
+                    None
+                }
+            };
+            if let Some(opid) = opid {
+                self.subset(&opid, subvals)?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn inc(&mut self, obj: JsValue, prop: JsValue, value: JsValue) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let prop = self.import_prop(prop)?;
+        let value: f64 = value
+            .as_f64()
+            .ok_or_else(|| to_js_err("inc needs a numberic value"))?;
+        self.0.inc(&obj, prop, value as i64)?;
+        Ok(())
+    }
+
+    pub fn value(
+        &mut self,
+        obj: JsValue,
+        prop: JsValue,
+        heads: Option<Array>,
+    ) -> Result<Option<Array>, JsValue> {
+        let obj = self.import(obj)?;
+        let result = Array::new();
+        let prop = to_prop(prop);
+        let heads = get_heads(heads);
+        if let Ok(prop) = prop {
+            let value = if let Some(h) = heads {
+                self.0.value_at(&obj, prop, &h)?
+            } else {
+                self.0.value(&obj, prop)?
+            };
+            match value {
+                Some((Value::Object(obj_type), obj_id)) => {
+                    result.push(&obj_type.to_string().into());
+                    result.push(&obj_id.to_string().into());
+                    Ok(Some(result))
+                }
+                Some((Value::Scalar(value), _)) => {
+                    result.push(&datatype(&value).into());
+                    result.push(&ScalarValue(value).into());
+                    Ok(Some(result))
+                }
+                None => Ok(None),
+            }
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn values(
+        &mut self,
+        obj: JsValue,
+        arg: JsValue,
+        heads: Option<Array>,
+    ) -> Result<Array, JsValue> {
+        let obj = self.import(obj)?;
+        let result = Array::new();
+        let prop = to_prop(arg);
+        if let Ok(prop) = prop {
+            let values = if let Some(heads) = get_heads(heads) {
+                self.0.values_at(&obj, prop, &heads)
+            } else {
+                self.0.values(&obj, prop)
+            }
+            .map_err(to_js_err)?;
+            for value in values {
+                match value {
+                    (Value::Object(obj_type), obj_id) => {
+                        let sub = Array::new();
+                        sub.push(&obj_type.to_string().into());
+                        sub.push(&obj_id.to_string().into());
+                        result.push(&sub.into());
+                    }
+                    (Value::Scalar(value), id) => {
+                        let sub = Array::new();
+                        sub.push(&datatype(&value).into());
+                        sub.push(&ScalarValue(value).into());
+                        sub.push(&id.to_string().into());
+                        result.push(&sub.into());
+                    }
+                }
+            }
+        }
+        Ok(result)
+    }
+
+    pub fn length(&mut self, obj: JsValue, heads: Option<Array>) -> Result<f64, JsValue> {
+        let obj = self.import(obj)?;
+        if let Some(heads) = get_heads(heads) {
+            Ok(self.0.length_at(&obj, &heads) as f64)
+        } else {
+            Ok(self.0.length(&obj) as f64)
+        }
+    }
+
+    pub fn del(&mut self, obj: JsValue, prop: JsValue) -> Result<(), JsValue> {
+        let obj = self.import(obj)?;
+        let prop = to_prop(prop)?;
+        self.0.del(&obj, prop).map_err(to_js_err)?;
+        Ok(())
+    }
+
+    pub fn save(&mut self) -> Uint8Array {
+        Uint8Array::from(self.0.save().as_slice())
+    }
+
+    #[wasm_bindgen(js_name = saveIncremental)]
+    pub fn save_incremental(&mut self) -> Uint8Array {
+        let bytes = self.0.save_incremental();
+        Uint8Array::from(bytes.as_slice())
+    }
+
+    #[wasm_bindgen(js_name = loadIncremental)]
+    pub fn load_incremental(&mut self, data: Uint8Array) -> Result<f64, JsValue> {
+        let data = data.to_vec();
+        let len = self.0.load_incremental(&data).map_err(to_js_err)?;
+        Ok(len as f64)
+    }
+
+    #[wasm_bindgen(js_name = applyChanges)]
+    pub fn apply_changes(&mut self, changes: JsValue) -> Result<(), JsValue> {
+        let changes: Vec<_> = JS(changes).try_into()?;
+        self.0.apply_changes(changes).map_err(to_js_err)?;
+        Ok(())
+    }
+
+    #[wasm_bindgen(js_name = getChanges)]
+    pub fn get_changes(&mut self, have_deps: JsValue) -> Result<Array, JsValue> {
+        let deps: Vec<_> = JS(have_deps).try_into()?;
+        let changes = self.0.get_changes(&deps);
+        let changes: Array = changes
+            .iter()
+            .map(|c| Uint8Array::from(c.raw_bytes()))
+            .collect();
+        Ok(changes)
+    }
+
+    #[wasm_bindgen(js_name = getChangesAdded)]
+    pub fn get_changes_added(&mut self, other: &mut Automerge) -> Result<Array, JsValue> {
+        let changes = self.0.get_changes_added(&mut other.0);
+        let changes: Array = changes
+            .iter()
+            .map(|c| Uint8Array::from(c.raw_bytes()))
+            .collect();
+        Ok(changes)
+    }
+
+    #[wasm_bindgen(js_name = getHeads)]
+    pub fn get_heads(&mut self) -> Array {
+        let heads = self.0.get_heads();
+        let heads: Array = heads
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        heads
+    }
+
+    #[wasm_bindgen(js_name = getActorId)]
+    pub fn get_actor_id(&mut self) -> String {
+        let actor = self.0.get_actor();
+        actor.to_string()
+    }
+
+    #[wasm_bindgen(js_name = getLastLocalChange)]
+    pub fn get_last_local_change(&mut self) -> Result<Uint8Array, JsValue> {
+        if let Some(change) = self.0.get_last_local_change() {
+            Ok(Uint8Array::from(change.raw_bytes()))
+        } else {
+            Err(to_js_err("no local changes"))
+        }
+    }
+
+    pub fn dump(&self) {
+        self.0.dump()
+    }
+
+    #[wasm_bindgen(js_name = getMissingDeps)]
+    pub fn get_missing_deps(&mut self, heads: Option<Array>) -> Result<Array, JsValue> {
+        let heads = get_heads(heads).unwrap_or_default();
+        let deps = self.0.get_missing_deps(&heads);
+        let deps: Array = deps
+            .iter()
+            .map(|h| JsValue::from_str(&hex::encode(&h.0)))
+            .collect();
+        Ok(deps)
+    }
+
+    #[wasm_bindgen(js_name = receiveSyncMessage)]
+    pub fn receive_sync_message(
+        &mut self,
+        state: &mut SyncState,
+        message: Uint8Array,
+    ) -> Result<(), JsValue> {
+        let message = message.to_vec();
+        let message = am::sync::Message::decode(message.as_slice()).map_err(to_js_err)?;
+        self.0
+            .receive_sync_message(&mut state.0, message)
+            .map_err(to_js_err)?;
+        Ok(())
+    }
+
+    #[wasm_bindgen(js_name = generateSyncMessage)]
+    pub fn generate_sync_message(&mut self, state: &mut SyncState) -> Result<JsValue, JsValue> {
+        if let Some(message) = self.0.generate_sync_message(&mut state.0) {
+            Ok(Uint8Array::from(message.encode().as_slice()).into())
+        } else {
+            Ok(JsValue::null())
+        }
+    }
+
+    #[wasm_bindgen(js_name = toJS)]
+    pub fn to_js(&self) -> JsValue {
+        map_to_js(&self.0, &ROOT)
+    }
+
+    pub fn materialize(&self, obj: JsValue) -> Result<JsValue, JsValue> {
+        let obj = self.import(obj).unwrap_or(ROOT);
+        match self.0.object_type(&obj) {
+            Some(am::ObjType::Map) => Ok(map_to_js(&self.0, &obj)),
+            Some(am::ObjType::List) => Ok(list_to_js(&self.0, &obj)),
+            Some(am::ObjType::Text) => Ok(self.0.text(&obj)?.into()),
+            Some(am::ObjType::Table) => Ok(map_to_js(&self.0, &obj)),
+            None => Err(to_js_err(format!("invalid obj {}", obj))),
+        }
+    }
+
+    fn import(&self, id: JsValue) -> Result<ObjId, JsValue> {
+        if let Some(s) = id.as_string() {
+            if let Some(post) = s.strip_prefix('/') {
+                let mut obj = ROOT;
+                let mut is_map = true;
+                let parts = post.split('/');
+                for prop in parts {
+                    if prop.is_empty() {
+                        break;
+                    }
+                    let val = if is_map {
+                        self.0.value(obj, prop)?
+                    } else {
+                        self.0.value(obj, am::Prop::Seq(prop.parse().unwrap()))?
+                    };
+                    match val {
+                        Some((am::Value::Object(am::ObjType::Map), id)) => {
+                            is_map = true;
+                            obj = id;
+                        }
+                        Some((am::Value::Object(am::ObjType::Table), id)) => {
+                            is_map = true;
+                            obj = id;
+                        }
+                        Some((am::Value::Object(_), id)) => {
+                            is_map = false;
+                            obj = id;
+                        }
+                        None => return Err(to_js_err(format!("invalid path '{}'", s))),
+                        _ => return Err(to_js_err(format!("path '{}' is not an object", s))),
+                    };
+                }
+                Ok(obj)
+            } else {
+                Ok(self.0.import(&s)?)
+            }
+        } else {
+            Err(to_js_err("invalid objid"))
+        }
+    }
+
+    fn import_prop(&mut self, prop: JsValue) -> Result<Prop, JsValue> {
+        if let Some(s) = prop.as_string() {
+            Ok(s.into())
+        } else if let Some(n) = prop.as_f64() {
+            Ok((n as usize).into())
+        } else {
+            Err(to_js_err(format!("invalid prop {:?}", prop)))
+        }
+    }
+
+    fn import_scalar(
+        &mut self,
+        value: &JsValue,
+        datatype: &Option<String>,
+    ) -> Option<am::ScalarValue> {
+        match datatype.as_deref() {
+            Some("boolean") => value.as_bool().map(am::ScalarValue::Boolean),
+            Some("int") => value.as_f64().map(|v| am::ScalarValue::Int(v as i64)),
+            Some("uint") => value.as_f64().map(|v| am::ScalarValue::Uint(v as u64)),
+            Some("f64") => value.as_f64().map(am::ScalarValue::F64),
+            Some("bytes") => Some(am::ScalarValue::Bytes(
+                value.clone().dyn_into::<Uint8Array>().unwrap().to_vec(),
+            )),
+            Some("counter") => value.as_f64().map(|v| am::ScalarValue::counter(v as i64)),
+            Some("timestamp") => value.as_f64().map(|v| am::ScalarValue::Timestamp(v as i64)),
+            Some("null") => Some(am::ScalarValue::Null),
+            Some(_) => None,
+            None => {
+                if value.is_null() {
+                    Some(am::ScalarValue::Null)
+                } else if let Some(b) = value.as_bool() {
+                    Some(am::ScalarValue::Boolean(b))
+                } else if let Some(s) = value.as_string() {
+                    Some(am::ScalarValue::Str(s.into()))
+                } else if let Some(n) = value.as_f64() {
+                    if (n.round() - n).abs() < f64::EPSILON {
+                        Some(am::ScalarValue::Int(n as i64))
+                    } else {
+                        Some(am::ScalarValue::F64(n))
+                    }
+                } else if let Ok(d) = value.clone().dyn_into::<js_sys::Date>() {
+                    Some(am::ScalarValue::Timestamp(d.get_time() as i64))
+                } else if let Ok(o) = &value.clone().dyn_into::<Uint8Array>() {
+                    Some(am::ScalarValue::Bytes(o.to_vec()))
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    fn import_value(
+        &mut self,
+        value: &JsValue,
+        datatype: Option<String>,
+    ) -> Result<(Value, Vec<(Prop, JsValue)>), JsValue> {
+        match self.import_scalar(value, &datatype) {
+            Some(val) => Ok((val.into(), vec![])),
+            None => {
+                if let Some((o, subvals)) = to_objtype(value, &datatype) {
+                    Ok((o.into(), subvals))
+                } else {
+                    web_sys::console::log_2(&"Invalid value".into(), value);
+                    Err(to_js_err("invalid value"))
+                }
+            }
+        }
+    }
+}
+
+#[wasm_bindgen(js_name = create)]
+pub fn init(actor: Option<String>) -> Result<Automerge, JsValue> {
+    console_error_panic_hook::set_once();
+    Automerge::new(actor)
+}
+
+#[wasm_bindgen(js_name = loadDoc)]
+pub fn load(data: Uint8Array, actor: Option<String>) -> Result<Automerge, JsValue> {
+    let data = data.to_vec();
+    let mut automerge = am::AutoCommit::load(&data).map_err(to_js_err)?;
+    if let Some(s) = actor {
+        let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
+        automerge.set_actor(actor);
+    }
+    Ok(Automerge(automerge))
+}
+
+#[wasm_bindgen(js_name = encodeChange)]
+pub fn encode_change(change: JsValue) -> Result<Uint8Array, JsValue> {
+    let change: am::ExpandedChange = change.into_serde().map_err(to_js_err)?;
+    let change: Change = change.into();
+    Ok(Uint8Array::from(change.raw_bytes()))
+}
+
+#[wasm_bindgen(js_name = decodeChange)]
+pub fn decode_change(change: Uint8Array) -> Result<JsValue, JsValue> {
+    let change = Change::from_bytes(change.to_vec()).map_err(to_js_err)?;
+    let change: am::ExpandedChange = change.decode();
+    JsValue::from_serde(&change).map_err(to_js_err)
+}
+
+#[wasm_bindgen(js_name = initSyncState)]
+pub fn init_sync_state() -> SyncState {
+    SyncState(am::sync::State::new())
+}
+
+// this is needed to be compatible with the automerge-js api
+#[wasm_bindgen(js_name = importSyncState)]
+pub fn import_sync_state(state: JsValue) -> Result<SyncState, JsValue> {
+    Ok(SyncState(JS(state).try_into()?))
+}
+
+// this is needed to be compatible with the automerge-js api
+#[wasm_bindgen(js_name = exportSyncState)]
+pub fn export_sync_state(state: SyncState) -> JsValue {
+    JS::from(state.0).into()
+}
+
+#[wasm_bindgen(js_name = encodeSyncMessage)]
+pub fn encode_sync_message(message: JsValue) -> Result<Uint8Array, JsValue> {
+    let heads = js_get(&message, "heads")?.try_into()?;
+    let need = js_get(&message, "need")?.try_into()?;
+    let changes = js_get(&message, "changes")?.try_into()?;
+    let have = js_get(&message, "have")?.try_into()?;
+    Ok(Uint8Array::from(
+        am::sync::Message {
+            heads,
+            need,
+            have,
+            changes,
+        }
+        .encode()
+        .as_slice(),
+    ))
+}
+
+#[wasm_bindgen(js_name = decodeSyncMessage)]
+pub fn decode_sync_message(msg: Uint8Array) -> Result<JsValue, JsValue> {
+    let data = msg.to_vec();
+    let msg = am::sync::Message::decode(&data).map_err(to_js_err)?;
+    let heads = AR::from(msg.heads.as_slice());
+    let need = AR::from(msg.need.as_slice());
+    let changes = AR::from(msg.changes.as_slice());
+    let have = AR::from(msg.have.as_slice());
+    let obj = Object::new().into();
+    js_set(&obj, "heads", heads)?;
+    js_set(&obj, "need", need)?;
+    js_set(&obj, "have", have)?;
+    js_set(&obj, "changes", changes)?;
+    Ok(obj)
+}
+
+#[wasm_bindgen(js_name = encodeSyncState)]
+pub fn encode_sync_state(state: SyncState) -> Result<Uint8Array, JsValue> {
+    let state = state.0;
+    Ok(Uint8Array::from(state.encode().as_slice()))
+}
+
+#[wasm_bindgen(js_name = decodeSyncState)]
+pub fn decode_sync_state(data: Uint8Array) -> Result<SyncState, JsValue> {
+    SyncState::decode(data)
+}

automerge-wasm/src/sync.rs

@@ -1,11 +1,11 @@
 use automerge as am;
 use automerge::ChangeHash;
 use js_sys::Uint8Array;
-use std::collections::{BTreeSet, HashMap};
+use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
 use wasm_bindgen::prelude::*;
 
-use crate::interop::{self, to_js_err, AR, JS};
+use crate::interop::{to_js_err, AR, JS};
 
 #[wasm_bindgen]
 #[derive(Debug)]
@@ -24,10 +24,7 @@ impl SyncState {
     }
 
     #[wasm_bindgen(setter, js_name = lastSentHeads)]
-    pub fn set_last_sent_heads(
-        &mut self,
-        heads: JsValue,
-    ) -> Result<(), interop::error::BadChangeHashes> {
+    pub fn set_last_sent_heads(&mut self, heads: JsValue) -> Result<(), JsValue> {
         let heads: Vec<ChangeHash> = JS(heads).try_into()?;
         self.0.last_sent_heads = heads;
         Ok(())
@@ -35,9 +32,8 @@ impl SyncState {
 
     #[wasm_bindgen(setter, js_name = sentHashes)]
     pub fn set_sent_hashes(&mut self, hashes: JsValue) -> Result<(), JsValue> {
-        let hashes_map: HashMap<ChangeHash, bool> =
-            serde_wasm_bindgen::from_value(hashes).map_err(to_js_err)?;
-        let hashes_set: BTreeSet<ChangeHash> = hashes_map.keys().cloned().collect();
+        let hashes_map: HashMap<ChangeHash, bool> = hashes.into_serde().map_err(to_js_err)?;
+        let hashes_set: HashSet<ChangeHash> = hashes_map.keys().cloned().collect();
         self.0.sent_hashes = hashes_set;
         Ok(())
     }
@@ -47,19 +43,10 @@ impl SyncState {
         SyncState(self.0.clone())
     }
 
-    pub(crate) fn decode(data: Uint8Array) -> Result<SyncState, DecodeSyncStateErr> {
+    pub(crate) fn decode(data: Uint8Array) -> Result<SyncState, JsValue> {
         let data = data.to_vec();
-        let s = am::sync::State::decode(&data)?;
+        let s = am::sync::State::decode(&data);
+        let s = s.map_err(to_js_err)?;
         Ok(SyncState(s))
     }
 }
-
-#[derive(Debug, thiserror::Error)]
-#[error(transparent)]
-pub struct DecodeSyncStateErr(#[from] automerge::sync::DecodeStateError);
-
-impl From<DecodeSyncStateErr> for JsValue {
-    fn from(e: DecodeSyncStateErr) -> Self {
-        JsValue::from(e.to_string())
-    }
-}

automerge-wasm/src/value.rs

@@ -0,0 +1,36 @@
+use automerge as am;
+use js_sys::Uint8Array;
+use wasm_bindgen::prelude::*;
+
+#[derive(Debug)]
+pub struct ScalarValue(pub(crate) am::ScalarValue);
+
+impl From<ScalarValue> for JsValue {
+    fn from(val: ScalarValue) -> Self {
+        match &val.0 {
+            am::ScalarValue::Bytes(v) => Uint8Array::from(v.as_slice()).into(),
+            am::ScalarValue::Str(v) => v.to_string().into(),
+            am::ScalarValue::Int(v) => (*v as f64).into(),
+            am::ScalarValue::Uint(v) => (*v as f64).into(),
+            am::ScalarValue::F64(v) => (*v).into(),
+            am::ScalarValue::Counter(v) => (f64::from(v)).into(),
+            am::ScalarValue::Timestamp(v) => js_sys::Date::new(&(*v as f64).into()).into(),
+            am::ScalarValue::Boolean(v) => (*v).into(),
+            am::ScalarValue::Null => JsValue::null(),
+        }
+    }
+}
+
+pub(crate) fn datatype(s: &am::ScalarValue) -> String {
+    match s {
+        am::ScalarValue::Bytes(_) => "bytes".into(),
+        am::ScalarValue::Str(_) => "str".into(),
+        am::ScalarValue::Int(_) => "int".into(),
+        am::ScalarValue::Uint(_) => "uint".into(),
+        am::ScalarValue::F64(_) => "f64".into(),
+        am::ScalarValue::Counter(_) => "counter".into(),
+        am::ScalarValue::Timestamp(_) => "timestamp".into(),
+        am::ScalarValue::Boolean(_) => "boolean".into(),
+        am::ScalarValue::Null => "null".into(),
+    }
+}

automerge-wasm/test/helpers/common.js

@@ -1,5 +1,5 @@
 function isObject(obj) {
-  return typeof obj === "object" && obj !== null
+  return typeof obj === 'object' && obj !== null
 }
 
 /**
@@ -20,11 +20,11 @@ function copyObject(obj) {
  * with an actor ID, separated by an `@` sign) and returns an object `{counter, actorId}`.
  */
 function parseOpId(opId) {
-  const match = /^(\d+)@(.*)$/.exec(opId || "")
+  const match = /^(\d+)@(.*)$/.exec(opId || '')
   if (!match) {
     throw new RangeError(`Not a valid opId: ${opId}`)
   }
-  return { counter: parseInt(match[1], 10), actorId: match[2] }
+  return {counter: parseInt(match[1], 10), actorId: match[2]}
 }
 
 /**
@@ -32,7 +32,7 @@ function parseOpId(opId) {
  */
 function equalBytes(array1, array2) {
   if (!(array1 instanceof Uint8Array) || !(array2 instanceof Uint8Array)) {
-    throw new TypeError("equalBytes can only compare Uint8Arrays")
+    throw new TypeError('equalBytes can only compare Uint8Arrays')
   }
   if (array1.byteLength !== array2.byteLength) return false
   for (let i = 0; i < array1.byteLength; i++) {
@@ -41,19 +41,6 @@ function equalBytes(array1, array2) {
   return true
 }
 
-/**
- * Creates an array containing the value `null` repeated `length` times.
- */
-function createArrayOfNulls(length) {
-  const array = new Array(length)
-  for (let i = 0; i < length; i++) array[i] = null
-  return array
-}
-
 module.exports = {
-  isObject,
-  copyObject,
-  parseOpId,
-  equalBytes,
-  createArrayOfNulls,
+  isObject, copyObject, parseOpId, equalBytes
 }

automerge-wasm/test/helpers/encoding.js

@@ -6,7 +6,7 @@
  * https://github.com/anonyco/FastestSmallestTextEncoderDecoder
  */
 const utf8encoder = new TextEncoder()
-const utf8decoder = new TextDecoder("utf-8")
+const utf8decoder = new TextDecoder('utf-8')
 
 function stringToUtf8(string) {
   return utf8encoder.encode(string)
@@ -20,48 +20,30 @@ function utf8ToString(buffer) {
  * Converts a string consisting of hexadecimal digits into an Uint8Array.
  */
 function hexStringToBytes(value) {
-  if (typeof value !== "string") {
-    throw new TypeError("value is not a string")
+  if (typeof value !== 'string') {
+    throw new TypeError('value is not a string')
   }
   if (!/^([0-9a-f][0-9a-f])*$/.test(value)) {
-    throw new RangeError("value is not hexadecimal")
+    throw new RangeError('value is not hexadecimal')
   }
-  if (value === "") {
+  if (value === '') {
     return new Uint8Array(0)
   } else {
     return new Uint8Array(value.match(/../g).map(b => parseInt(b, 16)))
   }
 }
 
-const NIBBLE_TO_HEX = [
-  "0",
-  "1",
-  "2",
-  "3",
-  "4",
-  "5",
-  "6",
-  "7",
-  "8",
-  "9",
-  "a",
-  "b",
-  "c",
-  "d",
-  "e",
-  "f",
-]
+const NIBBLE_TO_HEX = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
 const BYTE_TO_HEX = new Array(256)
 for (let i = 0; i < 256; i++) {
-  BYTE_TO_HEX[i] = `${NIBBLE_TO_HEX[(i >>> 4) & 0xf]}${NIBBLE_TO_HEX[i & 0xf]}`
+  BYTE_TO_HEX[i] = `${NIBBLE_TO_HEX[(i >>> 4) & 0xf]}${NIBBLE_TO_HEX[i & 0xf]}`;
 }
 
 /**
  * Converts a Uint8Array into the equivalent hexadecimal string.
  */
 function bytesToHexString(bytes) {
-  let hex = "",
-    len = bytes.byteLength
+  let hex = '', len = bytes.byteLength
   for (let i = 0; i < len; i++) {
     hex += BYTE_TO_HEX[bytes[i]]
   }
@@ -113,17 +95,14 @@ class Encoder {
    * appends it to the buffer. Returns the number of bytes written.
    */
   appendUint32(value) {
-    if (!Number.isInteger(value))
-      throw new RangeError("value is not an integer")
-    if (value < 0 || value > 0xffffffff)
-      throw new RangeError("number out of range")
+    if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
+    if (value < 0 || value > 0xffffffff) throw new RangeError('number out of range')
 
     const numBytes = Math.max(1, Math.ceil((32 - Math.clz32(value)) / 7))
     if (this.offset + numBytes > this.buf.byteLength) this.grow()
 
     for (let i = 0; i < numBytes; i++) {
-      this.buf[this.offset + i] =
-        (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
+      this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
       value >>>= 7 // zero-filling right shift
     }
     this.offset += numBytes
@@ -136,19 +115,14 @@ class Encoder {
    * it to the buffer. Returns the number of bytes written.
    */
   appendInt32(value) {
-    if (!Number.isInteger(value))
-      throw new RangeError("value is not an integer")
-    if (value < -0x80000000 || value > 0x7fffffff)
-      throw new RangeError("number out of range")
+    if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
+    if (value < -0x80000000 || value > 0x7fffffff) throw new RangeError('number out of range')
 
-    const numBytes = Math.ceil(
-      (33 - Math.clz32(value >= 0 ? value : -value - 1)) / 7
-    )
+    const numBytes = Math.ceil((33 - Math.clz32(value >= 0 ? value : -value - 1)) / 7)
     if (this.offset + numBytes > this.buf.byteLength) this.grow()
 
     for (let i = 0; i < numBytes; i++) {
-      this.buf[this.offset + i] =
-        (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
+      this.buf[this.offset + i] = (value & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
       value >>= 7 // sign-propagating right shift
     }
     this.offset += numBytes
@@ -161,10 +135,9 @@ class Encoder {
    * (53 bits).
    */
   appendUint53(value) {
-    if (!Number.isInteger(value))
-      throw new RangeError("value is not an integer")
+    if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
     if (value < 0 || value > Number.MAX_SAFE_INTEGER) {
-      throw new RangeError("number out of range")
+      throw new RangeError('number out of range')
     }
     const high32 = Math.floor(value / 0x100000000)
     const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned
@@ -177,10 +150,9 @@ class Encoder {
    * (53 bits).
    */
   appendInt53(value) {
-    if (!Number.isInteger(value))
-      throw new RangeError("value is not an integer")
+    if (!Number.isInteger(value)) throw new RangeError('value is not an integer')
     if (value < Number.MIN_SAFE_INTEGER || value > Number.MAX_SAFE_INTEGER) {
-      throw new RangeError("number out of range")
+      throw new RangeError('number out of range')
     }
     const high32 = Math.floor(value / 0x100000000)
     const low32 = (value & 0xffffffff) >>> 0 // right shift to interpret as unsigned
@@ -195,10 +167,10 @@ class Encoder {
    */
   appendUint64(high32, low32) {
     if (!Number.isInteger(high32) || !Number.isInteger(low32)) {
-      throw new RangeError("value is not an integer")
+      throw new RangeError('value is not an integer')
     }
     if (high32 < 0 || high32 > 0xffffffff || low32 < 0 || low32 > 0xffffffff) {
-      throw new RangeError("number out of range")
+      throw new RangeError('number out of range')
     }
     if (high32 === 0) return this.appendUint32(low32)
 
@@ -208,12 +180,10 @@ class Encoder {
       this.buf[this.offset + i] = (low32 & 0x7f) | 0x80
       low32 >>>= 7 // zero-filling right shift
     }
-    this.buf[this.offset + 4] =
-      (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
+    this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
     high32 >>>= 3
     for (let i = 5; i < numBytes; i++) {
-      this.buf[this.offset + i] =
-        (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
+      this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
       high32 >>>= 7
     }
     this.offset += numBytes
@@ -230,35 +200,25 @@ class Encoder {
    */
   appendInt64(high32, low32) {
     if (!Number.isInteger(high32) || !Number.isInteger(low32)) {
-      throw new RangeError("value is not an integer")
+      throw new RangeError('value is not an integer')
     }
-    if (
-      high32 < -0x80000000 ||
-      high32 > 0x7fffffff ||
-      low32 < -0x80000000 ||
-      low32 > 0xffffffff
-    ) {
-      throw new RangeError("number out of range")
+    if (high32 < -0x80000000 || high32 > 0x7fffffff || low32 < -0x80000000 || low32 > 0xffffffff) {
+      throw new RangeError('number out of range')
     }
     low32 >>>= 0 // interpret as unsigned
     if (high32 === 0 && low32 <= 0x7fffffff) return this.appendInt32(low32)
-    if (high32 === -1 && low32 >= 0x80000000)
-      return this.appendInt32(low32 - 0x100000000)
+    if (high32 === -1 && low32 >= 0x80000000) return this.appendInt32(low32 - 0x100000000)
 
-    const numBytes = Math.ceil(
-      (65 - Math.clz32(high32 >= 0 ? high32 : -high32 - 1)) / 7
-    )
+    const numBytes = Math.ceil((65 - Math.clz32(high32 >= 0 ? high32 : -high32 - 1)) / 7)
     if (this.offset + numBytes > this.buf.byteLength) this.grow()
     for (let i = 0; i < 4; i++) {
       this.buf[this.offset + i] = (low32 & 0x7f) | 0x80
       low32 >>>= 7 // zero-filling right shift
     }
-    this.buf[this.offset + 4] =
-      (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
+    this.buf[this.offset + 4] = (low32 & 0x0f) | ((high32 & 0x07) << 4) | (numBytes === 5 ? 0x00 : 0x80)
     high32 >>= 3 // sign-propagating right shift
     for (let i = 5; i < numBytes; i++) {
-      this.buf[this.offset + i] =
-        (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
+      this.buf[this.offset + i] = (high32 & 0x7f) | (i === numBytes - 1 ? 0x00 : 0x80)
       high32 >>= 7
     }
     this.offset += numBytes
@@ -283,7 +243,7 @@ class Encoder {
    * number of bytes appended.
    */
   appendRawString(value) {
-    if (typeof value !== "string") throw new TypeError("value is not a string")
+    if (typeof value !== 'string') throw new TypeError('value is not a string')
     return this.appendRawBytes(stringToUtf8(value))
   }
 
@@ -302,7 +262,7 @@ class Encoder {
    * (where the length is encoded as an unsigned LEB128 integer).
    */
   appendPrefixedString(value) {
-    if (typeof value !== "string") throw new TypeError("value is not a string")
+    if (typeof value !== 'string') throw new TypeError('value is not a string')
     this.appendPrefixedBytes(stringToUtf8(value))
     return this
   }
@@ -321,7 +281,8 @@ class Encoder {
    * Flushes any unwritten data to the buffer. Call this before reading from
    * the buffer constructed by this Encoder.
    */
-  finish() {}
+  finish() {
+  }
 }
 
 /**
@@ -360,7 +321,7 @@ class Decoder {
    */
   skip(bytes) {
     if (this.offset + bytes > this.buf.byteLength) {
-      throw new RangeError("cannot skip beyond end of buffer")
+      throw new RangeError('cannot skip beyond end of buffer')
     }
     this.offset += bytes
   }
@@ -378,20 +339,18 @@ class Decoder {
    * Throws an exception if the value doesn't fit in a 32-bit unsigned int.
    */
   readUint32() {
-    let result = 0,
-      shift = 0
+    let result = 0, shift = 0
     while (this.offset < this.buf.byteLength) {
       const nextByte = this.buf[this.offset]
-      if (shift === 28 && (nextByte & 0xf0) !== 0) {
-        // more than 5 bytes, or value > 0xffffffff
-        throw new RangeError("number out of range")
+      if (shift === 28 && (nextByte & 0xf0) !== 0) { // more than 5 bytes, or value > 0xffffffff
+        throw new RangeError('number out of range')
       }
-      result = (result | ((nextByte & 0x7f) << shift)) >>> 0 // right shift to interpret value as unsigned
+      result = (result | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
       shift += 7
       this.offset++
       if ((nextByte & 0x80) === 0) return result
     }
-    throw new RangeError("buffer ended with incomplete number")
+    throw new RangeError('buffer ended with incomplete number')
   }
 
   /**
@@ -399,17 +358,13 @@ class Decoder {
    * Throws an exception if the value doesn't fit in a 32-bit signed int.
    */
   readInt32() {
-    let result = 0,
-      shift = 0
+    let result = 0, shift = 0
     while (this.offset < this.buf.byteLength) {
       const nextByte = this.buf[this.offset]
-      if (
-        (shift === 28 && (nextByte & 0x80) !== 0) || // more than 5 bytes
-        (shift === 28 && (nextByte & 0x40) === 0 && (nextByte & 0x38) !== 0) || // positive int > 0x7fffffff
-        (shift === 28 && (nextByte & 0x40) !== 0 && (nextByte & 0x38) !== 0x38)
-      ) {
-        // negative int < -0x80000000
-        throw new RangeError("number out of range")
+      if ((shift === 28 && (nextByte & 0x80) !== 0) || // more than 5 bytes
+          (shift === 28 && (nextByte & 0x40) === 0 && (nextByte & 0x38) !== 0) || // positive int > 0x7fffffff
+          (shift === 28 && (nextByte & 0x40) !== 0 && (nextByte & 0x38) !== 0x38)) { // negative int < -0x80000000
+        throw new RangeError('number out of range')
       }
       result |= (nextByte & 0x7f) << shift
       shift += 7
@@ -423,7 +378,7 @@ class Decoder {
         }
       }
     }
-    throw new RangeError("buffer ended with incomplete number")
+    throw new RangeError('buffer ended with incomplete number')
   }
 
   /**
@@ -434,7 +389,7 @@ class Decoder {
   readUint53() {
     const { low32, high32 } = this.readUint64()
     if (high32 < 0 || high32 > 0x1fffff) {
-      throw new RangeError("number out of range")
+      throw new RangeError('number out of range')
     }
     return high32 * 0x100000000 + low32
   }
@@ -446,12 +401,8 @@ class Decoder {
    */
   readInt53() {
     const { low32, high32 } = this.readInt64()
-    if (
-      high32 < -0x200000 ||
-      (high32 === -0x200000 && low32 === 0) ||
-      high32 > 0x1fffff
-    ) {
-      throw new RangeError("number out of range")
+    if (high32 < -0x200000 || (high32 === -0x200000 && low32 === 0) || high32 > 0x1fffff) {
+      throw new RangeError('number out of range')
     }
     return high32 * 0x100000000 + low32
   }
@@ -463,12 +414,10 @@ class Decoder {
    * `{high32, low32}`.
    */
   readUint64() {
-    let low32 = 0,
-      high32 = 0,
-      shift = 0
+    let low32 = 0, high32 = 0, shift = 0
     while (this.offset < this.buf.byteLength && shift <= 28) {
       const nextByte = this.buf[this.offset]
-      low32 = (low32 | ((nextByte & 0x7f) << shift)) >>> 0 // right shift to interpret value as unsigned
+      low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
       if (shift === 28) {
         high32 = (nextByte & 0x70) >>> 4
       }
@@ -480,16 +429,15 @@ class Decoder {
     shift = 3
     while (this.offset < this.buf.byteLength) {
       const nextByte = this.buf[this.offset]
-      if (shift === 31 && (nextByte & 0xfe) !== 0) {
-        // more than 10 bytes, or value > 2^64 - 1
-        throw new RangeError("number out of range")
+      if (shift === 31 && (nextByte & 0xfe) !== 0) { // more than 10 bytes, or value > 2^64 - 1
+        throw new RangeError('number out of range')
       }
-      high32 = (high32 | ((nextByte & 0x7f) << shift)) >>> 0
+      high32 = (high32 | (nextByte & 0x7f) << shift) >>> 0
       shift += 7
       this.offset++
       if ((nextByte & 0x80) === 0) return { high32, low32 }
     }
-    throw new RangeError("buffer ended with incomplete number")
+    throw new RangeError('buffer ended with incomplete number')
   }
 
   /**
@@ -500,20 +448,17 @@ class Decoder {
    * sign of the `high32` half indicates the sign of the 64-bit number.
    */
   readInt64() {
-    let low32 = 0,
-      high32 = 0,
-      shift = 0
+    let low32 = 0, high32 = 0, shift = 0
     while (this.offset < this.buf.byteLength && shift <= 28) {
       const nextByte = this.buf[this.offset]
-      low32 = (low32 | ((nextByte & 0x7f) << shift)) >>> 0 // right shift to interpret value as unsigned
+      low32 = (low32 | (nextByte & 0x7f) << shift) >>> 0 // right shift to interpret value as unsigned
       if (shift === 28) {
         high32 = (nextByte & 0x70) >>> 4
       }
       shift += 7
       this.offset++
       if ((nextByte & 0x80) === 0) {
-        if ((nextByte & 0x40) !== 0) {
-          // sign-extend negative integer
+        if ((nextByte & 0x40) !== 0) { // sign-extend negative integer
           if (shift < 32) low32 = (low32 | (-1 << shift)) >>> 0
           high32 |= -1 << Math.max(shift - 32, 0)
         }
@@ -527,20 +472,19 @@ class Decoder {
       // On the 10th byte there are only two valid values: all 7 value bits zero
       // (if the value is positive) or all 7 bits one (if the value is negative)
       if (shift === 31 && nextByte !== 0 && nextByte !== 0x7f) {
-        throw new RangeError("number out of range")
+        throw new RangeError('number out of range')
       }
       high32 |= (nextByte & 0x7f) << shift
       shift += 7
       this.offset++
       if ((nextByte & 0x80) === 0) {
-        if ((nextByte & 0x40) !== 0 && shift < 32) {
-          // sign-extend negative integer
+        if ((nextByte & 0x40) !== 0 && shift < 32) { // sign-extend negative integer
           high32 |= -1 << shift
         }
         return { high32, low32 }
       }
     }
-    throw new RangeError("buffer ended with incomplete number")
+    throw new RangeError('buffer ended with incomplete number')
   }
 
   /**
@@ -550,7 +494,7 @@ class Decoder {
   readRawBytes(length) {
     const start = this.offset
     if (start + length > this.buf.byteLength) {
-      throw new RangeError("subarray exceeds buffer size")
+      throw new RangeError('subarray exceeds buffer size')
     }
     this.offset += length
     return this.buf.subarray(start, this.offset)
@@ -615,7 +559,7 @@ class RLEEncoder extends Encoder {
   constructor(type) {
     super()
     this.type = type
-    this.state = "empty"
+    this.state = 'empty'
     this.lastValue = undefined
     this.count = 0
     this.literal = []
@@ -634,81 +578,76 @@ class RLEEncoder extends Encoder {
    */
   _appendValue(value, repetitions = 1) {
     if (repetitions <= 0) return
-    if (this.state === "empty") {
-      this.state =
-        value === null
-          ? "nulls"
-          : repetitions === 1
-          ? "loneValue"
-          : "repetition"
+    if (this.state === 'empty') {
+      this.state = (value === null ? 'nulls' : (repetitions === 1 ? 'loneValue' : 'repetition'))
       this.lastValue = value
       this.count = repetitions
-    } else if (this.state === "loneValue") {
+    } else if (this.state === 'loneValue') {
       if (value === null) {
         this.flush()
-        this.state = "nulls"
+        this.state = 'nulls'
         this.count = repetitions
       } else if (value === this.lastValue) {
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = 1 + repetitions
       } else if (repetitions > 1) {
         this.flush()
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = repetitions
         this.lastValue = value
       } else {
-        this.state = "literal"
+        this.state = 'literal'
         this.literal = [this.lastValue]
         this.lastValue = value
       }
-    } else if (this.state === "repetition") {
+    } else if (this.state === 'repetition') {
       if (value === null) {
         this.flush()
-        this.state = "nulls"
+        this.state = 'nulls'
         this.count = repetitions
       } else if (value === this.lastValue) {
         this.count += repetitions
       } else if (repetitions > 1) {
         this.flush()
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = repetitions
         this.lastValue = value
       } else {
         this.flush()
-        this.state = "loneValue"
+        this.state = 'loneValue'
         this.lastValue = value
       }
-    } else if (this.state === "literal") {
+    } else if (this.state === 'literal') {
       if (value === null) {
         this.literal.push(this.lastValue)
         this.flush()
-        this.state = "nulls"
+        this.state = 'nulls'
         this.count = repetitions
       } else if (value === this.lastValue) {
         this.flush()
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = 1 + repetitions
       } else if (repetitions > 1) {
         this.literal.push(this.lastValue)
         this.flush()
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = repetitions
         this.lastValue = value
       } else {
         this.literal.push(this.lastValue)
         this.lastValue = value
       }
-    } else if (this.state === "nulls") {
+    } else if (this.state === 'nulls') {
       if (value === null) {
         this.count += repetitions
       } else if (repetitions > 1) {
         this.flush()
-        this.state = "repetition"
+        this.state = 'repetition'
         this.count = repetitions
         this.lastValue = value
       } else {
         this.flush()
-        this.state = "loneValue"
+        this.state = 'loneValue'
         this.lastValue = value
       }
     }
@@ -727,16 +666,13 @@ class RLEEncoder extends Encoder {
    */
   copyFrom(decoder, options = {}) {
     const { count, sumValues, sumShift } = options
-    if (!(decoder instanceof RLEDecoder) || decoder.type !== this.type) {
-      throw new TypeError("incompatible type of decoder")
+    if (!(decoder instanceof RLEDecoder) || (decoder.type !== this.type)) {
+      throw new TypeError('incompatible type of decoder')
     }
-    let remaining = typeof count === "number" ? count : Number.MAX_SAFE_INTEGER
-    let nonNullValues = 0,
-      sum = 0
-    if (count && remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${count} values`)
-    if (remaining === 0 || decoder.done)
-      return sumValues ? { nonNullValues, sum } : { nonNullValues }
+    let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER)
+    let nonNullValues = 0, sum = 0
+    if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
+    if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
 
     // Copy a value so that we have a well-defined starting state. NB: when super.copyFrom() is
     // called by the DeltaEncoder subclass, the following calls to readValue() and appendValue()
@@ -748,101 +684,87 @@ class RLEEncoder extends Encoder {
       remaining -= numNulls
       decoder.count -= numNulls - 1
       this.appendValue(null, numNulls)
-      if (count && remaining > 0 && decoder.done)
-        throw new RangeError(`cannot copy ${count} values`)
-      if (remaining === 0 || decoder.done)
-        return sumValues ? { nonNullValues, sum } : { nonNullValues }
+      if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
+      if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
       firstValue = decoder.readValue()
-      if (firstValue === null)
-        throw new RangeError("null run must be followed by non-null value")
+      if (firstValue === null) throw new RangeError('null run must be followed by non-null value')
     }
     this.appendValue(firstValue)
     remaining--
     nonNullValues++
-    if (sumValues) sum += sumShift ? firstValue >>> sumShift : firstValue
-    if (count && remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${count} values`)
-    if (remaining === 0 || decoder.done)
-      return sumValues ? { nonNullValues, sum } : { nonNullValues }
+    if (sumValues) sum += (sumShift ? (firstValue >>> sumShift) : firstValue)
+    if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
+    if (remaining === 0 || decoder.done) return sumValues ? {nonNullValues, sum} : {nonNullValues}
 
     // Copy data at the record level without expanding repetitions
-    let firstRun = decoder.count > 0
+    let firstRun = (decoder.count > 0)
     while (remaining > 0 && !decoder.done) {
       if (!firstRun) decoder.readRecord()
       const numValues = Math.min(decoder.count, remaining)
       decoder.count -= numValues
 
-      if (decoder.state === "literal") {
+      if (decoder.state === 'literal') {
         nonNullValues += numValues
         for (let i = 0; i < numValues; i++) {
-          if (decoder.done) throw new RangeError("incomplete literal")
+          if (decoder.done) throw new RangeError('incomplete literal')
           const value = decoder.readRawValue()
-          if (value === decoder.lastValue)
-            throw new RangeError(
-              "Repetition of values is not allowed in literal"
-            )
+          if (value === decoder.lastValue) throw new RangeError('Repetition of values is not allowed in literal')
           decoder.lastValue = value
           this._appendValue(value)
-          if (sumValues) sum += sumShift ? value >>> sumShift : value
+          if (sumValues) sum += (sumShift ? (value >>> sumShift) : value)
         }
-      } else if (decoder.state === "repetition") {
+      } else if (decoder.state === 'repetition') {
         nonNullValues += numValues
-        if (sumValues)
-          sum +=
-            numValues *
-            (sumShift ? decoder.lastValue >>> sumShift : decoder.lastValue)
+        if (sumValues) sum += numValues * (sumShift ? (decoder.lastValue >>> sumShift) : decoder.lastValue)
         const value = decoder.lastValue
         this._appendValue(value)
         if (numValues > 1) {
           this._appendValue(value)
-          if (this.state !== "repetition")
-            throw new RangeError(`Unexpected state ${this.state}`)
+          if (this.state !== 'repetition') throw new RangeError(`Unexpected state ${this.state}`)
           this.count += numValues - 2
         }
-      } else if (decoder.state === "nulls") {
+      } else if (decoder.state === 'nulls') {
         this._appendValue(null)
-        if (this.state !== "nulls")
-          throw new RangeError(`Unexpected state ${this.state}`)
+        if (this.state !== 'nulls') throw new RangeError(`Unexpected state ${this.state}`)
         this.count += numValues - 1
       }
 
       firstRun = false
       remaining -= numValues
     }
-    if (count && remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${count} values`)
-    return sumValues ? { nonNullValues, sum } : { nonNullValues }
+    if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
+    return sumValues ? {nonNullValues, sum} : {nonNullValues}
   }
 
   /**
    * Private method, do not call from outside the class.
    */
   flush() {
-    if (this.state === "loneValue") {
+    if (this.state === 'loneValue') {
       this.appendInt32(-1)
       this.appendRawValue(this.lastValue)
-    } else if (this.state === "repetition") {
+    } else if (this.state === 'repetition') {
       this.appendInt53(this.count)
       this.appendRawValue(this.lastValue)
-    } else if (this.state === "literal") {
+    } else if (this.state === 'literal') {
       this.appendInt53(-this.literal.length)
       for (let v of this.literal) this.appendRawValue(v)
-    } else if (this.state === "nulls") {
+    } else if (this.state === 'nulls') {
       this.appendInt32(0)
       this.appendUint53(this.count)
     }
-    this.state = "empty"
+    this.state = 'empty'
   }
 
   /**
    * Private method, do not call from outside the class.
    */
   appendRawValue(value) {
-    if (this.type === "int") {
+    if (this.type === 'int') {
       this.appendInt53(value)
-    } else if (this.type === "uint") {
+    } else if (this.type === 'uint') {
       this.appendUint53(value)
-    } else if (this.type === "utf8") {
+    } else if (this.type === 'utf8') {
       this.appendPrefixedString(value)
     } else {
       throw new RangeError(`Unknown RLEEncoder datatype: ${this.type}`)
@@ -854,9 +776,9 @@ class RLEEncoder extends Encoder {
    * the buffer constructed by this Encoder.
    */
   finish() {
-    if (this.state === "literal") this.literal.push(this.lastValue)
+    if (this.state === 'literal') this.literal.push(this.lastValue)
     // Don't write anything if the only values we have seen are nulls
-    if (this.state !== "nulls" || this.offset > 0) this.flush()
+    if (this.state !== 'nulls' || this.offset > 0) this.flush()
   }
 }
 
@@ -878,7 +800,7 @@ class RLEDecoder extends Decoder {
    * position, and true if we are at the end of the buffer.
    */
   get done() {
-    return this.count === 0 && this.offset === this.buf.byteLength
+    return (this.count === 0) && (this.offset === this.buf.byteLength)
   }
 
   /**
@@ -899,10 +821,9 @@ class RLEDecoder extends Decoder {
     if (this.done) return null
     if (this.count === 0) this.readRecord()
     this.count -= 1
-    if (this.state === "literal") {
+    if (this.state === 'literal') {
       const value = this.readRawValue()
-      if (value === this.lastValue)
-        throw new RangeError("Repetition of values is not allowed in literal")
+      if (value === this.lastValue) throw new RangeError('Repetition of values is not allowed in literal')
       this.lastValue = value
       return value
     } else {
@@ -918,22 +839,20 @@ class RLEDecoder extends Decoder {
       if (this.count === 0) {
         this.count = this.readInt53()
         if (this.count > 0) {
-          this.lastValue =
-            this.count <= numSkip ? this.skipRawValues(1) : this.readRawValue()
-          this.state = "repetition"
+          this.lastValue = (this.count <= numSkip) ? this.skipRawValues(1) : this.readRawValue()
+          this.state = 'repetition'
         } else if (this.count < 0) {
           this.count = -this.count
-          this.state = "literal"
-        } else {
-          // this.count == 0
+          this.state = 'literal'
+        } else { // this.count == 0
           this.count = this.readUint53()
           this.lastValue = null
-          this.state = "nulls"
+          this.state = 'nulls'
         }
       }
 
       const consume = Math.min(numSkip, this.count)
-      if (this.state === "literal") this.skipRawValues(consume)
+      if (this.state === 'literal') this.skipRawValues(consume)
       numSkip -= consume
       this.count -= consume
     }
@@ -947,34 +866,23 @@ class RLEDecoder extends Decoder {
     this.count = this.readInt53()
     if (this.count > 1) {
       const value = this.readRawValue()
-      if (
-        (this.state === "repetition" || this.state === "literal") &&
-        this.lastValue === value
-      ) {
-        throw new RangeError(
-          "Successive repetitions with the same value are not allowed"
-        )
+      if ((this.state === 'repetition' || this.state === 'literal') && this.lastValue === value) {
+        throw new RangeError('Successive repetitions with the same value are not allowed')
       }
-      this.state = "repetition"
+      this.state = 'repetition'
       this.lastValue = value
     } else if (this.count === 1) {
-      throw new RangeError(
-        "Repetition count of 1 is not allowed, use a literal instead"
-      )
+      throw new RangeError('Repetition count of 1 is not allowed, use a literal instead')
     } else if (this.count < 0) {
       this.count = -this.count
-      if (this.state === "literal")
-        throw new RangeError("Successive literals are not allowed")
-      this.state = "literal"
-    } else {
-      // this.count == 0
-      if (this.state === "nulls")
-        throw new RangeError("Successive null runs are not allowed")
+      if (this.state === 'literal') throw new RangeError('Successive literals are not allowed')
+      this.state = 'literal'
+    } else { // this.count == 0
+      if (this.state === 'nulls') throw new RangeError('Successive null runs are not allowed')
       this.count = this.readUint53()
-      if (this.count === 0)
-        throw new RangeError("Zero-length null runs are not allowed")
+      if (this.count === 0) throw new RangeError('Zero-length null runs are not allowed')
       this.lastValue = null
-      this.state = "nulls"
+      this.state = 'nulls'
     }
   }
 
@@ -983,11 +891,11 @@ class RLEDecoder extends Decoder {
    * Reads one value of the datatype configured on construction.
    */
   readRawValue() {
-    if (this.type === "int") {
+    if (this.type === 'int') {
       return this.readInt53()
-    } else if (this.type === "uint") {
+    } else if (this.type === 'uint') {
       return this.readUint53()
-    } else if (this.type === "utf8") {
+    } else if (this.type === 'utf8') {
       return this.readPrefixedString()
     } else {
       throw new RangeError(`Unknown RLEDecoder datatype: ${this.type}`)
@@ -999,14 +907,14 @@ class RLEDecoder extends Decoder {
    * Skips over `num` values of the datatype configured on construction.
    */
   skipRawValues(num) {
-    if (this.type === "utf8") {
+    if (this.type === 'utf8') {
       for (let i = 0; i < num; i++) this.skip(this.readUint53())
     } else {
       while (num > 0 && this.offset < this.buf.byteLength) {
         if ((this.buf[this.offset] & 0x80) === 0) num--
         this.offset++
       }
-      if (num > 0) throw new RangeError("cannot skip beyond end of buffer")
+      if (num > 0) throw new RangeError('cannot skip beyond end of buffer')
     }
   }
 }
@@ -1023,7 +931,7 @@ class RLEDecoder extends Decoder {
  */
 class DeltaEncoder extends RLEEncoder {
   constructor() {
-    super("int")
+    super('int')
     this.absoluteValue = 0
   }
 
@@ -1033,7 +941,7 @@ class DeltaEncoder extends RLEEncoder {
    */
   appendValue(value, repetitions = 1) {
     if (repetitions <= 0) return
-    if (typeof value === "number") {
+    if (typeof value === 'number') {
       super.appendValue(value - this.absoluteValue, 1)
       this.absoluteValue = value
       if (repetitions > 1) super.appendValue(0, repetitions - 1)
@@ -1049,29 +957,26 @@ class DeltaEncoder extends RLEEncoder {
    */
   copyFrom(decoder, options = {}) {
     if (options.sumValues) {
-      throw new RangeError("unsupported options for DeltaEncoder.copyFrom()")
+      throw new RangeError('unsupported options for DeltaEncoder.copyFrom()')
     }
     if (!(decoder instanceof DeltaDecoder)) {
-      throw new TypeError("incompatible type of decoder")
+      throw new TypeError('incompatible type of decoder')
     }
 
     let remaining = options.count
-    if (remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${remaining} values`)
+    if (remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${remaining} values`)
     if (remaining === 0 || decoder.done) return
 
     // Copy any null values, and the first non-null value, so that appendValue() computes the
     // difference between the encoder's last value and the decoder's first (absolute) value.
-    let value = decoder.readValue(),
-      nulls = 0
+    let value = decoder.readValue(), nulls = 0
     this.appendValue(value)
     if (value === null) {
       nulls = decoder.count + 1
       if (remaining !== undefined && remaining < nulls) nulls = remaining
       decoder.count -= nulls - 1
       this.count += nulls - 1
-      if (remaining > nulls && decoder.done)
-        throw new RangeError(`cannot copy ${remaining} values`)
+      if (remaining > nulls && decoder.done) throw new RangeError(`cannot copy ${remaining} values`)
       if (remaining === nulls || decoder.done) return
 
       // The next value read is certain to be non-null because we're not at the end of the decoder,
@@ -1084,10 +989,7 @@ class DeltaEncoder extends RLEEncoder {
     // value, while subsequent values are relative. Thus, the sum of all of the (non-null) copied
     // values must equal the absolute value of the final element copied.
     if (remaining !== undefined) remaining -= nulls + 1
-    const { nonNullValues, sum } = super.copyFrom(decoder, {
-      count: remaining,
-      sumValues: true,
-    })
+    const { nonNullValues, sum } = super.copyFrom(decoder, {count: remaining, sumValues: true})
     if (nonNullValues > 0) {
       this.absoluteValue = sum
       decoder.absoluteValue = sum
@@ -1101,7 +1003,7 @@ class DeltaEncoder extends RLEEncoder {
  */
 class DeltaDecoder extends RLEDecoder {
   constructor(buffer) {
-    super("int", buffer)
+    super('int', buffer)
     this.absoluteValue = 0
   }
 
@@ -1134,12 +1036,12 @@ class DeltaDecoder extends RLEDecoder {
     while (numSkip > 0 && !this.done) {
       if (this.count === 0) this.readRecord()
       const consume = Math.min(numSkip, this.count)
-      if (this.state === "literal") {
+      if (this.state === 'literal') {
         for (let i = 0; i < consume; i++) {
           this.lastValue = this.readRawValue()
           this.absoluteValue += this.lastValue
         }
-      } else if (this.state === "repetition") {
+      } else if (this.state === 'repetition') {
         this.absoluteValue += consume * this.lastValue
       }
       numSkip -= consume
@@ -1188,13 +1090,12 @@ class BooleanEncoder extends Encoder {
    */
   copyFrom(decoder, options = {}) {
     if (!(decoder instanceof BooleanDecoder)) {
-      throw new TypeError("incompatible type of decoder")
+      throw new TypeError('incompatible type of decoder')
     }
 
     const { count } = options
-    let remaining = typeof count === "number" ? count : Number.MAX_SAFE_INTEGER
-    if (count && remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${count} values`)
+    let remaining = (typeof count === 'number' ? count : Number.MAX_SAFE_INTEGER)
+    if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
     if (remaining === 0 || decoder.done) return
 
     // Copy one value to bring decoder and encoder state into sync, then finish that value's repetitions
@@ -1207,8 +1108,7 @@ class BooleanEncoder extends Encoder {
 
     while (remaining > 0 && !decoder.done) {
       decoder.count = decoder.readUint53()
-      if (decoder.count === 0)
-        throw new RangeError("Zero-length runs are not allowed")
+      if (decoder.count === 0) throw new RangeError('Zero-length runs are not allowed')
       decoder.lastValue = !decoder.lastValue
       this.appendUint53(this.count)
 
@@ -1219,8 +1119,7 @@ class BooleanEncoder extends Encoder {
       remaining -= numCopied
     }
 
-    if (count && remaining > 0 && decoder.done)
-      throw new RangeError(`cannot copy ${count} values`)
+    if (count && remaining > 0 && decoder.done) throw new RangeError(`cannot copy ${count} values`)
   }
 
   /**
@@ -1252,7 +1151,7 @@ class BooleanDecoder extends Decoder {
    * position, and true if we are at the end of the buffer.
    */
   get done() {
-    return this.count === 0 && this.offset === this.buf.byteLength
+    return (this.count === 0) && (this.offset === this.buf.byteLength)
   }
 
   /**
@@ -1275,7 +1174,7 @@ class BooleanDecoder extends Decoder {
       this.count = this.readUint53()
       this.lastValue = !this.lastValue
       if (this.count === 0 && !this.firstRun) {
-        throw new RangeError("Zero-length runs are not allowed")
+        throw new RangeError('Zero-length runs are not allowed')
       }
       this.firstRun = false
     }
@@ -1291,8 +1190,7 @@ class BooleanDecoder extends Decoder {
       if (this.count === 0) {
         this.count = this.readUint53()
         this.lastValue = !this.lastValue
-        if (this.count === 0)
-          throw new RangeError("Zero-length runs are not allowed")
+        if (this.count === 0) throw new RangeError('Zero-length runs are not allowed')
       }
       if (this.count < numSkip) {
         numSkip -= this.count
@@ -1306,16 +1204,6 @@ class BooleanDecoder extends Decoder {
 }
 
 module.exports = {
-  stringToUtf8,
-  utf8ToString,
-  hexStringToBytes,
-  bytesToHexString,
-  Encoder,
-  Decoder,
-  RLEEncoder,
-  RLEDecoder,
-  DeltaEncoder,
-  DeltaDecoder,
-  BooleanEncoder,
-  BooleanDecoder,
+  stringToUtf8, utf8ToString, hexStringToBytes, bytesToHexString,
+  Encoder, Decoder, RLEEncoder, RLEDecoder, DeltaEncoder, DeltaDecoder, BooleanEncoder, BooleanDecoder
 }

automerge-wasm/tsconfig.json

@@ -11,9 +11,7 @@
     "paths": { "dev": ["*"]},
     "rootDir": "",
     "target": "es2016",
-    "types": ["mocha", "node"],
-    "typeRoots": ["./index.d.ts"]
+    "typeRoots": ["./dev/index.d.ts"]
   },
-  "include": ["test/**/*.ts"],
-  "exclude": ["dist/**/*", "examples/**/*"]
+  "exclude": ["dist/**/*"]
 }

automerge/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "automerge"
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+optree-visualisation = ["dot"]
+wasm = ["js-sys", "wasm-bindgen"]
+storage-v2 = []
+
+[dependencies]
+hex = "^0.4.3"
+leb128 = "^0.2.5"
+sha2 = "^0.10.0"
+rand = { version = "^0.8.4" }
+thiserror = "^1.0.16"
+itertools = "^0.10.3"
+flate2 = "^1.0.22"
+nonzero_ext = "^0.2.0"
+uuid = { version = "^0.8.2", features=["v4", "wasm-bindgen", "serde"] }
+smol_str = "^0.1.21"
+tracing = { version = "^0.1.29", features = ["log"] }
+fxhash = "^0.2.1"
+tinyvec = { version = "^1.5.1", features = ["alloc"] }
+unicode-segmentation = "1.7.1"
+serde = { version = "^1.0", features=["derive"] }
+dot = { version = "0.1.4", optional = true }
+js-sys = { version = "^0.3", optional = true }
+wasm-bindgen = { version = "^0.2", optional = true }
+
+[dependencies.web-sys]
+version = "^0.3.55"
+features = ["console"]
+
+[dev-dependencies]
+pretty_assertions = "1.0.0"
+proptest = { version = "^1.0.0", default-features = false, features = ["std"] }
+serde_json = { version = "^1.0.73", features=["float_roundtrip"], default-features=true }
+maplit = { version = "^1.0" }
+decorum = "0.3.1"

automerge/examples/quickstart.rs

@@ -2,7 +2,7 @@ use automerge::transaction::CommitOptions;
 use automerge::transaction::Transactable;
 use automerge::AutomergeError;
 use automerge::ObjType;
-use automerge::{Automerge, ReadDoc, ROOT};
+use automerge::{Automerge, ROOT};
 
 // Based on https://automerge.github.io/docs/quickstart
 fn main() {
@@ -11,13 +11,13 @@ fn main() {
         .transact_with::<_, _, AutomergeError, _>(
             |_| CommitOptions::default().with_message("Add card".to_owned()),
             |tx| {
-                let cards = tx.put_object(ROOT, "cards", ObjType::List).unwrap();
+                let cards = tx.set_object(ROOT, "cards", ObjType::List).unwrap();
                 let card1 = tx.insert_object(&cards, 0, ObjType::Map)?;
-                tx.put(&card1, "title", "Rewrite everything in Clojure")?;
-                tx.put(&card1, "done", false)?;
+                tx.set(&card1, "title", "Rewrite everything in Clojure")?;
+                tx.set(&card1, "done", false)?;
                 let card2 = tx.insert_object(&cards, 0, ObjType::Map)?;
-                tx.put(&card2, "title", "Rewrite everything in Haskell")?;
-                tx.put(&card2, "done", false)?;
+                tx.set(&card2, "title", "Rewrite everything in Haskell")?;
+                tx.set(&card2, "done", false)?;
                 Ok((cards, card1))
             },
         )
@@ -33,7 +33,7 @@ fn main() {
     doc1.transact_with::<_, _, AutomergeError, _>(
         |_| CommitOptions::default().with_message("Mark card as done".to_owned()),
         |tx| {
-            tx.put(&card1, "done", true)?;
+            tx.set(&card1, "done", true)?;
             Ok(())
         },
     )
@@ -42,7 +42,7 @@ fn main() {
     doc2.transact_with::<_, _, AutomergeError, _>(
         |_| CommitOptions::default().with_message("Delete card".to_owned()),
         |tx| {
-            tx.delete(&cards, 0)?;
+            tx.del(&cards, 0)?;
             Ok(())
         },
     )
@@ -50,7 +50,7 @@ fn main() {
 
     doc1.merge(&mut doc2).unwrap();
 
-    for change in doc1.get_changes(&[]).unwrap() {
+    for change in doc1.get_changes(&[]) {
         let length = doc1.length_at(&cards, &[change.hash()]);
         println!("{} {}", change.message().unwrap(), length);
     }

automerge/src/autocommit.rs

@@ -0,0 +1,446 @@
+use crate::exid::ExId;
+use crate::transaction::{CommitOptions, Transactable};
+use crate::{
+    transaction::TransactionInner, ActorId, Automerge, AutomergeError,
+    Change, ChangeHash, Prop, Value,
+};
+#[cfg(not(feature = "storage-v2"))]
+use crate::change::export_change;
+use crate::{sync, Keys, KeysAt, ObjType, ScalarValue};
+
+/// An automerge document that automatically manages transactions.
+#[derive(Debug, Clone)]
+pub struct AutoCommit {
+    doc: Automerge,
+    transaction: Option<TransactionInner>,
+}
+
+impl Default for AutoCommit {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AutoCommit {
+    pub fn new() -> Self {
+        Self {
+            doc: Automerge::new(),
+            transaction: None,
+        }
+    }
+
+    /// Get the inner document.
+    #[doc(hidden)]
+    pub fn document(&mut self) -> &Automerge {
+        self.ensure_transaction_closed();
+        &self.doc
+    }
+
+    pub fn with_actor(mut self, actor: ActorId) -> Self {
+        self.ensure_transaction_closed();
+        self.doc.set_actor(actor);
+        self
+    }
+
+    pub fn set_actor(&mut self, actor: ActorId) -> &mut Self {
+        self.ensure_transaction_closed();
+        self.doc.set_actor(actor);
+        self
+    }
+
+    pub fn get_actor(&self) -> &ActorId {
+        self.doc.get_actor()
+    }
+
+    fn ensure_transaction_open(&mut self) {
+        if self.transaction.is_none() {
+            let actor = self.doc.get_actor_index();
+
+            let seq = self.doc.states.entry(actor).or_default().len() as u64 + 1;
+            let mut deps = self.doc.get_heads();
+            if seq > 1 {
+                let last_hash = self.get_hash(actor, seq - 1).unwrap();
+                if !deps.contains(&last_hash) {
+                    deps.push(last_hash);
+                }
+            }
+
+            self.transaction = Some(TransactionInner {
+                actor,
+                seq,
+                start_op: self.doc.max_op + 1,
+                time: 0,
+                message: None,
+                extra_bytes: Default::default(),
+                hash: None,
+                operations: vec![],
+                deps,
+            });
+        }
+    }
+
+    fn get_hash(&mut self, actor: usize, seq: u64) -> Result<ChangeHash, AutomergeError> {
+        self.doc
+            .states
+            .get(&actor)
+            .and_then(|v| v.get(seq as usize - 1))
+            .and_then(|&i| self.doc.history.get(i))
+            .map(|c| c.hash())
+            .ok_or(AutomergeError::InvalidSeq(seq))
+    }
+
+    fn update_history(&mut self, change: Change) -> usize {
+        self.doc.max_op = std::cmp::max(self.doc.max_op, change.start_op() + change.len() as u64 - 1);
+
+        self.update_deps(&change);
+
+        let history_index = self.doc.history.len();
+
+        self.doc
+            .states
+            .entry(self.doc.ops.m.actors.cache(change.actor_id().clone()))
+            .or_default()
+            .push(history_index);
+
+        self.doc.history_index.insert(change.hash(), history_index);
+        self.doc.history.push(change);
+
+        history_index
+    }
+
+    fn update_deps(&mut self, change: &Change) {
+        for d in change.deps() {
+            self.doc.deps.remove(d);
+        }
+        self.doc.deps.insert(change.hash());
+    }
+
+    pub fn fork(&mut self) -> Self {
+        self.ensure_transaction_closed();
+        Self {
+            doc: self.doc.fork(),
+            transaction: self.transaction.clone(),
+        }
+    }
+
+    fn ensure_transaction_closed(&mut self) {
+        if let Some(tx) = self.transaction.take() {
+            self.update_history(tx.export(
+                &self.doc.ops.m.actors,
+                &self.doc.ops.m.props,
+            ));
+        }
+    }
+
+    pub fn load(data: &[u8]) -> Result<Self, AutomergeError> {
+        let doc = Automerge::load(data)?;
+        Ok(Self {
+            doc,
+            transaction: None,
+        })
+    }
+
+    pub fn load_incremental(&mut self, data: &[u8]) -> Result<usize, AutomergeError> {
+        self.ensure_transaction_closed();
+        self.doc.load_incremental(data)
+    }
+
+    pub fn apply_changes(&mut self, changes: Vec<Change>) -> Result<(), AutomergeError> {
+        self.ensure_transaction_closed();
+        self.doc.apply_changes(changes)
+    }
+
+    /// Takes all the changes in `other` which are not in `self` and applies them
+    pub fn merge(&mut self, other: &mut Self) -> Result<Vec<ChangeHash>, AutomergeError> {
+        self.ensure_transaction_closed();
+        other.ensure_transaction_closed();
+        self.doc.merge(&mut other.doc)
+    }
+
+    pub fn save(&mut self) -> Vec<u8> {
+        self.ensure_transaction_closed();
+        self.doc.save()
+    }
+
+    // should this return an empty vec instead of None?
+    pub fn save_incremental(&mut self) -> Vec<u8> {
+        self.ensure_transaction_closed();
+        self.doc.save_incremental()
+    }
+
+    pub fn get_missing_deps(&mut self, heads: &[ChangeHash]) -> Vec<ChangeHash> {
+        self.ensure_transaction_closed();
+        self.doc.get_missing_deps(heads)
+    }
+
+    pub fn get_last_local_change(&mut self) -> Option<&Change> {
+        self.ensure_transaction_closed();
+        self.doc.get_last_local_change()
+    }
+
+    pub fn get_changes(&mut self, have_deps: &[ChangeHash]) -> Vec<&Change> {
+        self.ensure_transaction_closed();
+        self.doc.get_changes(have_deps)
+    }
+
+    pub fn get_change_by_hash(&mut self, hash: &ChangeHash) -> Option<&Change> {
+        self.ensure_transaction_closed();
+        self.doc.get_change_by_hash(hash)
+    }
+
+    pub fn get_changes_added<'a>(&mut self, other: &'a mut Self) -> Vec<&'a Change> {
+        self.ensure_transaction_closed();
+        other.ensure_transaction_closed();
+        self.doc.get_changes_added(&other.doc)
+    }
+
+    pub fn import(&self, s: &str) -> Result<ExId, AutomergeError> {
+        self.doc.import(s)
+    }
+
+    pub fn dump(&self) {
+        self.doc.dump()
+    }
+
+    pub fn generate_sync_message(&mut self, sync_state: &mut sync::State) -> Option<sync::Message> {
+        self.ensure_transaction_closed();
+        self.doc.generate_sync_message(sync_state)
+    }
+
+    pub fn receive_sync_message(
+        &mut self,
+        sync_state: &mut sync::State,
+        message: sync::Message,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_closed();
+        self.doc.receive_sync_message(sync_state, message)
+    }
+
+    #[cfg(feature = "optree-visualisation")]
+    pub fn visualise_optree(&self) -> String {
+        self.doc.visualise_optree()
+    }
+
+    /// Get the current heads of the document.
+    ///
+    /// This closes the transaction first, if one is in progress.
+    pub fn get_heads(&mut self) -> Vec<ChangeHash> {
+        self.ensure_transaction_closed();
+        self.doc.get_heads()
+    }
+
+    pub fn commit(&mut self) -> ChangeHash {
+        // ensure that even no changes triggers a change
+        self.ensure_transaction_open();
+        let tx = self.transaction.take().unwrap();
+        tx.commit(&mut self.doc, None, None)
+    }
+
+    /// Commit the current operations with some options.
+    ///
+    /// ```
+    /// # use automerge::transaction::CommitOptions;
+    /// # use automerge::transaction::Transactable;
+    /// # use automerge::ROOT;
+    /// # use automerge::AutoCommit;
+    /// # use automerge::ObjType;
+    /// # use std::time::SystemTime;
+    /// let mut doc = AutoCommit::new();
+    /// doc.set_object(&ROOT, "todos", ObjType::List).unwrap();
+    /// let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as
+    /// i64;
+    /// doc.commit_with(CommitOptions::default().with_message("Create todos list").with_time(now));
+    /// ```
+    pub fn commit_with(&mut self, options: CommitOptions) -> ChangeHash {
+        self.ensure_transaction_open();
+        let tx = self.transaction.take().unwrap();
+        tx.commit(&mut self.doc, options.message, options.time)
+    }
+
+    pub fn rollback(&mut self) -> usize {
+        self.transaction
+            .take()
+            .map(|tx| tx.rollback(&mut self.doc))
+            .unwrap_or(0)
+    }
+}
+
+impl Transactable for AutoCommit {
+    fn pending_ops(&self) -> usize {
+        self.transaction
+            .as_ref()
+            .map(|t| t.pending_ops())
+            .unwrap_or(0)
+    }
+
+    // KeysAt::()
+    // LenAt::()
+    // PropAt::()
+    // NthAt::()
+
+    fn keys<O: AsRef<ExId>>(&self, obj: O) -> Keys {
+        self.doc.keys(obj)
+    }
+
+    fn keys_at<O: AsRef<ExId>>(&self, obj: O, heads: &[ChangeHash]) -> KeysAt {
+        self.doc.keys_at(obj, heads)
+    }
+
+    fn length<O: AsRef<ExId>>(&self, obj: O) -> usize {
+        self.doc.length(obj)
+    }
+
+    fn length_at<O: AsRef<ExId>>(&self, obj: O, heads: &[ChangeHash]) -> usize {
+        self.doc.length_at(obj, heads)
+    }
+
+    fn object_type<O: AsRef<ExId>>(&self, obj: O) -> Option<ObjType> {
+        self.doc.object_type(obj)
+    }
+
+    // set(obj, prop, value) - value can be scalar or objtype
+    // del(obj, prop)
+    // inc(obj, prop, value)
+    // insert(obj, index, value)
+
+    /// Set the value of property `P` to value `V` in object `obj`.
+    ///
+    /// # Returns
+    ///
+    /// The opid of the operation which was created, or None if this operation doesn't change the
+    /// document or create a new object.
+    ///
+    /// # Errors
+    ///
+    /// This will return an error if
+    /// - The object does not exist
+    /// - The key is the wrong type for the object
+    /// - The key does not exist in the object
+    fn set<O: AsRef<ExId>, P: Into<Prop>, V: Into<ScalarValue>>(
+        &mut self,
+        obj: O,
+        prop: P,
+        value: V,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.set(&mut self.doc, obj.as_ref(), prop, value)
+    }
+
+    fn set_object<O: AsRef<ExId>, P: Into<Prop>>(
+        &mut self,
+        obj: O,
+        prop: P,
+        value: ObjType,
+    ) -> Result<ExId, AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.set_object(&mut self.doc, obj.as_ref(), prop, value)
+    }
+
+    fn insert<O: AsRef<ExId>, V: Into<ScalarValue>>(
+        &mut self,
+        obj: O,
+        index: usize,
+        value: V,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.insert(&mut self.doc, obj.as_ref(), index, value)
+    }
+
+    fn insert_object(
+        &mut self,
+        obj: &ExId,
+        index: usize,
+        value: ObjType,
+    ) -> Result<ExId, AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.insert_object(&mut self.doc, obj, index, value)
+    }
+
+    fn inc<O: AsRef<ExId>, P: Into<Prop>>(
+        &mut self,
+        obj: O,
+        prop: P,
+        value: i64,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.inc(&mut self.doc, obj.as_ref(), prop, value)
+    }
+
+    fn del<O: AsRef<ExId>, P: Into<Prop>>(
+        &mut self,
+        obj: O,
+        prop: P,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.del(&mut self.doc, obj.as_ref(), prop)
+    }
+
+    /// Splice new elements into the given sequence. Returns a vector of the OpIds used to insert
+    /// the new elements
+    fn splice<O: AsRef<ExId>, V: IntoIterator<Item = ScalarValue>>(
+        &mut self,
+        obj: O,
+        pos: usize,
+        del: usize,
+        vals: V,
+    ) -> Result<(), AutomergeError> {
+        self.ensure_transaction_open();
+        let tx = self.transaction.as_mut().unwrap();
+        tx.splice(&mut self.doc, obj.as_ref(), pos, del, vals)
+    }
+
+    fn text<O: AsRef<ExId>>(&self, obj: O) -> Result<String, AutomergeError> {
+        self.doc.text(obj)
+    }
+
+    fn text_at<O: AsRef<ExId>>(
+        &self,
+        obj: O,
+        heads: &[ChangeHash],
+    ) -> Result<String, AutomergeError> {
+        self.doc.text_at(obj, heads)
+    }
+
+    // TODO - I need to return these OpId's here **only** to get
+    // the legacy conflicts format of { [opid]: value }
+    // Something better?
+    fn value<O: AsRef<ExId>, P: Into<Prop>>(
+        &self,
+        obj: O,
+        prop: P,
+    ) -> Result<Option<(Value, ExId)>, AutomergeError> {
+        self.doc.value(obj, prop)
+    }
+
+    fn value_at<O: AsRef<ExId>, P: Into<Prop>>(
+        &self,
+        obj: O,
+        prop: P,
+        heads: &[ChangeHash],
+    ) -> Result<Option<(Value, ExId)>, AutomergeError> {
+        self.doc.value_at(obj, prop, heads)
+    }
+
+    fn values<O: AsRef<ExId>, P: Into<Prop>>(
+        &self,
+        obj: O,
+        prop: P,
+    ) -> Result<Vec<(Value, ExId)>, AutomergeError> {
+        self.doc.values(obj, prop)
+    }
+
+    fn values_at<O: AsRef<ExId>, P: Into<Prop>>(
+        &self,
+        obj: O,
+        prop: P,
+        heads: &[ChangeHash],
+    ) -> Result<Vec<(Value, ExId)>, AutomergeError> {
+        self.doc.values_at(obj, prop, heads)
+    }
+}

automerge/src/autoserde.rs

@@ -0,0 +1,89 @@
+use serde::ser::{SerializeSeq, SerializeMap};
+
+use crate::{ObjId, Automerge, Value, ObjType};
+
+pub struct AutoSerde<'a>(&'a Automerge);
+
+impl<'a> From<&'a Automerge> for AutoSerde<'a> {
+    fn from(a: &'a Automerge) -> Self {
+        AutoSerde(a) 
+    }
+}
+
+impl<'a> serde::Serialize for AutoSerde<'a> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+            S: serde::Serializer {
+        AutoSerdeMap{doc: self.0, obj: ObjId::Root}.serialize(serializer)
+    }
+}
+
+struct AutoSerdeMap<'a>{doc: &'a Automerge, obj: ObjId}
+
+impl<'a> serde::Serialize for AutoSerdeMap<'a> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+            S: serde::Serializer {
+        let mut map_ser = serializer.serialize_map(Some(self.doc.length(&ObjId::Root)))?;
+        for key in self.doc.keys(&self.obj) {
+            // SAFETY: This only errors if the object ID is unknown, but we construct this type
+            // with a known real object ID
+            let (val, obj) = self.doc.value(&self.obj, &key).unwrap().unwrap();
+            let serdeval = AutoSerdeVal{
+                doc: &self.doc,
+                val,
+                obj,
+            };
+            map_ser.serialize_entry(&key, &serdeval)?;
+        }
+        map_ser.end()
+    }
+}
+
+struct AutoSerdeSeq<'a>{doc: &'a Automerge, obj: ObjId}
+
+impl<'a> serde::Serialize for AutoSerdeSeq<'a> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+            S: serde::Serializer {
+        let mut seq_ser = serializer.serialize_seq(None)?;
+        for i in 0..self.doc.length(&self.obj) {
+            // SAFETY: This only errors if the object ID is unknown, but we construct this type
+            // with a known real object ID
+            let (val, obj) = self.doc.value(&self.obj, i).unwrap().unwrap();
+            let serdeval = AutoSerdeVal{
+                doc: &self.doc,
+                val,
+                obj,
+            };
+            seq_ser.serialize_element(&serdeval)?;
+        }
+        seq_ser.end()
+    }
+}
+
+struct AutoSerdeVal<'a>{
+    doc: &'a Automerge,
+    val: Value,
+    obj: ObjId,
+}
+
+impl<'a> serde::Serialize for AutoSerdeVal<'a> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+            S: serde::Serializer {
+        match &self.val {
+            Value::Object(ObjType::Map | ObjType::Table) => {
+                let map = AutoSerdeMap{doc: &self.doc, obj: self.obj.clone()};
+                map.serialize(serializer)
+            },
+            Value::Object(ObjType::List | ObjType::Text) => {
+                let seq = AutoSerdeSeq{doc: &self.doc, obj: self.obj.clone()};
+                seq.serialize(serializer)
+            },
+            Value::Scalar(v) => {
+                v.serialize(serializer)
+            }
+        }
+    }
+}

automerge/src/change_v2.rs

@@ -0,0 +1,179 @@
+use crate::{
+    columnar_2::{
+        rowblock::{
+            change_op_columns::{ChangeOp, ChangeOpsColumns},
+            RowBlock,
+        },
+        storage::{Change as StoredChange, Chunk, ChunkType},
+    },
+    types::{ActorId, ChangeHash},
+};
+
+#[derive(Clone, Debug)]
+pub struct Change {
+    stored: StoredChange<'static>,
+    hash: ChangeHash,
+    len: usize,
+}
+
+impl Change {
+    pub(crate) fn new(stored: StoredChange<'static>, hash: ChangeHash, len: usize) -> Self {
+        Self{
+            stored,
+            hash,
+            len,
+        }
+    }
+
+    pub fn actor_id(&self) -> &ActorId {
+        &self.stored.actor
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn max_op(&self) -> u64 {
+        self.stored.start_op + (self.len as u64) - 1
+    }
+
+    pub fn start_op(&self) -> u64 {
+        self.stored.start_op
+    }
+
+    pub fn message(&self) -> Option<&String> {
+        self.stored.message.as_ref()
+    }
+
+    pub fn deps(&self) -> &[ChangeHash] {
+        &self.stored.dependencies
+    }
+
+    pub fn hash(&self) -> ChangeHash {
+        self.hash
+    }
+
+    pub fn seq(&self) -> u64 {
+        self.stored.seq
+    }
+
+    pub fn timestamp(&self) -> i64 {
+        self.stored.timestamp
+    }
+
+    pub fn compress(&mut self) {}
+
+    pub fn raw_bytes(&self) -> Vec<u8> {
+        let vec = self.stored.write();
+        let chunk = Chunk::new_change(&vec);
+        chunk.write()
+    }
+
+    pub(crate) fn iter_ops<'a>(&'a self) -> impl Iterator<Item= ChangeOp<'a>> {
+        let rb = RowBlock::new(self.stored.ops_meta.iter(), self.stored.ops_data.clone()).unwrap();
+        let crb: RowBlock<ChangeOpsColumns> = rb.into_change_ops().unwrap();
+        let unwrapped = crb.into_iter().map(|r| r.unwrap().into_owned()).collect::<Vec<_>>();
+        return OperationIterator{
+            inner: unwrapped.into_iter(),
+        }
+    }
+
+    pub fn extra_bytes(&self) -> &[u8] {
+        self.stored.extra_bytes.as_ref()
+    }
+
+    // TODO replace all uses of this with TryFrom<&[u8]>
+    pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, LoadError> {
+        Self::try_from(&bytes[..])
+    }
+}
+
+struct OperationIterator<'a> {
+    inner: std::vec::IntoIter<ChangeOp<'a>>,
+}
+
+impl<'a> Iterator for OperationIterator<'a> {
+    type Item = ChangeOp<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+}
+
+impl AsRef<StoredChange<'static>> for Change {
+    fn as_ref(&self) -> &StoredChange<'static> {
+        &self.stored
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum LoadError {
+    #[error("unable to parse change: {0}")]
+    Parse(Box<dyn std::error::Error>),
+    #[error("leftover data after parsing")]
+    LeftoverData,
+    #[error("wrong chunk type")]
+    WrongChunkType,
+}
+
+impl<'a> TryFrom<&'a [u8]> for Change {
+    type Error = LoadError;
+
+    fn try_from(value: &'a [u8]) -> Result<Self, Self::Error> {
+        use crate::columnar_2::rowblock::change_op_columns::ReadChangeOpError;
+        let (remaining, chunk) = Chunk::parse(value).map_err(|e| LoadError::Parse(Box::new(e)))?;
+        if remaining.len() > 0 {
+            return Err(LoadError::LeftoverData);
+        }
+        match chunk.typ() {
+            ChunkType::Change => {
+                let chunkbytes = chunk.data();
+                let (_, c) = StoredChange::parse(chunkbytes.as_ref())
+                    .map_err(|e| LoadError::Parse(Box::new(e)))?;
+                let rb = RowBlock::new(c.ops_meta.iter(), c.ops_data.clone()).unwrap();
+                let crb: RowBlock<ChangeOpsColumns> = rb.into_change_ops().unwrap();
+                let mut iter = crb.into_iter();
+                let ops_len = iter
+                    .try_fold::<_, _, Result<_, ReadChangeOpError>>(0, |acc, op| {
+                        op?;
+                        Ok(acc + 1)
+                    })
+                    .map_err(|e| LoadError::Parse(Box::new(e)))?;
+                Ok(Self {
+                    stored: c.into_owned(),
+                    hash: chunk.hash(),
+                    len: ops_len,
+                })
+            }
+            _ => Err(LoadError::WrongChunkType),
+        }
+    }
+}
+
+impl<'a> TryFrom<StoredChange<'a>> for Change {
+    type Error = LoadError;
+
+    fn try_from(c: StoredChange) -> Result<Self, Self::Error> {
+        use crate::columnar_2::rowblock::change_op_columns::ReadChangeOpError;
+        let rb = RowBlock::new(c.ops_meta.iter(), c.ops_data.clone()).unwrap();
+        let crb: RowBlock<ChangeOpsColumns> = rb.into_change_ops().unwrap();
+        let mut iter = crb.into_iter();
+        let ops_len = iter
+            .try_fold::<_, _, Result<_, ReadChangeOpError>>(0, |acc, op| {
+                op?;
+                Ok(acc + 1)
+            })
+            .map_err(|e| LoadError::Parse(Box::new(e)))?;
+        let chunkbytes = c.write();
+        let chunk = Chunk::new_change(chunkbytes.as_ref());
+        Ok(Self {
+            stored: c.into_owned(),
+            hash: chunk.hash(),
+            len: ops_len,
+        })
+    }
+}

automerge/src/clock.rs

@@ -0,0 +1,52 @@
+use crate::types::OpId;
+use fxhash::FxBuildHasher;
+use std::cmp;
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct Clock(HashMap<usize, u64, FxBuildHasher>);
+
+impl Clock {
+    pub fn new() -> Self {
+        Clock(Default::default())
+    }
+
+    pub fn include(&mut self, key: usize, n: u64) {
+        self.0
+            .entry(key)
+            .and_modify(|m| *m = cmp::max(n, *m))
+            .or_insert(n);
+    }
+
+    pub fn covers(&self, id: &OpId) -> bool {
+        if let Some(val) = self.0.get(&id.1) {
+            val >= &id.0
+        } else {
+            false
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn covers() {
+        let mut clock = Clock::new();
+
+        clock.include(1, 20);
+        clock.include(2, 10);
+
+        assert!(clock.covers(&OpId(10, 1)));
+        assert!(clock.covers(&OpId(20, 1)));
+        assert!(!clock.covers(&OpId(30, 1)));
+
+        assert!(clock.covers(&OpId(5, 2)));
+        assert!(clock.covers(&OpId(10, 2)));
+        assert!(!clock.covers(&OpId(15, 2)));
+
+        assert!(!clock.covers(&OpId(1, 3)));
+        assert!(!clock.covers(&OpId(100, 3)));
+    }
+}

automerge/src/columnar_2/column_specification.rs

@@ -1,6 +1,3 @@
-/// An implementation of column specifications as specified in [1]
-///
-/// [1]: https://alexjg.github.io/automerge-storage-docs/#column-specifications
 #[derive(Eq, PartialEq, Clone, Copy)]
 pub(crate) struct ColumnSpec(u32);
 
@@ -28,14 +25,6 @@ impl ColumnSpec {
         self.0 & 0b00001000 > 0
     }
 
-    pub(crate) fn deflated(&self) -> Self {
-        Self::new(self.id(), self.col_type(), true)
-    }
-
-    pub(crate) fn inflated(&self) -> Self {
-        Self::new(self.id(), self.col_type(), false)
-    }
-
     pub(crate) fn normalize(&self) -> Normalized {
         Normalized(self.0 & 0b11110111)
     }
@@ -60,7 +49,7 @@ impl std::fmt::Debug for ColumnSpec {
 pub(crate) struct ColumnId(u32);
 
 impl ColumnId {
-    pub(crate) const fn new(raw: u32) -> Self {
+    pub const fn new(raw: u32) -> Self {
         ColumnId(raw)
     }
 }
@@ -77,9 +66,6 @@ impl std::fmt::Debug for ColumnId {
     }
 }
 
-/// The differente possible column types, as specified in [1]
-///
-/// [1]: https://alexjg.github.io/automerge-storage-docs/#column-specifications
 #[derive(Eq, PartialEq, Clone, Copy, Debug)]
 pub(crate) enum ColumnType {
     Group,

automerge/src/columnar_2/load.rs

@@ -0,0 +1,162 @@
+use std::collections::HashMap;
+
+use tracing::instrument;
+
+use super::{rowblock, storage};
+use crate::{op_set::OpSet, Change};
+
+mod change_collector;
+mod loading_document;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("unable to parse chunk: {0}")]
+    Parse(Box<dyn std::error::Error>),
+    #[error("invalid change columns: {0}")]
+    InvalidChangeColumns(Box<dyn std::error::Error>),
+    #[error("invalid ops columns: {0}")]
+    InvalidOpsColumns(Box<dyn std::error::Error>),
+    #[error("a chunk contained leftover data")]
+    LeftoverData,
+    #[error("error inflating document chunk ops: {0}")]
+    InflateDocument(Box<dyn std::error::Error>),
+    #[error("bad checksum")]
+    BadChecksum,
+}
+
+/// The result of `load_opset`. See the documentation for [`load_opset`] for details on why this is
+/// necessary
+pub(crate) enum LoadOpset {
+    /// The data was a "document" chunk so we loaded an op_set
+    Document {
+        /// The opset we loaded
+        op_set: OpSet,
+        /// The changes
+        history: Vec<Change>,
+        /// An index from history index to hash
+        history_index: HashMap<crate::types::ChangeHash, usize>,
+        /// An index from actor index to seq to change index
+        actor_to_history: HashMap<usize, Vec<usize>>,
+    },
+    /// The data was a change chunk so we just loaded the change
+    Change(Change),
+}
+
+/// The binary storage format defines several different "chunk types". When we're loading a
+/// document for the first time we wish to distinguish between "document" chunk types, and all the
+/// others. The reason for this is that the "document" chunk type contains operations encoded in a
+/// particular order which we can take advantage of to quickly load an OpSet. For all other chunk
+/// types we must proceed as usual by loading changes in order.
+///
+/// The tuple returned by this function contains as it's first component any data which was not
+/// consumed (i.e. data which could be more chunks) and as it's second component the [`LoadOpset`]
+/// which represents the two possible alternatives described above.
+#[instrument(level = "trace", skip(data))]
+pub(crate) fn load_opset<'a>(data: &'a [u8]) -> Result<(&'a [u8], LoadOpset), Error> {
+    let (remaining, chunk) = storage::Chunk::parse(data).map_err(|e| Error::Parse(Box::new(e)))?;
+    if !chunk.checksum_valid() {
+        return Err(Error::BadChecksum);
+    }
+    match chunk.typ() {
+        storage::ChunkType::Document => {
+            tracing::trace!("loading document chunk");
+            let data = chunk.data();
+            let (inner_remaining, doc) =
+                storage::Document::parse(&data).map_err(|e| Error::Parse(Box::new(e)))?;
+            if !inner_remaining.is_empty() {
+                tracing::error!(
+                    remaining = inner_remaining.len(),
+                    "leftover data when parsing document chunk"
+                );
+                return Err(Error::LeftoverData);
+            }
+            let change_rowblock =
+                rowblock::RowBlock::new(doc.change_metadata.iter(), doc.change_bytes)
+                    .map_err(|e| Error::InvalidChangeColumns(Box::new(e)))?
+                    .into_doc_change()
+                    .map_err(|e| Error::InvalidChangeColumns(Box::new(e)))?;
+
+            let ops_rowblock = rowblock::RowBlock::new(doc.op_metadata.iter(), doc.op_bytes)
+                .map_err(|e| Error::InvalidOpsColumns(Box::new(e)))?
+                .into_doc_ops()
+                .map_err(|e| Error::InvalidOpsColumns(Box::new(e)))?;
+
+            let loading_document::Loaded {
+                op_set,
+                history,
+                history_index,
+                actor_to_history,
+                ..
+            } = loading_document::load(
+                doc.actors,
+                doc.heads.into_iter().collect(),
+                change_rowblock.into_iter(),
+                ops_rowblock.into_iter(),
+            )
+            .map_err(|e| Error::InflateDocument(Box::new(e)))?;
+
+            // TODO: remove this unwrap because we already materialized all the ops
+            let history = history.into_iter().map(|h| h.try_into().unwrap()).collect();
+
+            Ok((
+                remaining,
+                LoadOpset::Document {
+                    op_set,
+                    history,
+                    history_index,
+                    actor_to_history,
+                },
+            ))
+        }
+        storage::ChunkType::Change => {
+            tracing::trace!("loading change chunk");
+            let data = chunk.data();
+            let (inner_remaining, change_chunk) =
+                storage::Change::parse(&data).map_err(|e| Error::Parse(Box::new(e)))?;
+            if !inner_remaining.is_empty() {
+                tracing::error!(
+                    remaining = inner_remaining.len(),
+                    "leftover data when parsing document chunk"
+                );
+                return Err(Error::LeftoverData);
+            }
+            let change_rowblock =
+                rowblock::RowBlock::new(change_chunk.ops_meta.iter(), change_chunk.ops_data.clone())
+                    .map_err(|e| Error::InvalidOpsColumns(Box::new(e)))?
+                    .into_change_ops()
+                    .map_err(|e| Error::InvalidOpsColumns(Box::new(e)))?;
+            let len = (&change_rowblock).into_iter().try_fold(0, |acc, c| {
+                c.map_err(|e| Error::InvalidChangeColumns(Box::new(e)))?;
+                Ok(acc + 1)
+            })?;
+            Ok((
+                remaining,
+                LoadOpset::Change(Change::new(change_chunk.into_owned(), chunk.hash(), len)),
+            ))
+        }
+        storage::ChunkType::Compressed => panic!(),
+    }
+}
+
+/// Load all the chunks in `data` returning a vector of changes. Note that this will throw an error
+/// if there is data left over.
+pub(crate) fn load(data: &[u8]) -> Result<Vec<Change>, Error> {
+    let mut changes = Vec::new();
+    let mut data = data;
+    while data.len() > 0 {
+        let (remaining, load_result) = load_opset(data)?;
+        match load_result {
+            LoadOpset::Change(c) => changes.push(c),
+            LoadOpset::Document { history, .. } => {
+                for stored_change in history {
+                    changes.push(
+                        Change::try_from(stored_change)
+                            .map_err(|e| Error::InvalidOpsColumns(Box::new(e)))?,
+                    );
+                }
+            }
+        }
+        data = remaining;
+    }
+    Ok(changes)
+}

automerge/src/columnar_2/load/change_collector.rs

@@ -0,0 +1,259 @@
+use std::{borrow::Cow, collections::{BTreeSet, HashMap}};
+
+use tracing::instrument;
+
+use crate::{
+    indexed_cache::IndexedCache,
+    columnar_2::{
+        rowblock::{
+            change_op_columns::{ChangeOp, ChangeOpsColumns},
+            doc_change_columns::ChangeMetadata,
+            Key as StoredKey, PrimVal,
+        },
+        storage::Change as StoredChange,
+    },
+    types::{ActorId, ChangeHash, ElemId, Key, Op, ObjId},
+    OpType,
+};
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("a change referenced an actor index we couldn't find")]
+    MissingActor,
+    #[error("changes out of order")]
+    ChangesOutOfOrder,
+    #[error("missing change")]
+    MissingChange,
+    #[error("some ops were missing")]
+    MissingOps,
+    #[error("unable to read change metadata: {0}")]
+    ReadChange(Box<dyn std::error::Error>),
+}
+
+pub(crate) struct ChangeCollector<'a> {
+    changes_by_actor: HashMap<usize, Vec<PartialChange<'a>>>,
+}
+
+pub(crate) struct CollectedChanges<'a> {
+    pub(crate) history: Vec<StoredChange<'a>>,
+    pub(crate) history_index: HashMap<ChangeHash, usize>,
+    pub(crate) actor_to_history: HashMap<usize, Vec<usize>>,
+    pub(crate) heads: BTreeSet<ChangeHash>,
+}
+
+impl<'a> ChangeCollector<'a> {
+    pub(crate) fn new<E: std::error::Error + 'static, I>(
+        changes: I,
+    ) -> Result<ChangeCollector<'a>, Error>
+    where
+        I: IntoIterator<Item = Result<ChangeMetadata<'a>, E>>,
+    {
+        let mut changes_by_actor: HashMap<usize, Vec<PartialChange<'_>>> = HashMap::new();
+        for (index, change) in changes.into_iter().enumerate() {
+            tracing::trace!(?change, "importing change metadata");
+            let change = change.map_err(|e| Error::ReadChange(Box::new(e)))?;
+            let actor_changes = changes_by_actor.entry(change.actor).or_default();
+            if let Some(prev) = actor_changes.last() {
+                if prev.max_op >= change.max_op {
+                    return Err(Error::ChangesOutOfOrder);
+                }
+            }
+            actor_changes.push(PartialChange {
+                index,
+                deps: change.deps,
+                actor: change.actor,
+                seq: change.seq,
+                timestamp: change.timestamp,
+                max_op: change.max_op,
+                message: change.message,
+                extra_bytes: change.extra,
+                ops: Vec::new(),
+            })
+        }
+        let num_changes: usize = changes_by_actor.values().map(|v| v.len()).sum();
+        tracing::trace!(num_changes, ?changes_by_actor, "change collection context created");
+        Ok(ChangeCollector { changes_by_actor })
+    }
+
+    #[instrument(skip(self))]
+    pub(crate) fn collect(&mut self, obj: ObjId, op: Op) -> Result<(), Error> {
+        let actor_changes = self
+            .changes_by_actor
+            .get_mut(&op.id.actor())
+            .ok_or_else(||{
+                tracing::error!(missing_actor=op.id.actor(), "missing actor for op");
+                Error::MissingActor
+            })?;
+        let change_index = actor_changes.partition_point(|c| c.max_op < op.id.counter());
+        let change = actor_changes
+            .get_mut(change_index)
+            .ok_or_else(||{
+                tracing::error!(missing_change_index=change_index, "missing change for op");
+                Error::MissingChange
+            })?;
+        change.ops.push((obj, op));
+        Ok(())
+    }
+
+    #[instrument(skip(self, actors, props))]
+    pub(crate) fn finish(
+        self,
+        actors: &IndexedCache<ActorId>,
+        props: &IndexedCache<String>,
+    ) -> Result<CollectedChanges<'static>, Error> {
+        let mut changes_in_order =
+            Vec::with_capacity(self.changes_by_actor.values().map(|c| c.len()).sum());
+        for (_, changes) in self.changes_by_actor {
+            let mut start_op = 0;
+            let mut seq = None;
+            for change in changes {
+                if change.max_op != start_op + (change.ops.len() as u64) {
+                    tracing::error!(?change, start_op, "missing operations");
+                    return Err(Error::MissingOps);
+                } else {
+                    start_op = change.max_op;
+                }
+                if let Some(seq) = seq {
+                    if seq != change.seq - 1 {
+                        return Err(Error::ChangesOutOfOrder);
+                    }
+                } else if change.seq != 1 {
+                    return Err(Error::ChangesOutOfOrder);
+                }
+                seq = Some(change.seq);
+                changes_in_order.push(change);
+            }
+        }
+        changes_in_order.sort_by_key(|c| c.index);
+
+        let mut hashes_by_index = HashMap::new();
+        let mut history = Vec::new();
+        let mut actor_to_history: HashMap<usize, Vec<usize>> = HashMap::new();
+        let mut heads = BTreeSet::new();
+        for (index, change) in changes_in_order.into_iter().enumerate() {
+            actor_to_history
+                .entry(change.actor)
+                .or_default()
+                .push(index);
+            let finished = change.finish(&hashes_by_index, actors, props)?;
+            let hash = finished.hash();
+            hashes_by_index.insert(index, hash);
+            for dep in &finished.dependencies {
+                heads.remove(dep);
+            }
+            tracing::trace!(?hash, "processing change hash");
+            heads.insert(hash);
+            history.push(finished.into_owned());
+        }
+
+        let indices_by_hash = hashes_by_index.into_iter().map(|(k, v)| (v, k)).collect();
+        Ok(CollectedChanges {
+            history,
+            history_index: indices_by_hash,
+            actor_to_history,
+            heads,
+        })
+    }
+}
+
+#[derive(Debug)]
+struct PartialChange<'a> {
+    index: usize,
+    deps: Vec<u64>,
+    actor: usize,
+    seq: u64,
+    max_op: u64,
+    timestamp: i64,
+    message: Option<smol_str::SmolStr>,
+    extra_bytes: Cow<'a, [u8]>,
+    ops: Vec<(ObjId, Op)>,
+}
+
+impl<'a> PartialChange<'a> {
+    /// # Panics
+    ///
+    /// If any op references a property index which is not in `props`
+    #[instrument(skip(self, known_changes, actors, props))]
+    fn finish(
+        self,
+        known_changes: &HashMap<usize, ChangeHash>,
+        actors: &IndexedCache<ActorId>,
+        props: &IndexedCache<String>,
+    ) -> Result<StoredChange<'a>, Error> {
+        let deps_len = self.deps.len();
+        let mut deps =
+            self.deps
+                .into_iter()
+                .try_fold(Vec::with_capacity(deps_len), |mut acc, dep| {
+                    acc.push(
+                        known_changes
+                            .get(&(dep as usize))
+                            .cloned()
+                            .ok_or_else(|| {
+                                tracing::error!(dependent_index=self.index, dep_index=dep, "could not find dependency");
+                                Error::MissingChange
+                            })?,
+                    );
+                    Ok(acc)
+                })?;
+        deps.sort();
+        let other_actors =
+            self.ops
+                .iter()
+                .try_fold(Vec::with_capacity(self.ops.len()), |mut acc, (_, op)| {
+                    match op.key {
+                        Key::Seq(ElemId(elem)) => {
+                            if elem.actor() != self.actor {
+                                acc.push(
+                                    actors
+                                        .safe_get(elem.actor())
+                                        .cloned()
+                                        .ok_or(Error::MissingActor)?,
+                                );
+                            }
+                        }
+                        Key::Map(_) => {}
+                    };
+                    Ok(acc)
+                })?;
+        let mut ops_data = Vec::new();
+        let num_ops = self.ops.len() as u64;
+        let columns = ChangeOpsColumns::empty().encode(
+            self.ops.into_iter().map(|(obj, op)| {
+                let action_index = op.action.action_index();
+                ChangeOp {
+                    key: match op.key {
+                        // SAFETY: The caller must ensure that all props in the ops are in the propmap
+                        Key::Map(idx) => StoredKey::Prop(props.safe_get(idx).unwrap().into()),
+                        Key::Seq(elem) => StoredKey::Elem(elem),
+                    },
+                    insert: op.insert,
+                    val: match op.action {
+                        OpType::Make(_) | OpType::Del => PrimVal::Null,
+                        OpType::Inc(i) => PrimVal::Int(i),
+                        OpType::Set(v) => v.into(),
+                    },
+                    action: action_index,
+                    pred: op.pred,
+                    obj,
+                }
+            }),
+            &mut ops_data,
+        );
+        Ok(StoredChange {
+            dependencies: deps,
+            actor: actors
+                .safe_get(self.actor)
+                .cloned()
+                .ok_or(Error::MissingActor)?,
+            other_actors,
+            seq: self.seq,
+            start_op: self.max_op - num_ops,
+            timestamp: self.timestamp,
+            message: self.message.map(|s| s.to_string()),
+            ops_meta: columns.metadata(),
+            ops_data: Cow::Owned(ops_data),
+            extra_bytes: self.extra_bytes,
+        })
+    }
+}

automerge/src/columnar_2/load/loading_document.rs

@@ -0,0 +1,241 @@
+use fxhash::FxBuildHasher;
+use std::collections::{HashMap, BTreeSet};
+use tracing::instrument;
+use super::change_collector::ChangeCollector;
+
+use crate::{
+    columnar_2::{
+        storage::Change as StoredChange,
+        rowblock::{
+            Key as DocOpKey,
+            doc_change_columns::ChangeMetadata,
+            doc_op_columns::DocOp,
+            PrimVal,
+        }
+    },
+    op_set::OpSet,
+    op_tree::{OpSetMetadata, OpTree},
+    types::{ActorId, ChangeHash, ElemId, Key, ObjId, ObjType, Op, OpId, OpType},
+};
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("the document contained ops which were out of order")]
+    OpsOutOfOrder,
+    #[error("error reading operation: {0:?}")]
+    ReadOp(Box<dyn std::error::Error>),
+    #[error("an operation contained an invalid action")]
+    InvalidAction,
+    #[error("an operation referenced a missing actor id")]
+    MissingActor,
+    #[error("invalid changes: {0}")]
+    InvalidChanges(#[from] super::change_collector::Error),
+    #[error("mismatching heads")]
+    MismatchingHeads,
+}
+
+struct LoadingObject {
+    id: ObjId,
+    ops: Vec<Op>,
+    obj_type: ObjType,
+    preds: HashMap<OpId, Vec<OpId>>,
+}
+
+impl LoadingObject {
+    fn root() -> Self {
+        LoadingObject {
+            id: ObjId::root(),
+            ops: Vec::new(),
+            obj_type: ObjType::Map,
+            preds: HashMap::new(),
+        }
+    }
+
+    fn new(id: ObjId, obj_type: ObjType) -> Self {
+        LoadingObject {
+            id: id.into(),
+            ops: Vec::new(),
+            obj_type,
+            preds: HashMap::new(),
+        }
+    }
+
+    fn append_op(&mut self, op: Op) -> Result<(), Error> {
+        if let Some(previous_op) = self.ops.last() {
+            if op.key < previous_op.key {
+                tracing::error!(
+                    ?op,
+                    ?previous_op,
+                    "op key was smaller than key of previous op"
+                );
+                return Err(Error::OpsOutOfOrder);
+            }
+        }
+        for succ in &op.succ {
+            self.preds.entry(*succ).or_default().push(op.id);
+        }
+        self.ops.push(op);
+        Ok(())
+    }
+
+    fn finish(mut self) -> (ObjId, ObjType, OpTree) {
+        let mut op_tree = OpTree::new();
+        for (index, mut op) in self.ops.into_iter().enumerate() {
+            if let Some(preds) = self.preds.remove(&op.id) {
+                op.pred = preds;
+            }
+            op_tree.insert(index, op);
+        }
+        (self.id, self.obj_type, op_tree)
+    }
+}
+
+pub(crate) struct Loaded<'a> {
+    pub(crate) op_set: OpSet,
+    pub(crate) history: Vec<StoredChange<'a>>,
+    pub(crate) history_index: HashMap<ChangeHash, usize>,
+    pub(crate) actor_to_history: HashMap<usize, Vec<usize>>,
+}
+
+#[instrument(skip(actors, expected_heads, changes, ops))]
+pub(crate) fn load<'a, I, C, OE, CE>(
+    actors: Vec<ActorId>,
+    expected_heads: BTreeSet<ChangeHash>,
+    changes: C,
+    ops: I,
+) -> Result<Loaded<'static>, Error>
+where
+    OE: std::error::Error + 'static,
+    CE: std::error::Error + 'static,
+    I: Iterator<Item = Result<DocOp<'a>, OE>>,
+    C: Iterator<Item = Result<ChangeMetadata<'a>, CE>>,
+{
+    let mut metadata = OpSetMetadata::from_actors(actors);
+    let mut completed_objects = HashMap::<_, _, FxBuildHasher>::default();
+    let mut current_object = LoadingObject::root();
+    let mut collector = ChangeCollector::new(changes)?;
+    let mut obj_types = HashMap::new();
+    obj_types.insert(ObjId::root(), ObjType::Map);
+    for op_res in ops {
+        let doc_op = op_res.map_err(|e| Error::ReadOp(Box::new(e)))?;
+        let obj = doc_op.object;
+        let op = import_op(&mut metadata, doc_op)?;
+        tracing::trace!(?op, "processing op");
+        collector.collect(current_object.id, op.clone())?;
+
+        // We have to record the object types of make operations so that when the object ID the
+        // incoming operations refer to switches we can lookup the object type for the new object.
+        // Ultimately we need this because the OpSet needs to know the object ID _and type_ for
+        // each OpTree it tracks.
+        if obj == current_object.id {
+            match op.action {
+                OpType::Make(obj_type) => {
+                    obj_types.insert(op.id.into(), obj_type.clone());
+                }
+                _ => {}
+            };
+            current_object.append_op(op)?;
+        } else {
+            let new_obj_type = match obj_types.get(&obj) {
+                Some(t) => Ok(t.clone()),
+                None => {
+                    tracing::error!(
+                        ?op,
+                        "operation referenced an object which we haven't seen a create op for yet"
+                    );
+                    Err(Error::OpsOutOfOrder)
+                }
+            }?;
+            if obj < current_object.id {
+                tracing::error!(?op, previous_obj=?current_object.id, "op referenced an object ID which was less than the previous object ID");
+                return Err(Error::OpsOutOfOrder);
+            } else {
+                let (id, obj_type, op_tree) = current_object.finish();
+                current_object = LoadingObject::new(obj, new_obj_type);
+                current_object.append_op(op)?;
+                completed_objects.insert(id, (obj_type, op_tree));
+            }
+        }
+    }
+    let super::change_collector::CollectedChanges{
+        history,
+        history_index,
+        actor_to_history,
+        heads,
+    } = collector.finish(
+        &metadata.actors,
+        &metadata.props,
+    )?;
+    if expected_heads != heads {
+        tracing::error!(?expected_heads, ?heads, "mismatching heads");
+        return Err(Error::MismatchingHeads);
+    }
+    let (id, obj_type, op_tree) = current_object.finish();
+    completed_objects.insert(id, (obj_type, op_tree));
+    let op_set = OpSet::from_parts(completed_objects, metadata);
+
+    Ok(Loaded {
+        op_set,
+        history,
+        history_index,
+        actor_to_history,
+    })
+}
+
+#[instrument(skip(m))]
+fn import_op<'a>(m: &mut OpSetMetadata, op: DocOp<'a>) -> Result<Op, Error> {
+    let key = match op.key {
+        DocOpKey::Prop(s) => Key::Map(m.import_prop(s)),
+        DocOpKey::Elem(ElemId(op)) => Key::Seq(ElemId(check_opid(m, op)?)),
+    };
+    for opid in &op.succ {
+        if m.actors.safe_get(opid.actor()).is_none() {
+            tracing::error!(?opid, "missing actor");
+            return Err(Error::MissingActor);
+        }
+    }
+    Ok(Op {
+        id: check_opid(m, op.id)?,
+        action: parse_optype(op.action, op.value)?,
+        key,
+        succ: op.succ,
+        pred: Vec::new(),
+        insert: op.insert,
+    })
+}
+
+/// We construct the OpSetMetadata directly from the vector of actors which are encoded in the
+/// start of the document. Therefore we need to check for each opid in the docuemnt that the actor
+/// ID which it references actually exists in the metadata.
+#[tracing::instrument(skip(m))]
+fn check_opid(m: &OpSetMetadata, opid: OpId) -> Result<OpId, Error> {
+    match m.actors.safe_get(opid.actor()) {
+        Some(_) => Ok(opid),
+        None => {
+            tracing::error!("missing actor");
+            Err(Error::MissingActor)
+        }
+    }
+}
+
+fn parse_optype<'a>(action_index: usize, value: PrimVal<'a>) -> Result<OpType, Error> {
+    match action_index {
+        0 => Ok(OpType::Make(ObjType::Map)),
+        1 => Ok(OpType::Set(value.into())),
+        2 => Ok(OpType::Make(ObjType::List)),
+        3 => Ok(OpType::Del),
+        4 => Ok(OpType::Make(ObjType::Text)),
+        5 => match value {
+            PrimVal::Int(i) => Ok(OpType::Inc(i)),
+            _ => {
+                tracing::error!(?value, "invalid value for counter op");
+                Err(Error::InvalidAction)
+            }
+        },
+        6 => Ok(OpType::Make(ObjType::Table)),
+        other => {
+            tracing::error!(action = other, "unknown action type");
+            Err(Error::InvalidAction)
+        }
+    }
+}

automerge/src/columnar_2/mod.rs

@@ -0,0 +1,8 @@
+mod column_specification;
+#[cfg(feature = "storage-v2")]
+pub(crate) mod load;
+#[cfg(feature = "storage-v2")]
+pub(crate) mod save;
+pub(crate) mod rowblock;
+pub(crate) mod storage;
+pub(crate) use column_specification::{ColumnId, ColumnSpec};

automerge/src/columnar_2/rowblock/column_layout/change_op_columns.rs

@@ -0,0 +1,454 @@
+use std::{borrow::Borrow, convert::TryFrom, ops::Range};
+
+use crate::{
+    columnar_2::{
+        column_specification::ColumnType,
+        rowblock::{
+            column_layout::{column::{GroupColRange, ColumnRanges}, ColumnLayout, MismatchingColumn, assert_col_type},
+            column_range::{
+                ActorRange, BooleanRange, DeltaIntRange, RawRange, RleIntRange, RleStringRange,
+            },
+            encoding::{
+                BooleanDecoder, DecodeColumnError, Key, KeyDecoder, ObjDecoder,
+                OpIdListDecoder, RleDecoder, ValueDecoder,
+            },
+            PrimVal,
+        }, ColumnSpec, ColumnId, storage::ColumnMetadata
+    },
+    types::{ElemId, ObjId, OpId},
+};
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct ChangeOp<'a> {
+    pub(crate) key: Key,
+    pub(crate) insert: bool,
+    pub(crate) val: PrimVal<'a>,
+    pub(crate) pred: Vec<OpId>,
+    pub(crate) action: u64,
+    pub(crate) obj: ObjId,
+}
+
+impl<'a> ChangeOp<'a> {
+    pub(crate) fn into_owned(self) -> ChangeOp<'static> {
+        ChangeOp {
+            key: self.key,
+            insert: self.insert,
+            val: self.val.into_owned(),
+            pred: self.pred,
+            action: self.action,
+            obj: self.obj,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct ChangeOpsColumns {
+    obj_actor: ActorRange,
+    obj_counter: RleIntRange,
+    key_actor: ActorRange,
+    key_counter: DeltaIntRange,
+    key_string: RleStringRange,
+    insert: BooleanRange,
+    action: RleIntRange,
+    val_meta: RleIntRange,
+    val_raw: RawRange,
+    pred_group: RleIntRange,
+    pred_actor: RleIntRange,
+    pred_ctr: DeltaIntRange,
+}
+
+impl ChangeOpsColumns {
+    pub(crate) fn empty() -> Self {
+        ChangeOpsColumns {
+            obj_actor: (0..0).into(),
+            obj_counter: (0..0).into(),
+            key_actor: (0..0).into(),
+            key_counter: (0..0).into(),
+            key_string: (0..0).into(),
+            insert: (0..0).into(),
+            action: (0..0).into(),
+            val_meta: (0..0).into(),
+            val_raw: (0..0).into(),
+            pred_group: (0..0).into(),
+            pred_actor: (0..0).into(),
+            pred_ctr: (0..0).into(),
+        }
+    }
+
+    pub(crate) fn iter<'a>(&self, data: &'a [u8]) -> ChangeOpsIter<'a> {
+        ChangeOpsIter {
+            failed: false,
+            obj: ObjDecoder::new(self.obj_actor.decoder(data), self.obj_counter.decoder(data)),
+            key: KeyDecoder::new(
+                self.key_actor.decoder(data),
+                self.key_counter.decoder(data),
+                self.key_string.decoder(data),
+            ),
+            insert: self.insert.decoder(data),
+            action: self.action.decoder(data),
+            val: ValueDecoder::new(self.val_meta.decoder(data), self.val_raw.decoder(data)),
+            pred: OpIdListDecoder::new(
+                self.pred_group.decoder(data),
+                self.pred_actor.decoder(data),
+                self.pred_ctr.decoder(data),
+            ),
+        }
+    }
+
+    pub(crate) fn encode<'a, I, C: Borrow<ChangeOp<'a>>>(&self, ops: I, out: &mut Vec<u8>) -> ChangeOpsColumns
+    where
+        I: Iterator<Item = C> + Clone,
+    {
+        let obj_actor = self.obj_actor.decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(OpId::from(o.borrow().obj).actor() as u64)),
+            out,
+        );
+        let obj_counter = self.obj_counter.decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(OpId::from(o.borrow().obj).counter())),
+            out,
+        );
+        let key_actor = self.key_actor.decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match o.borrow().key {
+                Key::Prop(_) => None,
+                Key::Elem(ElemId(o)) => Some(o.actor() as u64),
+            }),
+            out,
+        );
+        let key_counter = self.key_counter.decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match o.borrow().key {
+                Key::Prop(_) => None,
+                Key::Elem(ElemId(o)) => Some(o.counter() as i64),
+            }),
+            out,
+        );
+        let key_string = self.key_string.decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match &o.borrow().key {
+                Key::Prop(k) => Some(k.clone()),
+                Key::Elem(_) => None,
+            }),
+            out,
+        );
+        let insert = self
+            .insert
+            .decoder(&[])
+            .splice(0..0, ops.clone().map(|o| o.borrow().insert), out);
+        let action =
+            self.action
+                .decoder(&[])
+                .splice(0..0, ops.clone().map(|o| Some(o.borrow().action)), out);
+        let mut val_dec = ValueDecoder::new(self.val_meta.decoder(&[]), self.val_raw.decoder(&[]));
+        let (val_meta, val_raw) = val_dec.splice(0..0, ops.clone().map(|o| o.borrow().val.clone()), out);
+        let mut pred_dec = OpIdListDecoder::new(
+            self.pred_group.decoder(&[]),
+            self.pred_actor.decoder(&[]),
+            self.pred_ctr.decoder(&[]),
+        );
+        let (pred_group, pred_actor, pred_ctr) =
+            pred_dec.splice(0..0, ops.map(|o| o.borrow().pred.clone()), out);
+        Self {
+            obj_actor: obj_actor.into(),
+            obj_counter: obj_counter.into(),
+            key_actor: key_actor.into(),
+            key_counter: key_counter.into(),
+            key_string: key_string.into(),
+            insert: insert.into(),
+            action: action.into(),
+            val_meta: val_meta.into(),
+            val_raw: val_raw.into(),
+            pred_group: pred_group.into(),
+            pred_actor: pred_actor.into(),
+            pred_ctr: pred_ctr.into(),
+        }
+    }
+
+    pub(crate) fn metadata(&self) -> ColumnMetadata {
+        const OBJ_COL_ID: ColumnId = ColumnId::new(0);
+        const KEY_COL_ID: ColumnId = ColumnId::new(1);
+        const INSERT_COL_ID: ColumnId = ColumnId::new(3);
+        const ACTION_COL_ID: ColumnId = ColumnId::new(4);
+        const VAL_COL_ID: ColumnId = ColumnId::new(5);
+        const PRED_COL_ID: ColumnId = ColumnId::new(7); 
+        
+        let mut cols = vec![
+            (ColumnSpec::new(OBJ_COL_ID, ColumnType::Actor, false), self.obj_actor.clone().into()),
+            (ColumnSpec::new(OBJ_COL_ID, ColumnType::Integer, false), self.obj_counter.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::Actor, false), self.key_actor.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::DeltaInteger, false), self.key_counter.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::String, false), self.key_string.clone().into()),
+            (ColumnSpec::new(INSERT_COL_ID, ColumnType::Boolean, false), self.insert.clone().into()),
+            (ColumnSpec::new(ACTION_COL_ID, ColumnType::Integer, false), self.action.clone().into()),
+            (ColumnSpec::new(VAL_COL_ID, ColumnType::ValueMetadata, false), self.val_meta.clone().into()),
+        ];
+        if self.val_raw.len() > 0 {
+            cols.push((
+                ColumnSpec::new(VAL_COL_ID, ColumnType::Value, false), self.val_raw.clone().into()
+            ));
+        }
+        cols.push(
+            (ColumnSpec::new(PRED_COL_ID, ColumnType::Group, false), self.pred_group.clone().into()),
+        );
+        if self.pred_actor.len() > 0 {
+            cols.extend([
+                (ColumnSpec::new(PRED_COL_ID, ColumnType::Actor, false), self.pred_actor.clone().into()),
+                (ColumnSpec::new(PRED_COL_ID, ColumnType::DeltaInteger, false), self.pred_ctr.clone().into()),
+            ]);
+        }
+        cols.into_iter().collect()
+    }
+}
+
+
+#[derive(thiserror::Error, Debug)]
+pub(crate) enum ReadChangeOpError {
+    #[error("unexpected null in column {0}")]
+    UnexpectedNull(String),
+    #[error("invalid value in column {column}: {description}")]
+    InvalidValue { column: String, description: String },
+}
+
+pub(crate) struct ChangeOpsIter<'a> {
+    failed: bool,
+    obj: ObjDecoder<'a>,
+    key: KeyDecoder<'a>,
+    insert: BooleanDecoder<'a>,
+    action: RleDecoder<'a, u64>,
+    val: ValueDecoder<'a>,
+    pred: OpIdListDecoder<'a>,
+}
+
+impl<'a> ChangeOpsIter<'a> {
+    fn done(&self) -> bool {
+        [
+            self.obj.done(),
+            self.key.done(),
+            self.insert.done(),
+            self.action.done(),
+            self.val.done(),
+            self.pred.done(),
+        ]
+        .iter()
+        .all(|e| *e)
+    }
+
+    fn try_next(&mut self) -> Result<Option<ChangeOp<'a>>, ReadChangeOpError> {
+        if self.failed {
+            Ok(None)
+        } else if self.done() {
+            Ok(None)
+        } else {
+            let obj = self
+                .obj
+                .next()
+                .transpose()
+                .map_err(|e| self.handle_error("object", e))?
+                .ok_or(ReadChangeOpError::UnexpectedNull("object".to_string()))?;
+            let key = self
+                .key
+                .next()
+                .transpose()
+                .map_err(|e| self.handle_error("key", e))?
+                .ok_or(ReadChangeOpError::UnexpectedNull("key".to_string()))?;
+            let insert = self
+                .insert
+                .next()
+                .ok_or(ReadChangeOpError::UnexpectedNull("insert".to_string()))?;
+            let action = self
+                .action
+                .next()
+                .flatten()
+                .ok_or(ReadChangeOpError::UnexpectedNull("action".to_string()))?;
+            let val = self
+                .val
+                .next()
+                .transpose()
+                .map_err(|e| self.handle_error("value", e))?
+                .ok_or(ReadChangeOpError::UnexpectedNull("value".to_string()))?;
+            let pred = self
+                .pred
+                .next()
+                .transpose()
+                .map_err(|e| self.handle_error("pred", e))?
+                .ok_or(ReadChangeOpError::UnexpectedNull("pred".to_string()))?;
+            Ok(Some(ChangeOp {
+                obj: obj.into(),
+                key,
+                insert,
+                action,
+                val,
+                pred,
+            }))
+        }
+    }
+
+    fn handle_error(
+        &mut self,
+        outer_col: &'static str,
+        err: DecodeColumnError,
+    ) -> ReadChangeOpError {
+        match err {
+            DecodeColumnError::InvalidValue {
+                column,
+                description,
+            } => ReadChangeOpError::InvalidValue {
+                column: format!("{}:{}", outer_col, column),
+                description,
+            },
+            DecodeColumnError::UnexpectedNull(col) => {
+                ReadChangeOpError::UnexpectedNull(format!("{}:{}", outer_col, col))
+            }
+        }
+    }
+}
+
+impl<'a> Iterator for ChangeOpsIter<'a> {
+    type Item = Result<ChangeOp<'a>, ReadChangeOpError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.try_next() {
+            Ok(v) => v.map(Ok),
+            Err(e) => {
+                self.failed = true;
+                Some(Err(e))
+            }
+        }
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub(crate) enum ParseChangeColumnsError {
+    #[error("mismatching column at {index}.")]
+    MismatchingColumn { index: usize },
+    #[error("not enough columns")]
+    NotEnoughColumns,
+}
+
+impl From<MismatchingColumn> for ParseChangeColumnsError {
+    fn from(m: MismatchingColumn) -> Self {
+        Self::MismatchingColumn{index: m.index} 
+    }
+}
+
+impl TryFrom<ColumnLayout> for ChangeOpsColumns {
+    type Error = ParseChangeColumnsError;
+
+    fn try_from(columns: ColumnLayout) -> Result<Self, Self::Error> {
+        let mut obj_actor: Option<Range<usize>> = None;
+        let mut obj_ctr: Option<Range<usize>> = None;
+        let mut key_actor: Option<Range<usize>> = None;
+        let mut key_ctr: Option<Range<usize>> = None;
+        let mut key_str: Option<Range<usize>> = None;
+        let mut insert: Option<Range<usize>> = None;
+        let mut action: Option<Range<usize>> = None;
+        let mut val_meta: Option<Range<usize>> = None;
+        let mut val_raw: Option<Range<usize>> = None;
+        let mut pred_group: Option<Range<usize>> = None;
+        let mut pred_actor: Option<Range<usize>> = None;
+        let mut pred_ctr: Option<Range<usize>> = None;
+        let mut other = ColumnLayout::empty();
+
+        for (index, col) in columns.into_iter().enumerate() {
+            match index {
+                0 => assert_col_type(index, col, ColumnType::Actor, &mut obj_actor)?,
+                1 => assert_col_type(index, col, ColumnType::Integer, &mut obj_ctr)?,
+                2 => assert_col_type(index, col, ColumnType::Actor, &mut key_actor)?,
+                3 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut key_ctr)?,
+                4 => assert_col_type(index, col, ColumnType::String, &mut key_str)?,
+                5 => assert_col_type(index, col, ColumnType::Boolean, &mut insert)?,
+                6 => assert_col_type(index, col, ColumnType::Integer, &mut action)?,
+                7 => match col.ranges() {
+                    ColumnRanges::Value{meta, val} => {
+                        val_meta = Some(meta);
+                        val_raw = Some(val);
+                    },
+                    _ => return Err(ParseChangeColumnsError::MismatchingColumn{ index }),
+                },
+                8 => match col.ranges() {
+                    ColumnRanges::Group{num, mut cols} => {
+                        pred_group = Some(num.into());
+                        // If there was no data in the group at all then the columns won't be
+                        // present
+                        if cols.len() == 0 {
+                            pred_actor = Some((0..0).into());
+                            pred_ctr = Some((0..0).into());
+                        } else {
+                            let first = cols.next();
+                            let second = cols.next();
+                            match (first, second) {
+                                (Some(GroupColRange::Single(actor_range)), Some(GroupColRange::Single(ctr_range))) =>
+                                {
+                                    pred_actor = Some(actor_range.into());
+                                    pred_ctr = Some(ctr_range.into());
+                                },
+                                _ => return Err(ParseChangeColumnsError::MismatchingColumn{ index }),
+                            }
+                        }
+                        if let Some(_) = cols.next() {
+                            return Err(ParseChangeColumnsError::MismatchingColumn{ index });
+                        }
+                    },
+                    _ => return Err(ParseChangeColumnsError::MismatchingColumn{ index }),
+                },
+                _ => {
+                    other.append(col);
+                }
+            }
+        }
+        Ok(ChangeOpsColumns {
+            obj_actor: obj_actor.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            obj_counter: obj_ctr.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            key_actor: key_actor.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            key_counter: key_ctr.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            key_string: key_str.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            insert: insert.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            action: action.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            val_meta: val_meta.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            val_raw: val_raw.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            pred_group: pred_group.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            pred_actor: pred_actor.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+            pred_ctr: pred_ctr.ok_or(ParseChangeColumnsError::NotEnoughColumns)?.into(),
+        })
+    }
+
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::columnar_2::rowblock::encoding::properties::{key, opid, value};
+    use proptest::prelude::*;
+
+    prop_compose! {
+        fn change_op()
+                    (key in key(),
+                     value in value(),
+                     pred in proptest::collection::vec(opid(), 0..20),
+                     action in 0_u64..6,
+                     obj in opid(),
+                     insert in any::<bool>()) -> ChangeOp<'static> {
+            ChangeOp {
+                obj: obj.into(),
+                key,
+                val: value,
+                pred,
+                action,
+                insert,
+            }
+        }
+    }
+
+    proptest! {
+        #[test]
+        fn test_encode_decode_change_ops(ops in proptest::collection::vec(change_op(), 0..100)) {
+            let cols = ChangeOpsColumns::empty();
+            let mut out = Vec::new();
+            let cols2 = cols.encode(ops.iter(), &mut out);
+            let decoded = cols2.iter(&out[..]).collect::<Result<Vec<_>, _>>().unwrap();
+            assert_eq!(ops, decoded);
+        }
+    }
+}

automerge/src/columnar_2/rowblock/column_layout/column.rs

@@ -0,0 +1,603 @@
+use std::{borrow::Cow, ops::Range};
+
+use super::{
+    super::{
+        encoding::{
+            generic::{GenericColDecoder, GroupDecoder, SingleLogicalColDecoder},
+            RawDecoder, RleDecoder, RleEncoder, SimpleColDecoder, ValueDecoder, BooleanDecoder, DeltaDecoder,
+        },
+        CellValue, ColumnId, ColumnSpec,
+    },
+    ColumnSpliceError,
+};
+
+use crate::columnar_2::column_specification::ColumnType;
+
+#[derive(Debug)]
+pub(crate) struct Column(ColumnInner);
+
+impl Column {
+    pub(crate) fn range(&self) -> Range<usize> {
+        self.0.range()
+    }
+
+    pub(crate) fn ranges<'a>(&'a self) -> ColumnRanges<'a> {
+        self.0.ranges()
+    }
+
+    pub(crate) fn decoder<'a>(&self, data: &'a [u8]) -> GenericColDecoder<'a> {
+        self.0.decoder(data)
+    }
+
+    pub(crate) fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        output: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        Ok(Self(self.0.splice(source, output, replace, replace_with)?))
+    }
+
+    pub(crate) fn col_type(&self) -> ColumnType {
+        self.0.col_type()
+    }
+
+    pub fn id(&self) -> ColumnId {
+        match self.0 {
+            ColumnInner::Single(SingleColumn { spec: s, .. }) => s.id(),
+            ColumnInner::Composite(CompositeColumn::Value(ValueColumn { spec, .. })) => spec.id(),
+            ColumnInner::Composite(CompositeColumn::Group(GroupColumn { spec, .. })) => spec.id(),
+        }
+    }
+
+    pub(crate) fn spec(&self) -> ColumnSpec {
+        match &self.0 {
+            ColumnInner::Single(s) => s.spec,
+            ColumnInner::Composite(CompositeColumn::Value(v)) => v.spec,
+            ColumnInner::Composite(CompositeColumn::Group(g)) => g.spec,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+enum ColumnInner {
+    Single(SingleColumn),
+    Composite(CompositeColumn),
+}
+
+pub(crate) enum ColumnRanges<'a> {
+    Single(Range<usize>),
+    Group{
+        num: Range<usize>,
+        cols: ColRangeIter<'a>,
+    },
+    Value {
+        meta: Range<usize>,
+        val: Range<usize>,
+    }
+}
+
+pub(crate) enum GroupColRange {
+    Single(Range<usize>),
+    Value{
+        meta: Range<usize>,
+        val: Range<usize>,
+    }
+}
+
+pub(crate) struct ColRangeIter<'a> {
+    offset: usize,
+    cols: &'a [GroupedColumn]
+}
+
+impl<'a> Iterator for ColRangeIter<'a> {
+    type Item = GroupColRange;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.cols.get(self.offset) {
+            None => None,
+            Some(GroupedColumn::Single(SingleColumn{range, ..})) => {
+                self.offset += 1;
+                Some(GroupColRange::Single(range.clone()))
+            },
+            Some(GroupedColumn::Value(ValueColumn{meta, value, ..})) => {
+                self.offset += 1;
+                Some(GroupColRange::Value{meta: meta.clone(), val: value.clone()})
+            }
+        }
+    }
+}
+
+impl<'a> ExactSizeIterator for ColRangeIter<'a> {
+    fn len(&self) -> usize {
+        self.cols.len()
+    }
+}
+
+impl<'a> From<&'a [GroupedColumn]> for ColRangeIter<'a> {
+    fn from(cols: &'a [GroupedColumn]) -> Self {
+        ColRangeIter{
+            cols,
+            offset: 0,
+        }
+    }
+}
+
+impl ColumnInner {
+    pub(crate) fn range(&self) -> Range<usize> {
+        match self {
+            Self::Single(SingleColumn { range: r, .. }) => r.clone(),
+            Self::Composite(CompositeColumn::Value(ValueColumn { meta, value, .. })) => {
+                meta.start..value.end
+            }
+            Self::Composite(CompositeColumn::Group(GroupColumn { num, values, .. })) => {
+                num.start..values.last().map(|v| v.range().end).unwrap_or(num.end)
+            }
+        }
+    }
+
+    pub(crate) fn ranges<'a>(&'a self) -> ColumnRanges<'a> {
+        match self {
+            Self::Single(SingleColumn{range, ..}) => ColumnRanges::Single(range.clone()),
+            Self::Composite(CompositeColumn::Value(ValueColumn{ meta, value, ..})) => ColumnRanges::Value {
+                meta: meta.clone(),
+                val: value.clone(),
+            },
+            Self::Composite(CompositeColumn::Group(GroupColumn{num, values, ..})) => ColumnRanges::Group {
+                num: num.clone(),
+                cols: (&values[..]).into(),
+            }
+        }
+    }
+
+    pub(crate) fn decoder<'a>(&self, data: &'a [u8]) -> GenericColDecoder<'a> {
+        match self {
+            Self::Single(SingleColumn {
+                range, col_type, ..
+            }) => {
+                let simple = col_type.decoder(&data[range.clone()]);
+                GenericColDecoder::new_simple(simple)
+            },
+            Self::Composite(CompositeColumn::Value(ValueColumn{meta, value,..})) => GenericColDecoder::new_value(
+                ValueDecoder::new(
+                    RleDecoder::from(Cow::Borrowed(&data[meta.clone()])),
+                    RawDecoder::from(Cow::Borrowed(&data[value.clone()])),
+                )
+            ),
+            Self::Composite(CompositeColumn::Group(GroupColumn{num, values, ..})) => {
+                let num_coder = RleDecoder::from(Cow::from(&data[num.clone()]));
+                let values = values
+                    .iter()
+                    .map(|gc| match gc {
+                        GroupedColumn::Single(SingleColumn{col_type, range, ..}) => SingleLogicalColDecoder::Simple(
+                            col_type.decoder(&data[range.clone()])
+                        ),
+                        GroupedColumn::Value(ValueColumn{ meta, value, .. }) => {
+                            SingleLogicalColDecoder::Value(ValueDecoder::new(
+                                RleDecoder::from(Cow::Borrowed(&data[meta.clone()])),
+                                RawDecoder::from(Cow::Borrowed(&data[value.clone()])),
+                            ))
+                        }
+                    })
+                    .collect();
+                GenericColDecoder::new_group(GroupDecoder::new(num_coder, values))
+            }
+        }
+    }
+
+    pub(crate) fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        output: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        match self {
+            Self::Single(s) => Ok(Self::Single(s.splice(
+                source,
+                output,
+                replace,
+                replace_with,
+            )?)),
+            Self::Composite(s) => Ok(Self::Composite(s.splice(
+                source,
+                output,
+                replace,
+                replace_with,
+            )?)),
+        }
+    }
+
+    pub(crate) fn col_type(&self) -> ColumnType {
+        match self {
+            Self::Single(SingleColumn{spec, ..}) => spec.col_type(),
+            Self::Composite(CompositeColumn::Value(..)) => ColumnType::Value,
+            Self::Composite(CompositeColumn::Group(..)) => ColumnType::Group,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct SingleColumn {
+    pub(crate) spec: ColumnSpec,
+    pub(crate) col_type: SimpleColType,
+    pub(crate) range: Range<usize>,
+}
+
+impl SingleColumn {
+    fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        output: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        let output_start = output.len();
+        let mut decoder = self.col_type.decoder(&source[self.range.clone()]);
+        let end = decoder.splice(output, replace, replace_with)? + output_start;
+        Ok(Self {
+            spec: self.spec,
+            col_type: self.col_type,
+            range: (output_start..end).into(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+enum CompositeColumn {
+    Value(ValueColumn),
+    Group(GroupColumn),
+}
+
+impl CompositeColumn {
+    fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        output: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        match self {
+            Self::Value(value) => Ok(Self::Value(value.splice(
+                source,
+                replace,
+                replace_with,
+                output,
+            )?)),
+            Self::Group(group) => Ok(Self::Group(group.splice(
+                source,
+                output,
+                replace,
+                replace_with,
+            )?)),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ValueColumn {
+    spec: ColumnSpec,
+    meta: Range<usize>,
+    value: Range<usize>,
+}
+
+impl ValueColumn {
+    fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        replace: Range<usize>,
+        replace_with: I,
+        output: &mut Vec<u8>,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        let mut decoder = ValueDecoder::new(
+            RleDecoder::from(&source[self.meta.clone()]),
+            RawDecoder::from(&source[self.value.clone()]),
+        );
+        let replacements = replace_with.enumerate().map(|(i, r)| match r {
+            CellValue::Value(p) => Ok(p),
+            _ => Err(ColumnSpliceError::InvalidValueForRow(i)),
+        });
+        let (new_meta, new_data) = decoder.try_splice(replace, replacements, output)?;
+        Ok(ValueColumn {
+            spec: self.spec,
+            meta: new_meta.into(),
+            value: new_data.into(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+struct GroupColumn {
+    spec: ColumnSpec,
+    num: Range<usize>,
+    values: Vec<GroupedColumn>,
+}
+
+#[derive(Eq, PartialEq, Clone, Copy, Debug)]
+enum SimpleColType {
+    Actor,
+    Integer,
+    DeltaInteger,
+    Boolean,
+    String,
+}
+
+impl SimpleColType {
+    fn decoder<'a>(self, data: &'a [u8]) -> SimpleColDecoder<'a> {
+        match self {
+            SimpleColType::Actor => SimpleColDecoder::new_uint(RleDecoder::from(Cow::from(data))),
+            SimpleColType::Integer => SimpleColDecoder::new_uint(RleDecoder::from(Cow::from(data))),
+            SimpleColType::String => SimpleColDecoder::new_string(RleDecoder::from(Cow::from(data))),
+            SimpleColType::Boolean => SimpleColDecoder::new_bool(BooleanDecoder::from(Cow::from(data))),
+            SimpleColType::DeltaInteger => SimpleColDecoder::new_delta(DeltaDecoder::from(Cow::from(data))),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+enum GroupedColumn {
+    Single(SingleColumn),
+    Value(ValueColumn),
+}
+
+impl GroupedColumn {
+    fn range(&self) -> Range<usize> {
+        match self {
+            Self::Single(SingleColumn{range, ..}) => range.clone(),
+            Self::Value(ValueColumn { meta, value, .. }) => (meta.start..value.end),
+        }
+    }
+}
+
+impl GroupColumn {
+    fn splice<'a, I>(
+        &self,
+        source: &[u8],
+        output: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<Self, ColumnSpliceError>
+    where
+        I: Iterator<Item=CellValue<'a>> + Clone
+    {
+        // This is a little like ValueDecoder::splice. First we want to read off the values from `num`
+        // and insert them into the output - inserting replacements lengths as we go. Then we re-read
+        // num and use it to also iterate over the grouped values, inserting those into the subsidiary
+        // columns as we go.
+
+        // First encode the lengths
+        let output_start = output.len();
+        let mut num_decoder =
+            RleDecoder::<'_, u64>::from(Cow::from(&source[self.num.clone()]));
+        let mut num_encoder = RleEncoder::from(output);
+        let mut idx = 0;
+        while idx < replace.start {
+            match num_decoder.next() {
+                Some(next_num) => {
+                    num_encoder.append(next_num.as_ref());
+                }
+                None => {
+                    panic!("out of bounds");
+                }
+            }
+            idx += 1;
+        }
+        let mut num_replace_with = replace_with.clone();
+        while let Some(replacement) = num_replace_with.next() {
+            let rows = match &replacement {
+                CellValue::List(rows) => rows,
+                _ => return Err(ColumnSpliceError::InvalidValueForRow(idx)),
+            };
+            for row in rows {
+                if row.len() != self.values.len() {
+                    return Err(ColumnSpliceError::WrongNumberOfValues {
+                        row: idx - replace.start,
+                        expected: self.values.len(),
+                        actual: row.len(),
+                    });
+                }
+                num_encoder.append(Some(&(rows.len() as u64)));
+            }
+            idx += 1;
+        }
+        while let Some(num) = num_decoder.next() {
+            num_encoder.append(num.as_ref());
+            idx += 1;
+        }
+        let _num_range = output_start..num_encoder.finish();
+
+        // Now encode the values
+        let _num_decoder =
+            RleDecoder::<'_, u64>::from(Cow::from(&source[self.num.clone()]));
+
+        panic!()
+    }
+}
+
+pub(crate) struct ColumnBuilder {
+    
+}
+
+impl ColumnBuilder {
+    pub(crate) fn build_actor(spec: ColumnSpec, range: Range<usize>) -> Column {
+        Column(ColumnInner::Single(SingleColumn{spec, col_type: SimpleColType::Actor, range: range.into()}))
+    }
+
+    pub(crate) fn build_string(spec: ColumnSpec, range: Range<usize>) -> Column {
+        Column(ColumnInner::Single(SingleColumn{spec, col_type: SimpleColType::String, range: range.into()}))
+    }
+
+    pub(crate) fn build_integer(spec: ColumnSpec, range: Range<usize>) -> Column {
+        Column(ColumnInner::Single(SingleColumn{spec, col_type: SimpleColType::Integer, range: range.into()}))
+    }
+
+    pub(crate) fn build_delta_integer(spec: ColumnSpec, range: Range<usize>) -> Column {
+        Column(ColumnInner::Single(SingleColumn{spec, col_type: SimpleColType::Integer, range: range.into()}))
+    }
+
+    pub(crate) fn build_boolean(spec: ColumnSpec, range: Range<usize>) -> Column {
+        Column(ColumnInner::Single(SingleColumn{spec, col_type: SimpleColType::Boolean, range: range.into()}))
+    }
+
+    pub(crate) fn start_value(spec: ColumnSpec, meta: Range<usize>) -> AwaitingRawColumnValueBuilder {
+        AwaitingRawColumnValueBuilder { spec, meta }
+    }
+
+    pub(crate) fn start_group(spec: ColumnSpec, num: Range<usize>) -> GroupBuilder {
+        GroupBuilder{spec, num_range: num, columns: Vec::new()}
+    }
+}
+
+pub(crate) struct AwaitingRawColumnValueBuilder {
+    spec: ColumnSpec,
+    meta: Range<usize>,
+}
+
+impl AwaitingRawColumnValueBuilder {
+    pub(crate) fn id(&self) -> ColumnId {
+        self.spec.id()
+    }
+
+    pub(crate) fn meta_range(&self) -> &Range<usize> {
+        &self.meta
+    }
+
+    pub(crate) fn build(&mut self, raw: Range<usize>) -> Column {
+        Column(ColumnInner::Composite(CompositeColumn::Value(ValueColumn{
+            spec: self.spec,
+            meta: self.meta.clone().into(),
+            value: raw.into(),
+        })))
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct GroupBuilder{
+    spec: ColumnSpec,
+    num_range: Range<usize>,
+    columns: Vec<GroupedColumn>,
+}
+
+impl GroupBuilder {
+
+    pub(crate) fn range(&self) -> Range<usize> {
+        let start = self.num_range.start;
+        let end = self.columns.last().map(|c| c.range().end).unwrap_or(self.num_range.end);
+        start..end
+    }
+
+    pub(crate) fn add_actor(&mut self, spec: ColumnSpec, range: Range<usize>) {
+        self.columns.push(GroupedColumn::Single(SingleColumn{
+            col_type: SimpleColType::Actor,
+            range: range.into(),
+            spec,
+        }));
+    }
+
+    pub(crate) fn add_string(&mut self, spec: ColumnSpec, range: Range<usize>) {
+        self.columns.push(GroupedColumn::Single(SingleColumn{
+            col_type: SimpleColType::String,
+            range: range.into(),
+            spec,
+        }));
+    }
+
+    pub(crate) fn add_integer(&mut self, spec: ColumnSpec, range: Range<usize>) {
+        self.columns.push(GroupedColumn::Single(SingleColumn{
+            col_type: SimpleColType::Integer,
+            range: range.into(),
+            spec,
+        }));
+    }
+
+    pub(crate) fn add_delta_integer(&mut self, spec: ColumnSpec, range: Range<usize>) {
+        self.columns.push(GroupedColumn::Single(SingleColumn{
+            col_type: SimpleColType::DeltaInteger,
+            range: range.into(),
+            spec,
+        }));
+    }
+
+    pub(crate) fn add_boolean(&mut self, spec: ColumnSpec, range: Range<usize>) {
+        self.columns.push(GroupedColumn::Single(SingleColumn{
+            col_type: SimpleColType::Boolean,
+            range: range.into(),
+            spec,
+        }));
+    }
+
+    pub(crate) fn start_value(&mut self, spec: ColumnSpec, meta: Range<usize>) -> GroupAwaitingValue {
+        GroupAwaitingValue {
+            spec,
+            num_range: self.num_range.clone(),
+            columns: std::mem::take(&mut self.columns),
+            val_spec: spec,
+            val_meta: meta,
+        }
+    }
+
+    pub(crate) fn finish(&mut self) -> Column {
+        Column(ColumnInner::Composite(CompositeColumn::Group(GroupColumn{
+            spec: self.spec,
+            num: self.num_range.clone(),
+            values: std::mem::take(&mut self.columns),
+        })))
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct GroupAwaitingValue {
+    spec: ColumnSpec,
+    num_range: Range<usize>,
+    columns: Vec<GroupedColumn>,
+    val_spec: ColumnSpec,
+    val_meta: Range<usize>,
+}
+
+impl GroupAwaitingValue {
+    pub(crate) fn finish_empty(&mut self) -> GroupBuilder {
+        self.columns.push(GroupedColumn::Value(ValueColumn{
+            meta: self.val_meta.clone(),
+            value: 0..0,
+            spec: self.val_spec,
+        }));
+        GroupBuilder {
+            spec: self.spec,
+            num_range: self.num_range.clone(),
+            columns: std::mem::take(&mut self.columns),
+        }
+    }
+
+    pub(crate) fn finish_value(&mut self, raw: Range<usize>) -> GroupBuilder {
+        self.columns.push(GroupedColumn::Value(ValueColumn{
+            spec: self.val_spec,
+            value: raw.into(),
+            meta: self.val_meta.clone(),
+        }));
+        GroupBuilder {
+            spec: self.spec,
+            num_range: self.num_range.clone(),
+            columns: std::mem::take(&mut self.columns),
+        }
+    }
+
+    pub(crate) fn range(&self) -> Range<usize> {
+        self.num_range.start..self.val_meta.end
+    }
+}

automerge/src/columnar_2/rowblock/column_layout/doc_change_columns.rs

@@ -0,0 +1,480 @@
+use std::{borrow::{Borrow, Cow}, convert::TryFrom, ops::Range};
+
+use tracing::instrument;
+
+use crate::columnar_2::{
+    column_specification::ColumnType,
+    rowblock::{
+        column_range::{ActorRange, DeltaIntRange, RawRange, RleIntRange, RleStringRange},
+        encoding::{DecodeColumnError, DeltaDecoder, RawDecoder, RleDecoder, ValueDecoder},
+        PrimVal,
+    },
+    ColumnId, ColumnSpec,
+    storage::ColumnMetadata,
+};
+
+use super::{
+    assert_col_type,
+    column::{ColumnRanges, GroupColRange},
+    ColumnLayout, MismatchingColumn,
+};
+
+#[derive(Debug)]
+pub(crate) struct ChangeMetadata<'a> {
+    pub(crate) actor: usize,
+    pub(crate) seq: u64,
+    pub(crate) max_op: u64,
+    pub(crate) timestamp: i64,
+    pub(crate) message: Option<smol_str::SmolStr>,
+    pub(crate) deps: Vec<u64>,
+    pub(crate) extra: Cow<'a, [u8]>,
+}
+
+pub(crate) struct DocChangeColumns {
+    actor: ActorRange,
+    seq: DeltaIntRange,
+    max_op: DeltaIntRange,
+    time: DeltaIntRange,
+    message: RleStringRange,
+    deps_group: RleIntRange,
+    deps_index: DeltaIntRange,
+    extra_meta: RleIntRange,
+    extra_val: RawRange,
+    other: ColumnLayout,
+}
+
+impl DocChangeColumns {
+    pub(crate) fn iter<'a>(&self, data: &'a [u8]) -> DocChangeColumnIter<'a> {
+        DocChangeColumnIter {
+            actors: self.actor.decoder(data),
+            seq: self.seq.decoder(data),
+            max_op: self.max_op.decoder(data),
+            time: self.time.decoder(data),
+            message: self.message.decoder(data),
+            deps: DepsDecoder {
+                group: self.deps_group.decoder(data),
+                deps: self.deps_index.decoder(data),
+            },
+            extra: ExtraDecoder {
+                val: ValueDecoder::new(self.extra_meta.decoder(data), self.extra_val.decoder(data)),
+            },
+        }
+    }
+
+    pub(crate) fn encode<'a, I, C: Borrow<ChangeMetadata<'a>>>(changes: I, out: &mut Vec<u8>) -> DocChangeColumns
+    where
+        I: Iterator<Item = C> + Clone,
+    {
+        let actor = ActorRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            // TODO: make this fallible once iterators have a try_splice
+            changes.clone().map(|c| Some(c.borrow().actor as u64)),
+            out,
+        );
+        let seq = DeltaDecoder::from(&[] as &[u8]).splice(
+            0..0,
+            changes.clone().map(|c| Some(c.borrow().seq as i64)),
+            out,
+        );
+        let max_op = DeltaDecoder::from(&[] as &[u8]).splice(
+            0..0,
+            changes.clone().map(|c| Some(c.borrow().max_op as i64)),
+            out,
+        );
+        let time = DeltaDecoder::from(&[] as &[u8]).splice(
+            0..0,
+            changes.clone().map(|c| Some(c.borrow().timestamp)),
+            out,
+        );
+        let message = RleDecoder::<'a, smol_str::SmolStr>::from(&[] as &[u8]).splice(
+            0..0,
+            changes.clone().map(|c| c.borrow().message.clone()),
+            out,
+        );
+        let (deps_group, deps_index) = DepsDecoder {
+            group: RleDecoder::from(&[] as &[u8]),
+            deps: DeltaDecoder::from(&[] as &[u8]),
+        }
+        .splice(0..0, changes.clone().map(|c| c.borrow().deps.clone()), out);
+        let (extra_meta, extra_val) = ValueDecoder::new(
+            RleDecoder::from(&[] as &[u8]),
+            RawDecoder::from(&[] as &[u8]),
+        )
+        .splice(0..0, changes.clone().map(|c| PrimVal::Bytes(c.borrow().extra.clone())), out);
+        DocChangeColumns {
+            actor: actor.into(),
+            seq: seq.into(),
+            max_op: max_op.into(),
+            time: time.into(),
+            message: message.into(),
+            deps_group: deps_group.into(),
+            deps_index: deps_index.into(),
+            extra_meta: extra_meta.into(),
+            extra_val: extra_val.into(),
+            other: ColumnLayout::empty(),
+        }
+    }
+
+    pub(crate) fn metadata(&self) -> ColumnMetadata {
+        const ACTOR_COL_ID: ColumnId = ColumnId::new(0);
+        const SEQ_COL_ID: ColumnId = ColumnId::new(0);
+        const MAX_OP_COL_ID: ColumnId = ColumnId::new(1);
+        const TIME_COL_ID: ColumnId = ColumnId::new(2);
+        const MESSAGE_COL_ID: ColumnId = ColumnId::new(3);
+        const DEPS_COL_ID: ColumnId = ColumnId::new(4);
+        const EXTRA_COL_ID: ColumnId = ColumnId::new(5);
+        
+        let mut cols = vec![
+            (ColumnSpec::new(ACTOR_COL_ID, ColumnType::Actor, false), self.actor.clone().into()),
+            (ColumnSpec::new(SEQ_COL_ID, ColumnType::DeltaInteger, false), self.seq.clone().into()),
+            (ColumnSpec::new(MAX_OP_COL_ID, ColumnType::DeltaInteger, false), self.max_op.clone().into()),
+            (ColumnSpec::new(TIME_COL_ID, ColumnType::DeltaInteger, false), self.time.clone().into()),
+            (ColumnSpec::new(MESSAGE_COL_ID, ColumnType::String, false), self.message.clone().into()),
+            (ColumnSpec::new(DEPS_COL_ID, ColumnType::Group, false), self.deps_group.clone().into()),
+        ];
+        if self.deps_index.len() > 0 {
+            cols.push((
+                ColumnSpec::new(DEPS_COL_ID, ColumnType::DeltaInteger, false), self.deps_index.clone().into()
+            ))
+        }
+        cols.push(
+            (ColumnSpec::new(EXTRA_COL_ID, ColumnType::ValueMetadata, false), self.extra_meta.clone().into()),
+        );
+        if self.extra_val.len() > 0 {
+            cols.push((
+                ColumnSpec::new(EXTRA_COL_ID, ColumnType::Value, false), self.extra_val.clone().into()
+            ))
+        }
+        cols.into_iter().collect()
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum DecodeChangeError {
+    #[error("the depenencies column was invalid")]
+    InvalidDeps,
+    #[error("unexpected null value for {0}")]
+    UnexpectedNull(String),
+    #[error("mismatching column types for column {index}")]
+    MismatchingColumn { index: usize },
+    #[error("not enough columns")]
+    NotEnoughColumns,
+    #[error("incorrect value in extra bytes column")]
+    InvalidExtraBytes,
+    #[error("error reading column {column}: {description}")]
+    ReadColumn { column: String, description: String },
+}
+
+impl DecodeChangeError {
+    fn from_decode_col(col: &'static str, err: DecodeColumnError) -> Self {
+        match err {
+            DecodeColumnError::InvalidValue { description, .. } => Self::ReadColumn {
+                column: col.to_string(),
+                description,
+            },
+            DecodeColumnError::UnexpectedNull(inner_col) => {
+                Self::UnexpectedNull(format!("{}:{}", col, inner_col))
+            }
+        }
+    }
+}
+
+impl From<MismatchingColumn> for DecodeChangeError {
+    fn from(m: MismatchingColumn) -> Self {
+        Self::MismatchingColumn { index: m.index }
+    }
+}
+
+pub(crate) struct DocChangeColumnIter<'a> {
+    actors: RleDecoder<'a, u64>,
+    seq: DeltaDecoder<'a>,
+    max_op: DeltaDecoder<'a>,
+    time: DeltaDecoder<'a>,
+    message: RleDecoder<'a, smol_str::SmolStr>,
+    deps: DepsDecoder<'a>,
+    extra: ExtraDecoder<'a>,
+}
+
+macro_rules! next_or_invalid({$iter: expr, $col: literal} => {
+    match $iter.next() {
+        Some(Some(s)) => s,
+        Some(None) => return Some(Err(DecodeChangeError::UnexpectedNull($col.to_string()))),
+        None => return Some(Err(DecodeChangeError::UnexpectedNull($col.to_string()))),
+    }
+});
+
+impl<'a> Iterator for DocChangeColumnIter<'a> {
+    type Item = Result<ChangeMetadata<'a>, DecodeChangeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let actor = match self.actors.next() {
+            Some(Some(actor)) => actor as usize,
+            Some(None) => return Some(Err(DecodeChangeError::UnexpectedNull("actor".to_string()))),
+            None => {
+                // The actor column should always have a value so if the actor iterator returns None that
+                // means we should be done, we check by asserting that all the other iterators
+                // return none (which is what Self::is_done does).
+                if self.is_done() {
+                    return None;
+                } else {
+                    return Some(Err(DecodeChangeError::UnexpectedNull("actor".to_string())));
+                }
+            }
+        };
+        let seq = match next_or_invalid!(self.seq, "seq").try_into() {
+            Ok(s) => s,
+            Err(_) => {
+                return Some(Err(DecodeChangeError::ReadColumn {
+                    column: "seq".to_string(),
+                    description: "negative value".to_string(),
+                }))
+            }
+        };
+        let max_op = match next_or_invalid!(self.max_op, "max_op").try_into() {
+            Ok(o) => o,
+            Err(_) => {
+                return Some(Err(DecodeChangeError::ReadColumn {
+                    column: "max_op".to_string(),
+                    description: "negative value".to_string(),
+                }))
+            }
+        };
+        let time = next_or_invalid!(self.time, "time");
+        let message = match self.message.next() {
+            Some(Some(s)) => Some(s),
+            Some(None) => None,
+            None => return Some(Err(DecodeChangeError::UnexpectedNull("msg".to_string()))),
+        };
+        let deps = match self.deps.next() {
+            Some(Ok(d)) => d,
+            Some(Err(e)) => return Some(Err(e)),
+            None => return Some(Err(DecodeChangeError::UnexpectedNull("deps".to_string()))),
+        };
+        let extra = match self.extra.next() {
+            Some(Ok(e)) => e,
+            Some(Err(e)) => return Some(Err(e)),
+            None => return Some(Err(DecodeChangeError::UnexpectedNull("extra".to_string()))),
+        };
+        Some(Ok(ChangeMetadata {
+            actor,
+            seq,
+            max_op,
+            timestamp: time,
+            message,
+            deps,
+            extra,
+        }))
+    }
+}
+
+impl<'a> DocChangeColumnIter<'a> {
+    /// Given that we have read a `None` value in the actor column, check that every other column
+    /// also returns `None`.
+    fn is_done(&mut self) -> bool {
+        let other_cols = [
+            self.seq.next().is_none(),
+            self.max_op.next().is_none(),
+            self.time.next().is_none(),
+            self.message.next().is_none(),
+            self.deps.next().is_none(),
+        ];
+        other_cols.iter().all(|f| *f)
+    }
+}
+
+struct DepsDecoder<'a> {
+    group: RleDecoder<'a, u64>,
+    deps: DeltaDecoder<'a>,
+}
+
+impl<'a> DepsDecoder<'a> {
+
+    fn encode<'b, I>(deps: I, out: &'a mut Vec<u8>) -> DepsDecoder<'a> 
+    where
+        I: Iterator<Item=&'b [u64]> + Clone
+    {
+        let group = RleDecoder::encode(deps.clone().map(|d| d.len() as u64), out);
+        let deps = DeltaDecoder::encode(deps.flat_map(|d| d.iter().map(|d| *d as i64)), out);
+        DepsDecoder{
+            group: RleDecoder::from(&out[group]),
+            deps: DeltaDecoder::from(&out[deps]),
+        }
+    }
+
+    fn splice<'b, I>(
+        &self,
+        replace_range: Range<usize>,
+        items: I,
+        out: &mut Vec<u8>,
+    ) -> (Range<usize>, Range<usize>)
+    where
+        I: Iterator<Item = Vec<u64>> + Clone,
+    {
+        let mut replace_start = 0_usize;
+        let mut replace_len = 0_usize;
+        for (index, elems) in self.group.clone().enumerate() {
+            if let Some(elems) = elems {
+                if index < replace_range.start {
+                    replace_start += elems as usize;
+                } else if index < replace_range.end {
+                    replace_len += elems as usize;
+                }
+            }
+        }
+        let val_replace_range = replace_start..(replace_start + replace_len);
+        let group = self.group.clone().splice(
+            replace_range,
+            items.clone().map(|i| Some(i.len() as u64)),
+            out
+        );
+        let items = self.deps.clone().splice(
+            val_replace_range,
+            items.flat_map(|elems| elems.into_iter().map(|v| Some(v as i64))),
+            out,
+        );
+        (group, items)
+    }
+}
+
+impl<'a> Iterator for DepsDecoder<'a> {
+    type Item = Result<Vec<u64>, DecodeChangeError>;
+    fn next(&mut self) -> Option<Self::Item> {
+        let num = match self.group.next() {
+            Some(Some(n)) => n as usize,
+            Some(None) => return Some(Err(DecodeChangeError::InvalidDeps)),
+            None => return None,
+        };
+        let mut result = Vec::with_capacity(num);
+        while result.len() < num {
+            match self.deps.next() {
+                Some(Some(elem)) => {
+                    let elem = match u64::try_from(elem) {
+                        Ok(e) => e,
+                        Err(e) => {
+                            tracing::error!(err=?e, dep=elem, "error converting dep index to u64");
+                            return Some(Err(DecodeChangeError::InvalidDeps));
+                        }
+                    };
+                    result.push(elem);
+                }
+                _ => return Some(Err(DecodeChangeError::InvalidDeps)),
+            }
+        }
+        Some(Ok(result))
+    }
+}
+
+struct ExtraDecoder<'a> {
+    val: ValueDecoder<'a>,
+}
+
+impl<'a> Iterator for ExtraDecoder<'a> {
+    type Item = Result<Cow<'a, [u8]>, DecodeChangeError>;
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.val.next() {
+            Some(Ok(PrimVal::Bytes(b))) => Some(Ok(b)),
+            Some(Ok(_)) => Some(Err(DecodeChangeError::InvalidExtraBytes)),
+            Some(Err(e)) => Some(Err(DecodeChangeError::from_decode_col("value", e))),
+            None => None,
+        }
+    }
+}
+
+impl TryFrom<ColumnLayout> for DocChangeColumns {
+    type Error = DecodeChangeError;
+
+    #[instrument]
+    fn try_from(columns: ColumnLayout) -> Result<Self, Self::Error> {
+        let mut actor: Option<Range<usize>> = None;
+        let mut seq: Option<Range<usize>> = None;
+        let mut max_op: Option<Range<usize>> = None;
+        let mut time: Option<Range<usize>> = None;
+        let mut message: Option<Range<usize>> = None;
+        let mut deps_group: Option<Range<usize>> = None;
+        let mut deps_index: Option<Range<usize>> = None;
+        let mut extra_meta: Option<Range<usize>> = None;
+        let mut extra_val: Option<Range<usize>> = None;
+        let mut other = ColumnLayout::empty();
+
+        for (index, col) in columns.into_iter().enumerate() {
+            match index {
+                0 => assert_col_type(index, col, ColumnType::Actor, &mut actor)?,
+                1 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut seq)?,
+                2 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut max_op)?,
+                3 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut time)?,
+                4 => assert_col_type(index, col, ColumnType::String, &mut message)?,
+                5 => match col.ranges() {
+                    ColumnRanges::Group { num, mut cols } => {
+                        deps_group = Some(num.into());
+                        let first = cols.next();
+                        match first {
+                            Some(GroupColRange::Single(index_range)) => {
+                                deps_index = Some(index_range.into());
+                            }
+                            Some(_) => {
+                                tracing::error!("deps column contained more than one grouped column");
+                                return Err(DecodeChangeError::MismatchingColumn{index: 5});
+                            }
+                            None => {
+                                deps_index = (0..0).into()
+                            }
+                        };
+                        if let Some(_) = cols.next() {
+                            return Err(DecodeChangeError::MismatchingColumn { index });
+                        }
+                    }
+                    _ => return Err(DecodeChangeError::MismatchingColumn { index }),
+                },
+                6 => match col.ranges() {
+                    ColumnRanges::Value { meta, val } => {
+                        extra_meta = Some(meta);
+                        extra_val = Some(val);
+                    }
+                    _ => return Err(DecodeChangeError::MismatchingColumn { index }),
+                },
+                _ => {
+                    other.append(col);
+                }
+            }
+        }
+        Ok(DocChangeColumns {
+            actor: actor.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            seq: seq.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            max_op: max_op.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            time: time.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            message: message.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            deps_group: deps_group
+                .ok_or(DecodeChangeError::NotEnoughColumns)?
+                .into(),
+            deps_index: deps_index
+                .ok_or(DecodeChangeError::NotEnoughColumns)?
+                .into(),
+            extra_meta: extra_meta
+                .ok_or(DecodeChangeError::NotEnoughColumns)?
+                .into(),
+            extra_val: extra_val.ok_or(DecodeChangeError::NotEnoughColumns)?.into(),
+            other,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+    use proptest::collection::vec as propvec;
+
+    fn encodable_u64() -> impl Strategy<Value = u64> + Clone {
+        0_u64..((i64::MAX / 2) as u64)
+    }
+
+    proptest!{
+        #[test]
+        fn encode_decode_deps(deps in propvec(propvec(encodable_u64(), 0..100), 0..100)) {
+            let mut out = Vec::new();
+            let decoder = DepsDecoder::encode(deps.iter().map(|d| &d[..]), &mut out);
+            let decoded = decoder.collect::<Result<Vec<_>, _>>().unwrap();
+            assert_eq!(deps, decoded);
+        }
+    }
+}

automerge/src/columnar_2/rowblock/column_layout/doc_op_columns.rs

@@ -0,0 +1,420 @@
+use std::{convert::TryFrom, ops::Range};
+
+use tracing::instrument;
+
+use crate::columnar_2::storage::ColumnMetadata;
+
+use super::{
+    super::{
+        super::column_specification::ColumnType,
+        column_range::{
+            ActorRange, BooleanRange, DeltaIntRange, RawRange, RleIntRange, RleStringRange,
+        },
+        encoding::{
+            BooleanDecoder, DecodeColumnError, Key, KeyDecoder, ObjDecoder, OpIdDecoder,
+            OpIdListDecoder, RleDecoder, ValueDecoder,
+        },
+        ColumnSpec, ColumnId,
+    },
+    assert_col_type,
+    column::{ColumnRanges, GroupColRange},
+    ColumnLayout, MismatchingColumn,
+};
+
+use crate::{
+    columnar_2::rowblock::{PrimVal, encoding::{RawDecoder, DeltaDecoder}},
+    types::{ObjId, OpId, ElemId},
+};
+
+/// The form operations take in the compressed document format.
+#[derive(Debug)]
+pub(crate) struct DocOp<'a> {
+    pub(crate) id: OpId,
+    pub(crate) object: ObjId,
+    pub(crate) key: Key,
+    pub(crate) insert: bool,
+    pub(crate) action: usize,
+    pub(crate) value: PrimVal<'a>,
+    pub(crate) succ: Vec<OpId>,
+}
+
+pub(crate) struct DocOpColumns {
+    actor: ActorRange,
+    ctr: RleIntRange,
+    key_actor: ActorRange,
+    key_ctr: DeltaIntRange,
+    key_str: RleStringRange,
+    id_actor: RleIntRange,
+    id_ctr: DeltaIntRange,
+    insert: BooleanRange,
+    action: RleIntRange,
+    val_meta: RleIntRange,
+    val_raw: RawRange,
+    succ_group: RleIntRange,
+    succ_actor: RleIntRange,
+    succ_ctr: DeltaIntRange,
+    other: ColumnLayout,
+}
+
+impl DocOpColumns {
+    pub(crate) fn empty() -> DocOpColumns {
+        Self {
+            actor: (0..0).into(),
+            ctr: (0..0).into(),
+            key_actor: (0..0).into(),
+            key_ctr: (0..0).into(),
+            key_str: (0..0).into(),
+            id_actor: (0..0).into(),
+            id_ctr: (0..0).into(),
+            insert: (0..0).into(),
+            action: (0..0).into(),
+            val_meta: (0..0).into(),
+            val_raw: (0..0).into(),
+            succ_group: (0..0).into(),
+            succ_actor: (0..0).into(),
+            succ_ctr: (0..0).into(),
+            other: ColumnLayout::empty(),
+        }
+    }
+
+    pub(crate) fn encode<'a, I>(ops: I, out: &mut Vec<u8>) -> DocOpColumns
+    where
+        I: Iterator<Item = DocOp<'a>> + Clone,
+    {
+        let obj_actor = ActorRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(o.object.opid().actor() as u64)),
+            out,
+        );
+        let obj_ctr = RleIntRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(o.object.opid().counter() as u64)),
+            out,
+        );
+        let key_actor = ActorRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match o.key {
+                Key::Prop(_) => None,
+                Key::Elem(ElemId(opid)) if opid.actor() == 0 => None,
+                Key::Elem(ElemId(opid)) => Some(opid.actor() as u64),
+            }),
+            out,
+        );
+        let key_ctr = DeltaIntRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match o.key {
+                Key::Prop(_) => None,
+                Key::Elem(ElemId(opid)) => Some(opid.counter() as i64),
+            }),
+            out,
+        );
+        let key_str = RleStringRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| match o.key {
+                Key::Prop(s) => Some(s),
+                Key::Elem(_) => None,
+            }),
+            out,
+        );
+        let id_actor = RleIntRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(o.id.actor() as u64)),
+            out,
+        );
+        let id_counter = DeltaIntRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(o.id.counter() as i64)),
+            out,
+        );
+        let insert = BooleanRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| o.insert),
+            out,
+        );
+        let action = RleIntRange::from(0..0).decoder(&[]).splice(
+            0..0,
+            ops.clone().map(|o| Some(o.action as u64)),
+            out,
+        );
+        let (val_meta, val_raw) = ValueDecoder::new(RleDecoder::from(&[] as &[u8]), RawDecoder::from(&[] as &[u8])).splice(
+            0..0,
+            ops.clone().map(|o| o.value),
+            out,
+        );
+        let mut succ_dec = OpIdListDecoder::new(
+            RleDecoder::from(&[] as &[u8]),
+            RleDecoder::from(&[] as &[u8]),
+            DeltaDecoder::from(&[] as &[u8]),
+        );
+        let (succ_group, succ_actor, succ_ctr) =
+            succ_dec.splice(0..0, ops.map(|o| o.succ.clone()), out);
+        Self {
+            actor: obj_actor.into(),
+            ctr: obj_ctr.into(),
+            key_actor: key_actor.into(),
+            key_ctr: key_ctr.into(),
+            key_str: key_str.into(),
+            id_actor: id_actor.into(),
+            id_ctr: id_counter.into(),
+            insert: insert.into(),
+            action: action.into(),
+            val_meta: val_meta.into(),
+            val_raw: val_raw.into(),
+            succ_group: succ_group.into(),
+            succ_actor: succ_actor.into(),
+            succ_ctr: succ_ctr.into(),
+            other: ColumnLayout::empty(),
+        }
+    }
+
+    pub(crate) fn iter<'a>(&self, data: &'a [u8]) -> DocOpColumnIter<'a> {
+        DocOpColumnIter {
+            id: OpIdDecoder::new(self.id_actor.decoder(data), self.id_ctr.decoder(data)),
+            action: self.action.decoder(data),
+            objs: ObjDecoder::new(self.actor.decoder(data), self.ctr.decoder(data)),
+            keys: KeyDecoder::new(
+                self.key_actor.decoder(data),
+                self.key_ctr.decoder(data),
+                self.key_str.decoder(data),
+            ),
+            insert: self.insert.decoder(data),
+            value: ValueDecoder::new(self.val_meta.decoder(data), self.val_raw.decoder(data)),
+            succ: OpIdListDecoder::new(
+                self.succ_group.decoder(data),
+                self.succ_actor.decoder(data),
+                self.succ_ctr.decoder(data),
+            ),
+        }
+    }
+
+    pub(crate) fn metadata(&self) -> ColumnMetadata {
+        const OBJ_COL_ID: ColumnId = ColumnId::new(0);
+        const KEY_COL_ID: ColumnId = ColumnId::new(1);
+        const ID_COL_ID: ColumnId = ColumnId::new(2);
+        const INSERT_COL_ID: ColumnId = ColumnId::new(3);
+        const ACTION_COL_ID: ColumnId = ColumnId::new(4);
+        const VAL_COL_ID: ColumnId = ColumnId::new(5);
+        const SUCC_COL_ID: ColumnId = ColumnId::new(8); 
+        
+        let mut cols = vec![
+            (ColumnSpec::new(OBJ_COL_ID, ColumnType::Actor, false), self.actor.clone().into()),
+            (ColumnSpec::new(OBJ_COL_ID, ColumnType::Integer, false), self.ctr.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::Actor, false), self.key_actor.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::DeltaInteger, false), self.key_ctr.clone().into()),
+            (ColumnSpec::new(KEY_COL_ID, ColumnType::String, false), self.key_str.clone().into()),
+            (ColumnSpec::new(ID_COL_ID, ColumnType::Actor, false), self.id_actor.clone().into()),
+            (ColumnSpec::new(ID_COL_ID, ColumnType::DeltaInteger, false), self.id_ctr.clone().into()),
+            (ColumnSpec::new(INSERT_COL_ID, ColumnType::Boolean, false), self.insert.clone().into()),
+            (ColumnSpec::new(ACTION_COL_ID, ColumnType::Integer, false), self.action.clone().into()),
+            (ColumnSpec::new(VAL_COL_ID, ColumnType::ValueMetadata, false), self.val_meta.clone().into()),
+        ];
+        if self.val_raw.len() > 0 {
+            cols.push((
+                ColumnSpec::new(VAL_COL_ID, ColumnType::Value, false), self.val_raw.clone().into()
+            ));
+        }
+        cols.push(
+            (ColumnSpec::new(SUCC_COL_ID, ColumnType::Group, false), self.succ_group.clone().into()),
+        );
+        if self.succ_actor.len() > 0 {
+            cols.extend([
+                (ColumnSpec::new(SUCC_COL_ID, ColumnType::Actor, false), self.succ_actor.clone().into()),
+                (ColumnSpec::new(SUCC_COL_ID, ColumnType::DeltaInteger, false), self.succ_ctr.clone().into()),
+            ]);
+        }
+        cols.into_iter().collect()
+    }
+}
+
+pub(crate) struct DocOpColumnIter<'a> {
+    id: OpIdDecoder<'a>,
+    action: RleDecoder<'a, u64>,
+    objs: ObjDecoder<'a>,
+    keys: KeyDecoder<'a>,
+    insert: BooleanDecoder<'a>,
+    value: ValueDecoder<'a>,
+    succ: OpIdListDecoder<'a>,
+}
+
+impl<'a> DocOpColumnIter<'a> {
+    fn done(&self) -> bool {
+        [
+            self.id.done(),
+            self.action.done(),
+            self.objs.done(),
+            self.keys.done(),
+            self.insert.done(),
+            self.value.done(),
+            self.succ.done(),
+        ]
+        .iter()
+        .all(|c| *c)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum DecodeOpError {
+    #[error("unexpected null in column {0}")]
+    UnexpectedNull(String),
+    #[error("invalid value in column {column}: {description}")]
+    InvalidValue { column: String, description: String },
+}
+
+macro_rules! next_or_invalid({$iter: expr, $col: literal} => {
+    match $iter.next() {
+        Some(Ok(id)) => id,
+        Some(Err(e)) => match e {
+            DecodeColumnError::UnexpectedNull(inner_col) => {
+                return Some(Err(DecodeOpError::UnexpectedNull(format!(
+                    "{}:{}", $col, inner_col
+                ))));
+            },
+            DecodeColumnError::InvalidValue{column, description} => {
+                let col = format!("{}:{}", $col, column);
+                return Some(Err(DecodeOpError::InvalidValue{column: col, description}))
+            }
+        }
+        None => return Some(Err(DecodeOpError::UnexpectedNull($col.to_string()))),
+    }
+});
+
+impl<'a> Iterator for DocOpColumnIter<'a> {
+    type Item = Result<DocOp<'a>, DecodeOpError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done() {
+            None
+        } else {
+            let id = next_or_invalid!(self.id, "opid");
+            let action = match self.action.next() {
+                Some(Some(a)) => a,
+                Some(None) | None => {
+                    return Some(Err(DecodeOpError::UnexpectedNull("action".to_string())))
+                }
+            };
+            let obj = next_or_invalid!(self.objs, "obj").into();
+            let key = next_or_invalid!(self.keys, "key");
+            let value = next_or_invalid!(self.value, "value");
+            let succ = next_or_invalid!(self.succ, "succ");
+            let insert = self.insert.next().unwrap_or(false);
+            Some(Ok(DocOp {
+                id,
+                value,
+                action: action as usize,
+                object: obj,
+                key,
+                succ,
+                insert,
+            }))
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("mismatching column at {index}.")]
+    MismatchingColumn { index: usize },
+    #[error("not enough columns")]
+    NotEnoughColumns,
+}
+
+impl From<MismatchingColumn> for Error {
+    fn from(m: MismatchingColumn) -> Self {
+        Error::MismatchingColumn { index: m.index }
+    }
+}
+
+impl TryFrom<ColumnLayout> for DocOpColumns {
+    type Error = Error;
+
+    #[instrument]
+    fn try_from(columns: ColumnLayout) -> Result<Self, Self::Error> {
+        let mut obj_actor: Option<Range<usize>> = None;
+        let mut obj_ctr: Option<Range<usize>> = None;
+        let mut key_actor: Option<Range<usize>> = None;
+        let mut key_ctr: Option<Range<usize>> = None;
+        let mut key_str: Option<Range<usize>> = None;
+        let mut id_actor: Option<Range<usize>> = None;
+        let mut id_ctr: Option<Range<usize>> = None;
+        let mut insert: Option<Range<usize>> = None;
+        let mut action: Option<Range<usize>> = None;
+        let mut val_meta: Option<Range<usize>> = None;
+        let mut val_raw: Option<Range<usize>> = None;
+        let mut succ_group: Option<Range<usize>> = None;
+        let mut succ_actor: Option<Range<usize>> = None;
+        let mut succ_ctr: Option<Range<usize>> = None;
+        let mut other = ColumnLayout::empty();
+
+        for (index, col) in columns.into_iter().enumerate() {
+            match index {
+                0 => assert_col_type(index, col, ColumnType::Actor, &mut obj_actor)?,
+                1 => assert_col_type(index, col, ColumnType::Integer, &mut obj_ctr)?,
+                2 => assert_col_type(index, col, ColumnType::Actor, &mut key_actor)?,
+                3 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut key_ctr)?,
+                4 => assert_col_type(index, col, ColumnType::String, &mut key_str)?,
+                5 => assert_col_type(index, col, ColumnType::Actor, &mut id_actor)?,
+                6 => assert_col_type(index, col, ColumnType::DeltaInteger, &mut id_ctr)?,
+                7 => assert_col_type(index, col, ColumnType::Boolean, &mut insert)?,
+                8 => assert_col_type(index, col, ColumnType::Integer, &mut action)?,
+                9 => match col.ranges() {
+                    ColumnRanges::Value { meta, val } => {
+                        val_meta = Some(meta);
+                        val_raw = Some(val);
+                    }
+                    _ => {
+                        tracing::error!("col 9 should be a value column");
+                        return Err(Error::MismatchingColumn { index });
+                    },
+                },
+                10 => match col.ranges() {
+                    ColumnRanges::Group { num, mut cols } => {
+                        let first = cols.next();
+                        let second = cols.next();
+                        succ_group = Some(num.into());
+                        match (first, second) {
+                            (
+                                Some(GroupColRange::Single(actor_range)),
+                                Some(GroupColRange::Single(ctr_range)),
+                            ) => {
+                                succ_actor = Some(actor_range.into());
+                                succ_ctr = Some(ctr_range.into());
+                            },
+                            (None, None) => {
+                                succ_actor = Some((0..0).into());
+                                succ_ctr = Some((0..0).into());
+                            }
+                            _ => {
+                                tracing::error!("expected a two column group of (actor, rle int) for index 10");
+                                return Err(Error::MismatchingColumn { index });
+                            }
+                        };
+                        if let Some(_) = cols.next() {
+                            return Err(Error::MismatchingColumn { index });
+                        }
+                    }
+                    _ => return Err(Error::MismatchingColumn { index }),
+                },
+                _ => {
+                    other.append(col);
+                }
+            }
+        }
+        Ok(DocOpColumns {
+            actor: obj_actor.ok_or(Error::NotEnoughColumns)?.into(),
+            ctr: obj_ctr.ok_or(Error::NotEnoughColumns)?.into(),
+            key_actor: key_actor.ok_or(Error::NotEnoughColumns)?.into(),
+            key_ctr: key_ctr.ok_or(Error::NotEnoughColumns)?.into(),
+            key_str: key_str.ok_or(Error::NotEnoughColumns)?.into(),
+            id_actor: id_actor.ok_or(Error::NotEnoughColumns)?.into(),
+            id_ctr: id_ctr.ok_or(Error::NotEnoughColumns)?.into(),
+            insert: insert.ok_or(Error::NotEnoughColumns)?.into(),
+            action: action.ok_or(Error::NotEnoughColumns)?.into(),
+            val_meta: val_meta.ok_or(Error::NotEnoughColumns)?.into(),
+            val_raw: val_raw.ok_or(Error::NotEnoughColumns)?.into(),
+            succ_group: succ_group.ok_or(Error::NotEnoughColumns)?.into(),
+            succ_actor: succ_actor.ok_or(Error::NotEnoughColumns)?.into(),
+            succ_ctr: succ_ctr.ok_or(Error::NotEnoughColumns)?.into(),
+            other,
+        })
+    }
+}

automerge/src/columnar_2/rowblock/column_layout/generic.rs

@@ -1,92 +1,54 @@
-/// This module contains types which represent the column metadata which is encoded in the columnar
-/// storage format specified in [1]. In this format metadata about each column is packed into a 32
-/// bit integer, which is represented by the types in `column_specification`. The column data in
-/// the format is a sequence of (`ColumnSpecification`, `usize`) pairs where each pair represents
-/// the type of the column and the length of the column in the data which follows, these pairs are
-/// represented by `RawColumn` and `RawColumns`. Some columns are actually composites of several
-/// underlying columns and so not every `RawColumns` is valid. The types in `column` and
-/// `column_builder` take a `RawColumns` and produce a `Columns` - which is a valid set of possibly
-/// composite column metadata.
-///
-/// There are two typical workflows:
-///
-/// ## Reading
-/// * First parse a `RawColumns` from the underlying data using `RawColumns::parse`
-/// * Ensure that the columns are decompressed using `RawColumns::decompress` (checking first if
-///   you can avoid this using `RawColumns::uncompressed`)
-/// * Parse the `RawColumns` into a `Columns` using `Columns::parse`
-///
-/// ## Writing
-/// * Construct a `RawColumns`
-/// * Compress using `RawColumns::compress`
-/// * Write to output using `RawColumns::write`
-///
-/// [1]: https://alexjg.github.io/automerge-storage-docs/#_columnar_storage_format
 use std::ops::Range;
 
-mod column_specification;
-pub(crate) use column_specification::{ColumnId, ColumnSpec, ColumnType};
-mod column;
-pub(crate) use column::Column;
-mod column_builder;
-pub(crate) use column_builder::{
-    AwaitingRawColumnValueBuilder, ColumnBuilder, GroupAwaitingValue, GroupBuilder,
+use crate::columnar_2::{
+    column_specification::{ColumnId, ColumnSpec, ColumnType},
+    rowblock::column_layout::column::{
+        AwaitingRawColumnValueBuilder, Column, ColumnBuilder, GroupAwaitingValue, GroupBuilder,
+    },
 };
 
-pub(crate) mod raw_column;
-pub(crate) use raw_column::{RawColumn, RawColumns};
+#[derive(Debug)]
+pub(crate) struct ColumnLayout(Vec<Column>);
 
-#[derive(Debug, thiserror::Error)]
-#[error("mismatching column at {index}.")]
-pub(crate) struct MismatchingColumn {
-    pub(crate) index: usize,
-}
-
-pub(crate) mod compression {
-    #[derive(Clone, Debug)]
-    pub(crate) struct Unknown;
-    #[derive(Clone, Debug)]
-    pub(crate) struct Uncompressed;
-
-    /// A witness for what we know about whether or not a column is compressed
-    pub(crate) trait ColumnCompression {}
-    impl ColumnCompression for Unknown {}
-    impl ColumnCompression for Uncompressed {}
-}
-
-/// `Columns` represents a sequence of "logical" columns. "Logical" in this sense means that
-/// each column produces one value, but may be composed of multiple [`RawColumn`]s. For example, in a
-/// logical column containing values there are two `RawColumn`s, one for the metadata about the
-/// values, and one for the values themselves.
-#[derive(Clone, Debug)]
-pub(crate) struct Columns {
-    columns: Vec<Column>,
-}
-
-impl Columns {
-    pub(crate) fn empty() -> Self {
-        Self {
-            columns: Vec::new(),
-        }
+impl ColumnLayout {
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &Column> {
+        self.0.iter()
     }
 
-    pub(crate) fn append(&mut self, col: Column) {
-        self.columns.push(col)
+    pub(crate) fn num_cols(&self) -> usize {
+        self.0.len()
     }
 
-    pub(crate) fn parse<'a, I: Iterator<Item = &'a RawColumn<compression::Uncompressed>>>(
+    pub(crate) fn parse<I: Iterator<Item = (ColumnSpec, Range<usize>)>>(
         data_size: usize,
         cols: I,
-    ) -> Result<Columns, BadColumnLayout> {
+    ) -> Result<ColumnLayout, BadColumnLayout> {
         let mut parser = ColumnLayoutParser::new(data_size, None);
-        for raw_col in cols {
-            parser.add_column(raw_col.spec(), raw_col.data())?;
+        for (col, range) in cols {
+            parser.add_column(col, range)?;
         }
         parser.build()
     }
+
+    pub(crate) fn as_specs(&self) -> impl Iterator<Item=(ColumnSpec, Range<usize>)> {
+        panic!();
+        std::iter::empty()
+    }
+
+    pub(crate) fn empty() -> Self {
+        Self(Vec::new())
+    }
+
+    pub(crate) fn append(&mut self, col: Column) {
+        self.0.push(col)
+    }
+
+    pub(crate) fn unsafe_from_vec(v: Vec<Column>) -> Self {
+        Self(v)
+    }
 }
 
-impl FromIterator<Column> for Result<Columns, BadColumnLayout> {
+impl FromIterator<Column> for Result<ColumnLayout, BadColumnLayout> {
     fn from_iter<T: IntoIterator<Item = Column>>(iter: T) -> Self {
         let iter = iter.into_iter();
         let mut result = Vec::with_capacity(iter.size_hint().1.unwrap_or(0));
@@ -100,16 +62,16 @@ impl FromIterator<Column> for Result<Columns, BadColumnLayout> {
             last_column = Some(col.spec());
             result.push(col);
         }
-        Ok(Columns { columns: result })
+        Ok(ColumnLayout(result))
     }
 }
 
-impl IntoIterator for Columns {
+impl IntoIterator for ColumnLayout {
     type Item = Column;
     type IntoIter = std::vec::IntoIter<Column>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.columns.into_iter()
+        self.0.into_iter()
     }
 }
 
@@ -160,12 +122,12 @@ impl ColumnLayoutParser {
         }
     }
 
-    fn build(mut self) -> Result<Columns, BadColumnLayout> {
-        let columns = match self.state {
-            LayoutParserState::Ready => self.columns,
+    fn build(mut self) -> Result<ColumnLayout, BadColumnLayout> {
+        match self.state {
+            LayoutParserState::Ready => Ok(ColumnLayout(self.columns)),
             LayoutParserState::InValue(mut builder) => {
-                self.columns.push(builder.build((0..0).into()));
-                self.columns
+                self.columns.push(builder.build(0..0));
+                Ok(ColumnLayout(self.columns))
             }
             LayoutParserState::InGroup(_, groupstate) => {
                 match groupstate {
@@ -176,13 +138,11 @@ impl ColumnLayoutParser {
                         self.columns.push(builder.finish());
                     }
                 };
-                self.columns
+                Ok(ColumnLayout(self.columns))
             }
-        };
-        Ok(Columns { columns })
+        }
     }
 
-    #[tracing::instrument(skip(self), err)]
     fn add_column(
         &mut self,
         column: ColumnSpec,
@@ -202,41 +162,38 @@ impl ColumnLayoutParser {
                 ColumnType::Group => {
                     self.state = LayoutParserState::InGroup(
                         column.id(),
-                        GroupParseState::Ready(ColumnBuilder::start_group(column, range.into())),
+                        GroupParseState::Ready(ColumnBuilder::start_group(column, range)),
                     );
                     Ok(())
                 }
                 ColumnType::ValueMetadata => {
-                    self.state = LayoutParserState::InValue(ColumnBuilder::start_value(
-                        column,
-                        range.into(),
-                    ));
+                    self.state =
+                        LayoutParserState::InValue(ColumnBuilder::start_value(column, range));
                     Ok(())
                 }
                 ColumnType::Value => Err(BadColumnLayout::LoneRawValueColumn),
                 ColumnType::Actor => {
-                    self.columns
-                        .push(ColumnBuilder::build_actor(column, range.into()));
+                    self.columns.push(ColumnBuilder::build_actor(column, range));
                     Ok(())
                 }
                 ColumnType::String => {
                     self.columns
-                        .push(ColumnBuilder::build_string(column, range.into()));
+                        .push(ColumnBuilder::build_string(column, range));
                     Ok(())
                 }
                 ColumnType::Integer => {
                     self.columns
-                        .push(ColumnBuilder::build_integer(column, range.into()));
+                        .push(ColumnBuilder::build_integer(column, range));
                     Ok(())
                 }
                 ColumnType::DeltaInteger => {
                     self.columns
-                        .push(ColumnBuilder::build_delta_integer(column, range.into()));
+                        .push(ColumnBuilder::build_delta_integer(column, range));
                     Ok(())
                 }
                 ColumnType::Boolean => {
                     self.columns
-                        .push(ColumnBuilder::build_boolean(column, range.into()));
+                        .push(ColumnBuilder::build_boolean(column, range));
                     Ok(())
                 }
             },
@@ -245,12 +202,12 @@ impl ColumnLayoutParser {
                     if builder.id() != column.id() {
                         return Err(BadColumnLayout::MismatchingValueMetadataId);
                     }
-                    self.columns.push(builder.build(range.into()));
+                    self.columns.push(builder.build(range));
                     self.state = LayoutParserState::Ready;
                     Ok(())
                 }
                 _ => {
-                    self.columns.push(builder.build((0..0).into()));
+                    self.columns.push(builder.build(0..0));
                     self.state = LayoutParserState::Ready;
                     self.add_column(column, range)
                 }
@@ -315,7 +272,6 @@ impl ColumnLayoutParser {
             LayoutParserState::Ready => {
                 if let Some(prev) = self.columns.last() {
                     if prev.range().end != next_range.start {
-                        tracing::error!(prev=?prev.range(), next=?next_range, "it's here");
                         Err(BadColumnLayout::NonContiguousColumns)
                     } else {
                         Ok(())
@@ -325,7 +281,7 @@ impl ColumnLayoutParser {
                 }
             }
             LayoutParserState::InValue(builder) => {
-                if builder.meta_range().end() != next_range.start {
+                if builder.meta_range().end != next_range.start {
                     Err(BadColumnLayout::NonContiguousColumns)
                 } else {
                     Ok(())
@@ -337,6 +293,7 @@ impl ColumnLayoutParser {
                     GroupParseState::Ready(b) => b.range().end,
                 };
                 if end != next_range.start {
+                    println!("Group state: {:?}", group_state);
                     Err(BadColumnLayout::NonContiguousColumns)
                 } else {
                     Ok(())

automerge/src/columnar_2/rowblock/column_layout/mod.rs

@@ -0,0 +1,53 @@
+use std::ops::Range;
+
+pub(crate) mod column;
+pub(crate) mod generic;
+pub(crate) mod doc_op_columns;
+pub(crate) mod doc_change_columns;
+pub(crate) mod change_op_columns;
+
+pub(crate) use generic::{BadColumnLayout, ColumnLayout};
+pub(crate) use doc_op_columns::{DocOpColumns, Error as ParseDocColumnError};
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum ColumnSpliceError {
+    #[error("invalid value for row {0}")]
+    InvalidValueForRow(usize),
+    #[error("wrong number of values for row {0}, expected {expected} but got {actual}")]
+    WrongNumberOfValues {
+        row: usize,
+        expected: usize,
+        actual: usize,
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("mismatching column at {index}.")]
+struct MismatchingColumn {
+    index: usize,
+}
+
+/// Given a `column::Column` assert that it is of the given `typ` and if so update `target` to be
+/// `Some(range)`. Otherwise return a `MismatchingColumn{index}`
+fn assert_col_type(
+    index: usize,
+    col: column::Column,
+    typ: crate::columnar_2::column_specification::ColumnType,
+    target: &mut Option<Range<usize>>,
+) -> Result<(), MismatchingColumn> {
+    if col.col_type() == typ {
+        match col.ranges() {
+            column::ColumnRanges::Single(range) => {
+                *target = Some(range);
+                Ok(())
+            },
+            _ => {
+                tracing::error!("expected a single column range");
+                return Err(MismatchingColumn{ index });
+            }
+        }
+    } else {
+        tracing::error!(index, expected=?typ, actual=?col.col_type(), "unexpected columnt type");
+        Err(MismatchingColumn { index })
+    }
+}

automerge/src/columnar_2/rowblock/column_range.rs

@@ -0,0 +1,60 @@
+use std::{borrow::Cow, ops::Range};
+
+use smol_str::SmolStr;
+
+use super::encoding::{
+    BooleanDecoder, BooleanEncoder, DeltaDecoder, DeltaEncoder, RawDecoder, RawEncoder,
+    RleDecoder, RleEncoder,
+};
+
+macro_rules! make_col_range({$name: ident, $decoder_name: ident$(<$($dparam: tt),+>)?, $encoder_name: ident$(<$($eparam: tt),+>)?} => {
+    #[derive(Clone, Debug)]
+    pub(crate) struct $name(Range<usize>);
+
+    impl $name {
+        pub(crate) fn decoder<'a>(&self, data: &'a[u8]) -> $decoder_name $(<$($dparam,)+>)* {
+            $decoder_name::from(Cow::Borrowed(&data[self.0.clone()]))
+        }
+
+        pub(crate) fn encoder<'a>(&self, output: &'a mut Vec<u8>) -> $encoder_name $(<$($eparam,)+>)* {
+            $encoder_name::from(output)
+        }
+
+        pub(crate) fn len(&self) -> usize {
+            self.0.len()
+        }
+
+        pub(crate) fn is_empty(&self) -> bool {
+            self.0.is_empty()
+        }
+    }
+
+    impl AsRef<Range<usize>> for $name {
+        fn as_ref(&self) -> &Range<usize> {
+            &self.0
+        }
+    }
+
+    impl From<Range<usize>> for $name {
+        fn from(r: Range<usize>) -> $name {
+            $name(r)
+        }
+    }
+
+    impl From<$name> for Range<usize> {
+        fn from(r: $name) -> Range<usize> {
+            r.0
+        }
+    }
+});
+
+make_col_range!(ActorRange, RleDecoder<'a, u64>, RleEncoder<'a, u64>);
+make_col_range!(RleIntRange, RleDecoder<'a, u64>, RleEncoder<'a, u64>);
+make_col_range!(DeltaIntRange, DeltaDecoder<'a>, DeltaEncoder<'a>);
+make_col_range!(
+    RleStringRange,
+    RleDecoder<'a, SmolStr>,
+    RleEncoder<'a, SmolStr>
+);
+make_col_range!(BooleanRange, BooleanDecoder<'a>, BooleanEncoder<'a>);
+make_col_range!(RawRange, RawDecoder<'a>, RawEncoder<'a>);

automerge/src/columnar_2/rowblock/encoding/boolean.rs

@@ -0,0 +1,163 @@
+use std::{borrow::Cow, ops::Range};
+
+use super::{Encodable, RawDecoder};
+
+/// Encodes booleans by storing the count of the same value.
+///
+/// The sequence of numbers describes the count of false values on even indices (0-indexed) and the
+/// count of true values on odd indices (0-indexed).
+///
+/// Counts are encoded as usize.
+pub(crate) struct BooleanEncoder<'a> {
+    written: usize,
+    buf: &'a mut Vec<u8>,
+    last: bool,
+    count: usize,
+}
+
+impl<'a> BooleanEncoder<'a> {
+    pub fn new(output: &'a mut Vec<u8>) -> BooleanEncoder<'a> {
+        BooleanEncoder {
+            written: 0,
+            buf: output,
+            last: false,
+            count: 0,
+        }
+    }
+
+    pub fn append(&mut self, value: bool) {
+        if value == self.last {
+            self.count += 1;
+        } else {
+            self.written += self.count.encode(&mut self.buf);
+            self.last = value;
+            self.count = 1;
+        }
+    }
+
+    pub fn finish(mut self) -> usize {
+        if self.count > 0 {
+            self.written += self.count.encode(&mut self.buf);
+        }
+        self.written
+    }
+}
+
+impl<'a> From<&'a mut Vec<u8>> for BooleanEncoder<'a> {
+    fn from(output: &'a mut Vec<u8>) -> Self {
+        BooleanEncoder::new(output)
+    }
+}
+
+/// See the discussion of [`BooleanEncoder`] for details on this encoding
+pub(crate) struct BooleanDecoder<'a> {
+    decoder: RawDecoder<'a>,
+    last_value: bool,
+    count: usize,
+}
+
+impl<'a> BooleanDecoder<'a> {
+    pub(crate) fn done(&self) -> bool {
+        self.decoder.done()
+    }
+
+    pub(crate) fn splice<I: Iterator<Item=bool>>(&mut self, replace: Range<usize>, mut replace_with: I, out: &mut Vec<u8>) -> Range<usize> {
+        let start = out.len();
+        let mut encoder = BooleanEncoder::new(out);
+        let mut idx = 0;
+        while idx < replace.start {
+            match self.next() {
+                Some(elem) => encoder.append(elem),
+                None => panic!("out of bounds"),
+            }
+            idx += 1;
+        }
+        for _ in 0..replace.len() {
+            self.next();
+            if let Some(next) = replace_with.next() {
+                encoder.append(next);
+            }
+        }
+        while let Some(next) = replace_with.next() {
+            encoder.append(next);
+        }
+        while let Some(next) = self.next() {
+            encoder.append(next);
+        }
+        start..(start + encoder.finish())
+    }
+}
+
+impl<'a> From<Cow<'a, [u8]>> for BooleanDecoder<'a> {
+    fn from(bytes: Cow<'a, [u8]>) -> Self {
+        BooleanDecoder {
+            decoder: RawDecoder::from(bytes),
+            last_value: true,
+            count: 0,
+        }
+    }
+}
+
+impl<'a> From<&'a [u8]> for BooleanDecoder<'a> {
+    fn from(d: &'a [u8]) -> Self {
+        Cow::Borrowed(d).into() 
+    }
+}
+
+// this is an endless iterator that returns false after input is exhausted
+impl<'a> Iterator for BooleanDecoder<'a> {
+    type Item = bool;
+
+    fn next(&mut self) -> Option<bool> {
+        while self.count == 0 {
+            if self.decoder.done() && self.count == 0 {
+                return None;
+            }
+            self.count = self.decoder.read().unwrap_or_default();
+            self.last_value = !self.last_value;
+        }
+        self.count -= 1;
+        Some(self.last_value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::columnar_2::rowblock::encoding::properties::splice_scenario;
+
+    use proptest::prelude::*;
+
+    fn encode(vals: &[bool]) -> Vec<u8> {
+        let mut buf = Vec::new();
+        let mut encoder = BooleanEncoder::new(&mut buf);
+        for val in vals {
+            encoder.append(*val);
+        }
+        encoder.finish();
+        buf
+    }
+
+    fn decode(buf: &[u8]) -> Vec<bool> {
+        BooleanDecoder::from(buf).collect()
+    }
+
+    proptest!{
+        #[test]
+        fn encode_decode_bools(vals in proptest::collection::vec(any::<bool>(), 0..100)) {
+            assert_eq!(vals, decode(&encode(&vals)))
+        }
+
+        #[test]
+        fn splice_bools(scenario in splice_scenario(any::<bool>())) {
+            let encoded = encode(&scenario.initial_values);
+            let mut decoder = BooleanDecoder::from(&encoded[..]);
+            let mut out = Vec::new();
+            let r = decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter().copied(), &mut out);
+            let result = decode(&out);
+            scenario.check(result);
+            assert_eq!(r.len(), out.len());
+        }
+    }
+
+}

automerge/src/columnar_2/rowblock/encoding/decodable_impls.rs

@@ -0,0 +1,158 @@
+use std::{borrow::Cow, convert::TryFrom, str, io::Read};
+use smol_str::SmolStr;
+
+use super::Decodable;
+use crate::ActorId;
+
+impl Decodable for u8 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let mut buffer = [0; 1];
+        bytes.read_exact(&mut buffer).ok()?;
+        Some(buffer[0])
+    }
+}
+
+impl Decodable for u32 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        u64::decode::<R>(bytes).and_then(|val| Self::try_from(val).ok())
+    }
+}
+
+impl Decodable for usize {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        u64::decode::<R>(bytes).and_then(|val| Self::try_from(val).ok())
+    }
+}
+
+impl Decodable for isize {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        i64::decode::<R>(bytes).and_then(|val| Self::try_from(val).ok())
+    }
+}
+
+impl Decodable for i32 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        i64::decode::<R>(bytes).and_then(|val| Self::try_from(val).ok())
+    }
+}
+
+impl Decodable for i64 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        leb128::read::signed(bytes).ok()
+    }
+}
+
+impl Decodable for f64 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let mut buffer = [0; 8];
+        bytes.read_exact(&mut buffer).ok()?;
+        Some(Self::from_le_bytes(buffer))
+    }
+}
+
+impl Decodable for f32 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let mut buffer = [0; 4];
+        bytes.read_exact(&mut buffer).ok()?;
+        Some(Self::from_le_bytes(buffer))
+    }
+}
+
+impl Decodable for u64 {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        leb128::read::unsigned(bytes).ok()
+    }
+}
+
+impl Decodable for Vec<u8> {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let len = usize::decode::<R>(bytes)?;
+        if len == 0 {
+            return Some(vec![]);
+        }
+        let mut buffer = vec![0; len];
+        bytes.read_exact(buffer.as_mut_slice()).ok()?;
+        Some(buffer)
+    }
+}
+
+impl Decodable for SmolStr {
+    fn decode<R>(bytes: &mut R) -> Option<SmolStr>
+    where
+        R: Read,
+    {
+        let buffer = Vec::decode(bytes)?;
+        str::from_utf8(&buffer).map(|t| t.into()).ok()
+    }
+}
+
+impl Decodable for Cow<'static, SmolStr> {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+            R: std::io::Read {
+        SmolStr::decode(bytes).map(|s| Cow::Owned(s))
+    }
+}
+
+impl Decodable for String {
+    fn decode<R>(bytes: &mut R) -> Option<String>
+    where
+        R: Read,
+    {
+        let buffer = Vec::decode(bytes)?;
+        str::from_utf8(&buffer).map(|t| t.into()).ok()
+    }
+}
+
+impl Decodable for Option<String> {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let buffer = Vec::decode(bytes)?;
+        if buffer.is_empty() {
+            return Some(None);
+        }
+        Some(str::from_utf8(&buffer).map(|t| t.into()).ok())
+    }
+}
+
+impl Decodable for ActorId {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: Read,
+    {
+        let buffer = Vec::decode(bytes)?;
+        Some(buffer.into())
+    }
+}

automerge/src/columnar_2/rowblock/encoding/delta.rs

@@ -0,0 +1,169 @@
+use std::{borrow::Cow, ops::Range};
+
+use super::{RleEncoder, RleDecoder};
+
+/// Encodes integers as the change since the previous value.
+///
+/// The initial value is 0 encoded as u64. Deltas are encoded as i64.
+///
+/// Run length encoding is then applied to the resulting sequence.
+pub(crate) struct DeltaEncoder<'a> {
+    rle: RleEncoder<'a, i64>,
+    absolute_value: i64,
+}
+
+impl<'a> DeltaEncoder<'a> {
+    pub fn new(output: &'a mut Vec<u8>) -> DeltaEncoder<'a> {
+        DeltaEncoder {
+            rle: RleEncoder::new(output),
+            absolute_value: 0,
+        }
+    }
+
+    pub fn append_value(&mut self, value: i64) {
+        self.rle
+            .append_value(&(value.saturating_sub(self.absolute_value)));
+        self.absolute_value = value;
+    }
+
+    pub fn append_null(&mut self) {
+        self.rle.append_null();
+    }
+
+    pub fn append(&mut self, val: Option<i64>) {
+        match val {
+            Some(v) => self.append_value(v),
+            None => self.append_null(),
+        }
+    }
+
+    pub fn finish(self) -> usize {
+        self.rle.finish()
+    }
+}
+
+impl<'a> From<&'a mut Vec<u8>> for DeltaEncoder<'a> {
+    fn from(output: &'a mut Vec<u8>) -> Self {
+        DeltaEncoder::new(output) 
+    }
+}
+
+/// See discussion on [`DeltaEncoder`] for the format data is stored in.
+#[derive(Clone)]
+pub(crate) struct DeltaDecoder<'a> {
+    rle: RleDecoder<'a, i64>,
+    absolute_val: i64,
+}
+
+impl<'a> DeltaDecoder<'a> {
+    pub(crate) fn done(&self) -> bool {
+        self.rle.done()
+    }
+
+    pub(crate) fn encode<I>(items: I, out: &mut Vec<u8>) -> Range<usize> 
+    where
+        I: Iterator<Item=i64>
+    {
+        let mut decoder = DeltaDecoder::from(&[] as &[u8]);
+        decoder.splice(0..0, items.map(Some), out)
+    }
+
+    pub(crate) fn splice<I: Iterator<Item=Option<i64>>>(&mut self, replace: Range<usize>, mut replace_with: I, out: &mut Vec<u8>) -> Range<usize> {
+        let start = out.len();
+        let mut encoder = DeltaEncoder::new(out);
+        let mut idx = 0;
+        while idx < replace.start {
+            match self.next() {
+                Some(elem) => encoder.append(elem),
+                None => panic!("out of bounds"),
+            }
+            idx += 1;
+        }
+        for _ in 0..replace.len() {
+            self.next();
+            if let Some(next) = replace_with.next() {
+                encoder.append(next);
+            }
+        }
+        while let Some(next) = replace_with.next() {
+            encoder.append(next);
+        }
+        while let Some(next) = self.next() {
+            encoder.append(next);
+        }
+        start..(start + encoder.finish())
+    }
+}
+
+impl<'a> From<Cow<'a, [u8]>> for DeltaDecoder<'a> {
+    fn from(bytes: Cow<'a, [u8]>) -> Self {
+        DeltaDecoder {
+            rle: RleDecoder::from(bytes),
+            absolute_val: 0,
+        }
+    }
+}
+
+impl<'a> From<&'a [u8]> for DeltaDecoder<'a> {
+    fn from(d: &'a [u8]) -> Self {
+        Cow::Borrowed(d).into() 
+    }
+}
+
+impl<'a> Iterator for DeltaDecoder<'a> {
+    type Item = Option<i64>;
+
+    fn next(&mut self) -> Option<Option<i64>> {
+        match self.rle.next() {
+            Some(Some(delta)) => {
+                self.absolute_val = self.absolute_val.saturating_add(delta);
+                Some(Some(self.absolute_val))
+            },
+            Some(None) => Some(None),
+            None => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+    use crate::columnar_2::rowblock::encoding::properties::splice_scenario;
+
+    fn encode(vals: &[Option<i64>]) -> Vec<u8> {
+        let mut buf = Vec::<u8>::new();
+        let mut encoder = DeltaEncoder::from(&mut buf);
+        for val in vals {
+            encoder.append(val.clone());
+        }
+        encoder.finish();
+        buf
+    }
+
+    fn decode(buf: &[u8]) -> Vec<Option<i64>> {
+        DeltaDecoder::from(buf).collect()
+    }
+
+    fn encodable_int() -> impl Strategy<Value = i64> + Clone {
+        0..(i64::MAX / 2)
+    }
+
+    proptest!{
+        #[test]
+        fn encode_decode_delta(vals in proptest::collection::vec(proptest::option::of(encodable_int()), 0..100)) {
+            assert_eq!(vals, decode(&encode(&vals)));
+        }
+
+        #[test]
+        fn splice_delta(scenario in splice_scenario(proptest::option::of(encodable_int()))) {
+            let encoded = encode(&scenario.initial_values);
+            let mut decoder = DeltaDecoder::from(&encoded[..]);
+            let mut out = Vec::new();
+            let r = decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter().cloned(), &mut out);
+            let decoded = decode(&out[..]);
+            scenario.check(decoded);
+            assert_eq!(r.len(), out.len());
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/encodable_impls.rs

@@ -0,0 +1,142 @@
+use super::Encodable;
+
+use std::borrow::Cow;
+
+use smol_str::SmolStr;
+use std::io::Write;
+
+/// Encodes bytes without a length prefix
+pub(crate) struct RawBytes<'a>(Cow<'a, [u8]>);
+
+impl<'a> From<&'a [u8]> for RawBytes<'a> {
+    fn from(r: &'a [u8]) -> Self {
+        RawBytes(r.into()) 
+    }
+}
+
+impl<'a> From<Cow<'a, [u8]>> for RawBytes<'a> {
+    fn from(c: Cow<'a, [u8]>) -> Self {
+        RawBytes(c)
+    }
+}
+
+impl<'a> Encodable for RawBytes<'a> {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        out.write_all(&self.0).unwrap();
+        self.0.len() 
+    }
+}
+
+impl Encodable for SmolStr {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        let bytes = self.as_bytes();
+        let len_encoded = bytes.len().encode(buf);
+        let data_len = bytes.encode(buf);
+        len_encoded + data_len
+    }
+}
+
+impl<'a> Encodable for Cow<'a, SmolStr> {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        self.as_ref().encode(buf)
+    }
+}
+
+impl Encodable for String {
+    fn encode(&self, buf: &mut Vec<u8>) ->usize {
+        let bytes = self.as_bytes();
+        let len_encoded = bytes.len().encode(buf);
+        let data_len = bytes.encode(buf);
+        len_encoded + data_len
+    }
+}
+
+impl Encodable for Option<String> {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        if let Some(s) = self {
+            s.encode(buf)
+        } else {
+            0.encode(buf)
+        }
+    }
+}
+
+impl<'a> Encodable for Option<Cow<'a, SmolStr>> {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        if let Some(s) = self {
+            SmolStr::encode(s, out)
+        } else {
+            0.encode(out)
+        }
+    }
+}
+
+impl Encodable for u64 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize{
+        leb128::write::unsigned(buf, *self).unwrap()
+    }
+}
+
+impl Encodable for f64 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        let bytes = self.to_le_bytes();
+        buf.write_all(&bytes).unwrap();
+        bytes.len()
+    }
+}
+
+impl Encodable for f32 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        let bytes = self.to_le_bytes();
+        buf.write_all(&bytes).unwrap();
+        bytes.len()
+    }
+}
+
+impl Encodable for i64 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        leb128::write::signed(buf, *self).unwrap()
+    }
+}
+
+impl Encodable for usize {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize{
+        (*self as u64).encode(buf)
+    }
+}
+
+impl Encodable for u32 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        u64::from(*self).encode(buf)
+    }
+}
+
+impl Encodable for i32 {
+    fn encode(&self, buf: &mut Vec<u8>) -> usize {
+        i64::from(*self).encode(buf)
+    }
+}
+
+impl Encodable for [u8] {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        out.write(self).unwrap()
+    }
+}
+
+impl Encodable for &[u8] {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        out.write(self).unwrap()
+    }
+}
+
+impl<'a> Encodable for Cow<'a, [u8]> {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        out.write(self).unwrap()
+    }
+}
+
+impl Encodable for Vec<u8> {
+    fn encode(&self, out: &mut Vec<u8>) -> usize {
+        Encodable::encode(&self[..], out)
+    }
+}

automerge/src/columnar_2/rowblock/encoding/generic.rs

@@ -0,0 +1,181 @@
+use std::{
+    borrow::Cow,
+    ops::Range,
+};
+
+use crate::columnar_2::rowblock::{column_layout::ColumnSpliceError, value::CellValue};
+
+use super::{
+    BooleanDecoder, DecodeColumnError, DeltaDecoder, RleDecoder,
+    ValueDecoder,
+};
+
+pub(crate) enum SimpleColDecoder<'a> {
+    RleUint(RleDecoder<'a, u64>),
+    RleString(RleDecoder<'a, smol_str::SmolStr>),
+    Delta(DeltaDecoder<'a>),
+    Bool(BooleanDecoder<'a>),
+}
+
+impl<'a> SimpleColDecoder<'a> {
+    pub(crate) fn new_uint(d: RleDecoder<'a, u64>) -> Self {
+        Self::RleUint(d)
+    }
+
+    pub(crate) fn new_string(d: RleDecoder<'a, smol_str::SmolStr>) -> Self {
+        Self::RleString(d)
+    }
+
+    pub(crate) fn new_delta(d: DeltaDecoder<'a>) -> Self {
+        Self::Delta(d)
+    }
+
+    pub(crate) fn new_bool(d: BooleanDecoder<'a>) -> Self {
+        Self::Bool(d)
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        match self {
+            Self::RleUint(d) => d.done(),
+            Self::RleString(d) => d.done(),
+            Self::Delta(d) => d.done(),
+            Self::Bool(d) => d.done(),
+        }
+    }
+
+    pub(crate) fn next(&mut self) -> Option<CellValue<'a>> {
+        match self {
+            Self::RleUint(d) => d.next().and_then(|i| i.map(CellValue::Uint)),
+            Self::RleString(d) => d
+                .next()
+                .and_then(|s| s.map(|s| CellValue::String(Cow::Owned(s.into())))),
+            Self::Delta(d) => d.next().and_then(|i| i.map(CellValue::Int)),
+            Self::Bool(d) => d.next().map(CellValue::Bool),
+        }
+    }
+
+    pub(crate) fn splice<'b, I>(
+        &mut self,
+        out: &mut Vec<u8>,
+        replace: Range<usize>,
+        replace_with: I,
+    ) -> Result<usize, ColumnSpliceError> 
+    where
+        I: Iterator<Item=CellValue<'b>> + Clone
+    {
+        // Requires `try_splice` methods on all the basic decoders so that we can report an error
+        // if the cellvalue types don't match up
+        unimplemented!()
+    }
+}
+
+
+pub(crate) enum SingleLogicalColDecoder<'a> {
+    Simple(SimpleColDecoder<'a>),
+    Value(ValueDecoder<'a>),
+}
+
+impl<'a> Iterator for SingleLogicalColDecoder<'a> {
+    type Item = Result<CellValue<'a>, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            Self::Simple(s) => s.next().map(Ok),
+            Self::Value(v) => v.next().map(|v| v.map(|v| CellValue::Value(v))),
+        }
+    }
+}
+
+pub(crate) enum GenericColDecoder<'a> {
+    Simple(SimpleColDecoder<'a>),
+    Value(ValueDecoder<'a>),
+    Group(GroupDecoder<'a>),
+}
+
+impl<'a> GenericColDecoder<'a> {
+    pub(crate) fn new_simple(s: SimpleColDecoder<'a>) -> Self {
+        Self::Simple(s)
+    }
+
+    pub(crate) fn new_value(v: ValueDecoder<'a>) -> Self {
+        Self::Value(v)
+    }
+
+    pub(crate) fn new_group(g: GroupDecoder<'a>) -> Self {
+        Self::Group(g)
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        match self {
+            Self::Simple(s) => s.done(),
+            Self::Group(g) => g.done(),
+            Self::Value(v) => v.done(),
+        }
+    }
+
+    pub(crate) fn next(&mut self) -> Option<Result<CellValue<'a>, DecodeColumnError>> {
+        match self {
+            Self::Simple(s) => s.next().map(Ok),
+            Self::Value(v) => v.next().map(|v| v.map(|v| CellValue::Value(v))),
+            Self::Group(g) => g.next().map(|v| v.map(|v| CellValue::List(v))),
+        }
+    }
+}
+
+impl<'a> Iterator for GenericColDecoder<'a> {
+    type Item = Result<CellValue<'a>, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        GenericColDecoder::next(self)
+    }
+}
+
+pub(crate) struct GroupDecoder<'a> {
+    num: RleDecoder<'a, u64>,
+    values: Vec<SingleLogicalColDecoder<'a>>,
+}
+
+impl<'a> GroupDecoder<'a> {
+    pub(crate) fn new(
+        num: RleDecoder<'a, u64>,
+        values: Vec<SingleLogicalColDecoder<'a>>,
+    ) -> GroupDecoder<'a> {
+        GroupDecoder { num, values }
+    }
+
+    fn next(&mut self) -> Option<Result<Vec<Vec<CellValue<'a>>>, DecodeColumnError>> {
+        match self.num.next() {
+            Some(Some(num_rows)) => {
+                let mut result = Vec::with_capacity(num_rows as usize);
+                for _ in 0..num_rows {
+                    let mut row = Vec::with_capacity(self.values.len());
+                    for (index, column) in self.values.iter_mut().enumerate() {
+                        match column.next() {
+                            Some(Ok(v)) => row.push(v),
+                            Some(Err(e)) => {
+                                return Some(Err(DecodeColumnError::InvalidValue {
+                                    column: format!("group column {0}", index + 1),
+                                    description: e.to_string(),
+                                }))
+                            }
+                            None => {
+                                return Some(Err(DecodeColumnError::UnexpectedNull(format!(
+                                    "grouped column {0}",
+                                    index + 1
+                                ))))
+                            }
+                        }
+                    }
+                    result.push(row)
+                }
+                Some(Ok(result))
+            }
+            Some(None) => Some(Err(DecodeColumnError::UnexpectedNull("num".to_string()))),
+            _ => None,
+        }
+    }
+
+    fn done(&self) -> bool {
+        self.num.done()
+    }
+}

automerge/src/columnar_2/rowblock/encoding/interned_key.rs

@@ -0,0 +1,41 @@
+use crate::types::{ElemId, Key, OpId};
+
+use super::{DeltaDecoder, RleDecoder};
+
+pub(crate) struct InternedKeyDecoder<'a> {
+    actor: RleDecoder<'a, u64>,
+    ctr: DeltaDecoder<'a>,
+    str_idx: RleDecoder<'a, u64>,
+}
+
+impl<'a> InternedKeyDecoder<'a> {
+    pub(crate) fn new(
+        actor: RleDecoder<'a, u64>,
+        ctr: DeltaDecoder<'a>,
+        str_idx: RleDecoder<'a, u64>,
+    ) -> Self {
+        Self {
+            actor,
+            ctr,
+            str_idx,
+        }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.actor.done() && self.ctr.done() && self.str_idx.done()
+    }
+}
+
+impl<'a> Iterator for InternedKeyDecoder<'a> {
+    type Item = Key;
+
+    fn next(&mut self) -> Option<Key> {
+        match (self.actor.next(), self.ctr.next(), self.str_idx.next()) {
+            (None, None, Some(Some(key_idx))) => Some(Key::Map(key_idx as usize)),
+            (None, Some(Some(0)), None) => Some(Key::Seq(ElemId(OpId(0, 0)))),
+            (Some(Some(actor)), Some(Some(ctr)), None) => Some(Key::Seq(OpId(actor, ctr as usize).into())),
+            // TODO: This should be fallible and throw here
+            _ => None,
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/key.rs

@@ -0,0 +1,119 @@
+use std::{borrow::Cow, ops::Range};
+
+use smol_str::SmolStr;
+
+use super::{DecodeColumnError, DeltaDecoder, RleDecoder};
+use crate::types::{ElemId, OpId};
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) enum Key {
+    Prop(smol_str::SmolStr),
+    Elem(ElemId),
+}
+
+pub(crate) struct KeyDecoder<'a> {
+    actor: RleDecoder<'a, u64>,
+    ctr: DeltaDecoder<'a>,
+    str: RleDecoder<'a, SmolStr>,
+}
+
+impl<'a> KeyDecoder<'a> {
+    pub(crate) fn new(
+        actor: RleDecoder<'a, u64>,
+        ctr: DeltaDecoder<'a>,
+        str: RleDecoder<'a, SmolStr>,
+    ) -> Self {
+        Self { actor, ctr, str }
+    }
+
+    pub(crate) fn empty() -> KeyDecoder<'static> {
+        KeyDecoder {
+            actor: RleDecoder::from(Cow::Owned(Vec::new())),
+            ctr: DeltaDecoder::from(Cow::Owned(Vec::new())),
+            str: RleDecoder::from(Cow::Owned(Vec::new())),
+        }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.actor.done() && self.ctr.done() && self.str.done()
+    }
+
+    /// Splice new keys into this set of keys, encoding the resulting actor, counter, and str
+    /// columns in `out`. The result is (actor, ctr, str) where actor is the range of the output which
+    /// contains the new actor column, ctr the counter column, and str the str column.
+    pub(crate) fn splice<'b, I: Iterator<Item = &'b Key> + Clone>(
+        &mut self,
+        replace: Range<usize>,
+        replace_with: I,
+        out: &mut Vec<u8>,
+    ) -> (Range<usize>, Range<usize>, Range<usize>) {
+        panic!()
+    }
+}
+
+impl<'a> Iterator for KeyDecoder<'a> {
+    type Item = Result<Key, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match (self.actor.next(), self.ctr.next(), self.str.next()) {
+            (Some(Some(_)), Some(Some(_)), Some(Some(_))) => {
+                Some(Err(DecodeColumnError::InvalidValue {
+                    column: "key".to_string(),
+                    description: "too many values".to_string(),
+                }))
+            }
+            (Some(None), Some(None), Some(Some(string))) => Some(Ok(Key::Prop(string))),
+            (Some(None), Some(Some(0)), Some(None)) => Some(Ok(Key::Elem(ElemId(OpId(0, 0))))),
+            (Some(Some(actor)), Some(Some(ctr)), Some(None)) => match ctr.try_into() {
+                Ok(ctr) => Some(Ok(Key::Elem(ElemId(OpId(ctr, actor as usize))))),
+                Err(e) => Some(Err(DecodeColumnError::InvalidValue{
+                    column: "counter".to_string(),
+                    description: "negative value for counter".to_string(),
+                })),
+            }
+            (None, None, None) => None,
+            (None | Some(None), _, _) => {
+                Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string())))
+            }
+            (_, None | Some(None), _) => {
+                Some(Err(DecodeColumnError::UnexpectedNull("ctr".to_string())))
+            }
+            (_, _, None) => Some(Err(DecodeColumnError::UnexpectedNull("str".to_string()))),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+
+    fn encode(vals: &[Key]) -> (Vec<u8>, Range<usize>, Range<usize>, Range<usize>) {
+        let mut out = Vec::new();
+        let mut decoder = KeyDecoder::empty();
+        let (actor, ctr, string) = decoder.splice(0..0, vals.iter(), &mut out);
+        (out, actor, ctr, string)
+    }
+
+    //proptest! {
+    //#[test]
+    //fn splice_key(scenario in splice_scenario(row_op_key())) {
+    //let (buf, actor, ctr, string) = encode(&scenario.initial_values[..]);
+    //let mut decoder = KeyDecoder::new(
+    //RleDecoder::from(&buf[actor]),
+    //DeltaDecoder::from(&buf[ctr]),
+    //RleDecoder::from(&buf[string]),
+    //);
+    //let mut out = Vec::new();
+    //let (actor, ctr, string) = decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter(), &mut out);
+    //let decoder = KeyDecoder::new(
+    //RleDecoder::from(&buf[actor]),
+    //DeltaDecoder::from(&buf[ctr]),
+    //RleDecoder::from(&buf[string.clone()]),
+    //);
+    //let result = decoder.map(|c| c.unwrap()).collect();
+    //scenario.check(result);
+    //assert_eq!(string.end, out.len());
+    //}
+    //}
+}

automerge/src/columnar_2/rowblock/encoding/mod.rs

@@ -0,0 +1,52 @@
+mod raw;
+use std::borrow::Borrow;
+
+pub(crate) use raw::{RawEncoder, RawDecoder};
+mod rle;
+pub(crate) use rle::{RleEncoder, RleDecoder};
+mod boolean;
+pub(crate) use boolean::{BooleanDecoder, BooleanEncoder};
+mod delta;
+pub(crate) use delta::{DeltaDecoder, DeltaEncoder};
+mod value;
+pub(crate) use value::ValueDecoder;
+pub(crate) mod generic;
+pub(crate) use generic::{GenericColDecoder, SimpleColDecoder};
+mod opid;
+pub(crate) use opid::OpIdDecoder;
+mod opid_list;
+pub(crate) use opid_list::OpIdListDecoder;
+mod obj_id;
+pub(crate) use obj_id::ObjDecoder;
+mod key;
+pub(crate) use key::{Key, KeyDecoder};
+
+#[cfg(test)]
+pub(crate) mod properties;
+
+
+
+pub(crate) trait Encodable {
+    fn encode(&self, out: &mut Vec<u8>) -> usize;
+}
+mod encodable_impls;
+pub(crate) use encodable_impls::RawBytes;
+
+pub(crate) trait Decodable: Sized {
+    fn decode<R>(bytes: &mut R) -> Option<Self>
+    where
+        R: std::io::Read;
+}
+mod decodable_impls;
+
+
+#[derive(Clone, thiserror::Error, Debug)]
+pub(crate) enum DecodeColumnError {
+    #[error("unexpected null decoding column {0}")]
+    UnexpectedNull(String),
+    #[error("invalid value in column {column}: {description}")]
+    InvalidValue{
+        column: String,
+        description: String,
+    },
+}

automerge/src/columnar_2/rowblock/encoding/obj_id.rs

@@ -0,0 +1,35 @@
+use crate::types::{OpId, ObjId};
+
+use super::{DecodeColumnError, RleDecoder};
+
+pub(crate) struct ObjDecoder<'a> {
+    actor: RleDecoder<'a, u64>,
+    ctr: RleDecoder<'a, u64>,
+}
+
+impl<'a> ObjDecoder<'a> {
+    pub(crate) fn new(actor: RleDecoder<'a, u64>, ctr: RleDecoder<'a, u64>) -> Self {
+        Self{
+            actor,
+            ctr,
+        }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.actor.done() || self.ctr.done()
+    }
+}
+
+impl<'a> Iterator for ObjDecoder<'a> {
+    type Item = Result<ObjId, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match (self.actor.next(), self.ctr.next()) {
+            (None, None) => None,
+            (Some(None), Some(None)) => Some(Ok(ObjId::root())),
+            (Some(Some(a)), Some(Some(c))) => Some(Ok(ObjId(OpId(c, a as usize)))),
+            (Some(None), _) | (None, _) => Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string()))),
+            (_, Some(None)) | (_, None) => Some(Err(DecodeColumnError::UnexpectedNull("counter".to_string()))),
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/opid.rs

@@ -0,0 +1,110 @@
+use std::{borrow::Cow, ops::Range};
+
+use crate::types::OpId;
+
+use super::{DecodeColumnError, DeltaDecoder, RleDecoder};
+
+pub(crate) struct OpIdDecoder<'a> {
+    actor: RleDecoder<'a, u64>,
+    ctr: DeltaDecoder<'a>,
+}
+
+impl Default for OpIdDecoder<'static> {
+    fn default() -> Self {
+        Self::new(
+            RleDecoder::from(Cow::Owned(Vec::new())),
+            DeltaDecoder::from(Cow::Owned(Vec::new())),
+        )
+    }
+}
+
+impl<'a> OpIdDecoder<'a> {
+    pub(crate) fn new(actor: RleDecoder<'a, u64>, ctr: DeltaDecoder<'a>) -> Self {
+        Self { actor, ctr }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.actor.done() && self.ctr.done()
+    }
+
+    /// Splice new operations into this set of operations, encoding the resulting actor and counter
+    /// columns in `out`. The result is (actor, ctr) where actor is the range of the output which
+    /// contains the new actor column and ctr the counter column.
+    pub(crate) fn splice<'b, I: Iterator<Item = &'b OpId> + Clone>(
+        &mut self,
+        replace: Range<usize>,
+        replace_with: I,
+        out: &mut Vec<u8>,
+    ) -> (Range<usize>, Range<usize>) {
+        // first splice actors, then counters
+        let actor = self.actor.splice(
+            replace.clone(),
+            replace_with.clone().map(|i| Some(i.actor() as u64)),
+            out,
+        );
+        let counter = self
+            .ctr
+            .splice(replace, replace_with.map(|i| Some(i.counter() as i64)), out);
+        (actor, counter)
+    }
+}
+
+impl<'a> Iterator for OpIdDecoder<'a> {
+    type Item = Result<OpId, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match (self.actor.next(), self.ctr.next()) {
+            (Some(Some(a)), Some(Some(c))) => match c.try_into() {
+                Ok(c) => Some(Ok(OpId(c, a as usize))),
+                Err(e) => Some(Err(DecodeColumnError::InvalidValue{
+                    column: "counter".to_string(),
+                    description: "negative value encountered".to_string(),
+                }))
+            },
+            (Some(None), _) => Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string()))),
+            (_, Some(None)) => Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string()))),
+            (Some(_), None) => Some(Err(DecodeColumnError::UnexpectedNull("ctr".to_string()))),
+            (None, Some(_)) => Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string()))),
+            (None, None) => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::columnar_2::rowblock::encoding::properties::{opid, splice_scenario};
+    use proptest::prelude::*;
+
+    fn encode(vals: &[OpId]) -> (Vec<u8>, Range<usize>, Range<usize>) {
+        let mut out = Vec::new();
+        let mut decoder = OpIdDecoder::default();
+        let (actor, ctr) = decoder.splice(0..0, vals.into_iter(), &mut out);
+        (out, actor, ctr)
+    }
+
+    fn decode(buf: &[u8], actor: Range<usize>, ctr: Range<usize>) -> Vec<OpId> {
+        OpIdDecoder::new(RleDecoder::from(&buf[actor]), DeltaDecoder::from(&buf[ctr]))
+            .map(|c| c.unwrap())
+            .collect()
+    }
+
+    proptest! {
+        #[test]
+        fn encode_decode_opid(opids in proptest::collection::vec(opid(), 0..100)) {
+            let (encoded, actor, ctr) = encode(&opids);
+            assert_eq!(opids, decode(&encoded[..], actor, ctr));
+        }
+
+        #[test]
+        fn splice_opids(scenario in splice_scenario(opid())) {
+            let (encoded, actor, ctr) = encode(&scenario.initial_values);
+            let mut decoder = OpIdDecoder::new(RleDecoder::from(&encoded[actor]), DeltaDecoder::from(&encoded[ctr]));
+            let mut out = Vec::new();
+            let (actor, ctr) = decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter(), &mut out);
+            let result = decode(&out[..], actor, ctr.clone());
+            scenario.check(result);
+            assert_eq!(ctr.end, out.len());
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/opid_list.rs

@@ -0,0 +1,182 @@
+use std::{borrow::Cow, ops::Range};
+
+use crate::types::OpId;
+
+use super::{DecodeColumnError, DeltaDecoder, RleDecoder};
+
+pub(crate) struct OpIdListDecoder<'a> {
+    num: RleDecoder<'a, u64>,
+    actor: RleDecoder<'a, u64>,
+    ctr: DeltaDecoder<'a>,
+}
+
+impl<'a> OpIdListDecoder<'a> {
+    pub(crate) fn new(
+        num: RleDecoder<'a, u64>,
+        actor: RleDecoder<'a, u64>,
+        ctr: DeltaDecoder<'a>,
+    ) -> Self {
+        Self { num, actor, ctr }
+    }
+
+    /// A decoder which references empty arrays, therefore has no elements
+    pub(crate) fn empty() -> OpIdListDecoder<'static> {
+        OpIdListDecoder {
+            num: RleDecoder::from(Cow::Owned(Vec::new())),
+            actor: RleDecoder::from(Cow::Owned(Vec::new())),
+            ctr: DeltaDecoder::from(Cow::Owned(Vec::new())),
+        }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.num.done()
+    }
+
+    /// Splice new lists of opids into this set of lists of opids, encoding the resulting num, actor and counter
+    /// columns in `out`. The result is (num, actor, ctr) where num is the range of the output which
+    /// contains the new num column, actor the actor column, and ctr the counter column
+    pub(crate) fn splice<'b, I, II, IE>(
+        &mut self,
+        replace: Range<usize>,
+        replace_with: I,
+        out: &mut Vec<u8>,
+    ) -> (Range<usize>, Range<usize>, Range<usize>)
+    where
+        II: IntoIterator<Item = OpId, IntoIter=IE>,
+        IE: Iterator<Item=OpId> + ExactSizeIterator,
+        I: Iterator<Item = II> + Clone,
+    {
+        let group_replace = group_replace_range(replace.clone(), self.num.clone());
+        // first nums
+        let num = self.num.splice(
+            replace.clone(),
+            replace_with.clone().map(|elems| Some(elems.into_iter().len() as u64)),
+            out,
+        );
+        let actor = self.actor.splice(
+            group_replace.clone(),
+            replace_with
+                .clone()
+                .flat_map(|elem| elem.into_iter().map(|oid| Some(oid.actor() as u64))),
+            out,
+        );
+        let ctr = self.ctr.splice(
+            group_replace,
+            replace_with.flat_map(|elem| elem.into_iter().map(|oid| Some(oid.counter() as i64))),
+            out,
+        );
+        (num, actor, ctr)
+    }
+}
+
+/// Find the replace range for the grouped columns.
+fn group_replace_range(replace: Range<usize>, mut num: RleDecoder<u64>) -> Range<usize> {
+    let mut idx = 0;
+    let mut grouped_replace_start: usize = 0;
+    let mut grouped_replace_len: usize = 0;
+    while idx < replace.start {
+        if let Some(Some(count)) = num.next() {
+            grouped_replace_start += count as usize;
+        }
+        idx += 1;
+    }
+    for _ in 0..replace.len() {
+        if let Some(Some(count)) = num.next() {
+            grouped_replace_len += count as usize;
+        }
+    }
+    grouped_replace_start..(grouped_replace_start + grouped_replace_len)
+}
+
+impl<'a> Iterator for OpIdListDecoder<'a> {
+    type Item = Result<Vec<OpId>, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let num = match self.num.next() {
+            Some(Some(n)) => n,
+            Some(None) => return Some(Err(DecodeColumnError::UnexpectedNull("num".to_string()))),
+            None => return None,
+        };
+        let mut p = Vec::with_capacity(num as usize);
+        for _ in 0..num {
+            match (self.actor.next(), self.ctr.next()) {
+                (Some(Some(a)), Some(Some(ctr))) => match ctr.try_into() {
+                    Ok(ctr) => p.push(OpId(ctr, a as usize)),
+                    Err(e) => return Some(Err(DecodeColumnError::InvalidValue{
+                        column: "counter".to_string(),
+                        description: "negative value for counter".to_string(),
+                    }))
+                },
+                (Some(None) | None, _) => {
+                    return Some(Err(DecodeColumnError::UnexpectedNull("actor".to_string())))
+                }
+                (_, Some(None) | None) => {
+                    return Some(Err(DecodeColumnError::UnexpectedNull("ctr".to_string())))
+                }
+            }
+        }
+        Some(Ok(p))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::collection::vec as propvec;
+    use proptest::prelude::*;
+
+    use crate::columnar_2::rowblock::encoding::properties::{opid, splice_scenario};
+
+    fn encode(opids: Vec<Vec<OpId>>) -> (Vec<u8>, Range<usize>, Range<usize>, Range<usize>) {
+        let mut out = Vec::new();
+        let mut decoder = OpIdListDecoder::empty();
+        let (num, actor, ctr) = decoder.splice(
+            0..0,
+            opids.into_iter(),
+            &mut out,
+        );
+        (out, num, actor, ctr)
+    }
+
+    fn decode(
+        buf: &[u8],
+        num: Range<usize>,
+        actor: Range<usize>,
+        ctr: Range<usize>,
+    ) -> Vec<Vec<OpId>> {
+        let decoder = OpIdListDecoder::new(
+            RleDecoder::from(&buf[num]),
+            RleDecoder::from(&buf[actor]),
+            DeltaDecoder::from(&buf[ctr]),
+        );
+        decoder.map(|c| c.unwrap()).collect()
+    }
+
+    proptest! {
+        #[test]
+        fn encode_decode_opid_list(opids in propvec(propvec(opid(), 0..100), 0..100)){
+            let (encoded, num, actor, ctr) = encode(opids.clone());
+            let result = decode(&encoded, num, actor, ctr);
+            assert_eq!(opids, result)
+        }
+
+        #[test]
+        fn splice_opid_list(scenario in splice_scenario(propvec(opid(), 0..100))) {
+            let (encoded, num, actor, ctr) = encode(scenario.initial_values.clone());
+            let mut decoder = OpIdListDecoder::new(
+                RleDecoder::from(&encoded[num]),
+                RleDecoder::from(&encoded[actor]),
+                DeltaDecoder::from(&encoded[ctr]),
+            );
+            let mut out = Vec::new();
+            let (num, actor, ctr) = decoder.splice(
+                scenario.replace_range.clone(),
+                scenario.replacements.clone().into_iter(),
+                &mut out
+                );
+            let result = decode(&out[..], num, actor, ctr.clone());
+            scenario.check(result);
+            assert_eq!(ctr.end, out.len())
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/properties.rs

@@ -0,0 +1,109 @@
+//! Helpers for property tests.
+
+use std::{borrow::Cow, fmt::Debug, ops::Range};
+
+use proptest::prelude::*;
+use smol_str::SmolStr;
+
+use crate::{
+    columnar_2::rowblock::{PrimVal, Key},
+    types::{OpId, Key as InternedKey, ElemId}
+};
+
+#[derive(Clone, Debug)]
+pub(crate) struct SpliceScenario<T> {
+    pub(crate) initial_values: Vec<T>,
+    pub(crate) replace_range: Range<usize>,
+    pub(crate) replacements: Vec<T>,
+}
+
+impl<T: Debug + PartialEq + Clone> SpliceScenario<T> {
+    pub(crate) fn check(&self, results: Vec<T>) {
+        let mut expected = self
+            .initial_values
+            .clone();
+        expected.splice(self.replace_range.clone(), self.replacements.clone());
+        assert_eq!(expected, results)
+    }
+}
+
+pub(crate) fn splice_scenario<S: Strategy<Value = T> + Clone, T: Debug + Clone + 'static>(
+    item_strat: S,
+) -> impl Strategy<Value = SpliceScenario<T>> {
+    (
+        proptest::collection::vec(item_strat.clone(), 0..100),
+        proptest::collection::vec(item_strat, 0..10),
+    )
+        .prop_flat_map(move |(values, to_splice)| {
+            if values.len() == 0 {
+                Just(SpliceScenario {
+                    initial_values: values.clone(),
+                    replace_range: 0..0,
+                    replacements: to_splice.clone(),
+                })
+                .boxed()
+            } else {
+                // This is somewhat awkward to write because we have to carry the `values` and
+                // `to_splice` through as `Just(..)` to please the borrow checker.
+                (0..values.len(), Just(values), Just(to_splice))
+                    .prop_flat_map(move |(replace_range_start, values, to_splice)| {
+                        (
+                            0..(values.len() - replace_range_start),
+                            Just(values),
+                            Just(to_splice),
+                        )
+                            .prop_map(
+                                move |(replace_range_len, values, to_splice)| SpliceScenario {
+                                    initial_values: values.clone(),
+                                    replace_range: replace_range_start
+                                        ..(replace_range_start + replace_range_len),
+                                    replacements: to_splice.clone(),
+                                },
+                            )
+                    })
+                    .boxed()
+            }
+        })
+}
+
+pub(crate) fn opid() -> impl Strategy<Value = OpId> + Clone {
+    (0..(i64::MAX as usize), 0..(i64::MAX as u64)).prop_map(|(actor, ctr)| OpId(ctr, actor))
+}
+
+pub(crate) fn elemid() -> impl Strategy<Value = ElemId> + Clone {
+    opid().prop_map(ElemId)
+}
+
+pub(crate) fn interned_key() -> impl Strategy<Value = InternedKey> + Clone {
+    prop_oneof!{
+        elemid().prop_map(InternedKey::Seq),
+        (0..(i64::MAX as usize)).prop_map(InternedKey::Map),
+    }
+}
+
+pub(crate) fn key() -> impl Strategy<Value = Key> + Clone {
+    prop_oneof!{
+        elemid().prop_map(Key::Elem),
+        any::<String>().prop_map(|s| Key::Prop(s.into())),
+    }
+}
+
+pub(crate) fn value() -> impl Strategy<Value = PrimVal<'static>> + Clone {
+    prop_oneof! {
+        Just(PrimVal::Null),
+        any::<bool>().prop_map(|b| PrimVal::Bool(b)),
+        any::<u64>().prop_map(|i| PrimVal::Uint(i)),
+        any::<i64>().prop_map(|i| PrimVal::Int(i)),
+        any::<f64>().prop_map(|f| PrimVal::Float(f)),
+        any::<String>().prop_map(|s| PrimVal::String(Cow::Owned(s.into()))),
+        any::<Vec<u8>>().prop_map(|b| PrimVal::Bytes(Cow::Owned(b))),
+        any::<u64>().prop_map(|i| PrimVal::Counter(i)),
+        any::<u64>().prop_map(|i| PrimVal::Timestamp(i)),
+        (10..15_u8, any::<Vec<u8>>()).prop_map(|(c, b)| PrimVal::Unknown { type_code: c, data: b }),
+    }
+}
+
+
+fn smol_str() -> impl Strategy<Value = SmolStr> + Clone {
+    any::<String>().prop_map(SmolStr::from)
+}

automerge/src/columnar_2/rowblock/encoding/raw.rs

@@ -1,29 +1,26 @@
-use std::{
-    borrow::{Borrow, Cow},
-    fmt::Debug,
-};
+use std::{borrow::Cow, fmt::Debug};
 
-use super::{Decodable, DecodeError, Encodable, Sink};
+use super::{Decodable, Encodable};
 
 #[derive(Clone, Debug)]
 pub(crate) struct RawDecoder<'a> {
-    offset: usize,
-    last_read: usize,
+    pub offset: usize,
+    pub last_read: usize,
     data: Cow<'a, [u8]>,
 }
 
 #[derive(thiserror::Error, Debug)]
 pub(crate) enum Error {
+    #[error("no decoded value")]
+    NoDecodedValue,
     #[error("buffer size did not change")]
     BufferSizeDidNotChange,
     #[error("trying to read past end")]
     TryingToReadPastEnd,
-    #[error(transparent)]
-    Decode(#[from] DecodeError),
 }
 
 impl<'a> RawDecoder<'a> {
-    pub(crate) fn new(data: Cow<'a, [u8]>) -> Self {
+    pub fn new(data: Cow<'a, [u8]>) -> Self {
         RawDecoder {
             offset: 0,
             last_read: 0,
@@ -31,10 +28,10 @@ impl<'a> RawDecoder<'a> {
         }
     }
 
-    pub(crate) fn read<T: Decodable + Debug>(&mut self) -> Result<T, Error> {
+    pub fn read<T: Decodable + Debug>(&mut self) -> Result<T, Error> {
         let mut buf = &self.data[self.offset..];
         let init_len = buf.len();
-        let val = T::decode::<&[u8]>(&mut buf)?;
+        let val = T::decode::<&[u8]>(&mut buf).ok_or(Error::NoDecodedValue)?;
         let delta = init_len - buf.len();
         if delta == 0 {
             Err(Error::BufferSizeDidNotChange)
@@ -45,7 +42,7 @@ impl<'a> RawDecoder<'a> {
         }
     }
 
-    pub(crate) fn read_bytes(&mut self, index: usize) -> Result<&[u8], Error> {
+    pub fn read_bytes(&mut self, index: usize) -> Result<&[u8], Error> {
         if self.offset + index > self.data.len() {
             Err(Error::TryingToReadPastEnd)
         } else {
@@ -56,7 +53,7 @@ impl<'a> RawDecoder<'a> {
         }
     }
 
-    pub(crate) fn done(&self) -> bool {
+    pub fn done(&self) -> bool {
         self.offset >= self.data.len()
     }
 }
@@ -73,25 +70,27 @@ impl<'a> From<Cow<'a, [u8]>> for RawDecoder<'a> {
     }
 }
 
-pub(crate) struct RawEncoder<S> {
+
+pub(crate) struct RawEncoder<'a> {
     written: usize,
-    output: S,
+    output: &'a mut Vec<u8>,
 }
 
-impl<S: Sink> RawEncoder<S> {
-    pub(crate) fn append<B: Borrow<I>, I: Encodable>(&mut self, value: B) -> usize {
-        let written = value.borrow().encode(&mut self.output);
+impl<'a> RawEncoder<'a> {
+    pub(crate) fn append<I: Encodable>(&mut self, value: &I) -> usize {
+        let written = value.encode(&mut self.output);
         self.written += written;
         written
     }
 
-    pub(crate) fn finish(self) -> (S, usize) {
-        (self.output, self.written)
+    fn finish(self) -> usize {
+        self.written
     }
 }
 
-impl<S: Sink> From<S> for RawEncoder<S> {
-    fn from(output: S) -> Self {
-        RawEncoder { written: 0, output }
+impl<'a> From<&'a mut Vec<u8>> for RawEncoder<'a> {
+    fn from(output: &'a mut Vec<u8>) -> Self {
+        RawEncoder{ written: 0, output }
     }
 }
+

automerge/src/columnar_2/rowblock/encoding/rle.rs

@@ -0,0 +1,364 @@
+use std::{
+    borrow::{Borrow, Cow},
+    fmt::Debug,
+    ops::Range,
+};
+
+use super::{Encodable, Decodable, RawDecoder};
+
+pub(crate) struct RleEncoder<'a, T>
+where
+    T: Encodable + PartialEq + Clone,
+{
+    buf: &'a mut Vec<u8>,
+    written: usize,
+    state: RleState<T>,
+}
+
+impl<'a, T> RleEncoder<'a, T>
+where
+    T: Encodable + PartialEq + Clone,
+{
+    pub fn new(output_buf: &'a mut Vec<u8>) -> RleEncoder<'a, T> {
+        RleEncoder {
+            buf: output_buf,
+            written: 0,
+            state: RleState::Empty,
+        }
+    }
+
+    pub fn finish(mut self) -> usize {
+        match self.take_state() {
+            // this covers `only_nulls`
+            RleState::NullRun(size) => {
+                self.flush_null_run(size);
+            }
+            RleState::LoneVal(value) => self.flush_lit_run(vec![value]),
+            RleState::Run(value, len) => self.flush_run(&value, len),
+            RleState::LiteralRun(last, mut run) => {
+                run.push(last);
+                self.flush_lit_run(run);
+            }
+            RleState::Empty => {}
+        }
+        self.written
+    }
+
+    fn flush_run(&mut self, val: &T, len: usize) {
+        self.encode(&(len as i64));
+        self.encode(val);
+    }
+
+    fn flush_null_run(&mut self, len: usize) {
+        self.encode::<i64>(&0);
+        self.encode(&len);
+    }
+
+    fn flush_lit_run(&mut self, run: Vec<T>) {
+        self.encode(&-(run.len() as i64));
+        for val in run {
+            self.encode(&val);
+        }
+    }
+
+    fn take_state(&mut self) -> RleState<T> {
+        let mut state = RleState::Empty;
+        std::mem::swap(&mut self.state, &mut state);
+        state
+    }
+
+    pub fn append_null(&mut self) {
+        self.state = match self.take_state() {
+            RleState::Empty => RleState::NullRun(1),
+            RleState::NullRun(size) => RleState::NullRun(size + 1),
+            RleState::LoneVal(other) => {
+                self.flush_lit_run(vec![other]);
+                RleState::NullRun(1)
+            }
+            RleState::Run(other, len) => {
+                self.flush_run(&other, len);
+                RleState::NullRun(1)
+            }
+            RleState::LiteralRun(last, mut run) => {
+                run.push(last);
+                self.flush_lit_run(run);
+                RleState::NullRun(1)
+            }
+        }
+    }
+
+    pub fn append_value(&mut self, value: &T) {
+        self.state = match self.take_state() {
+            RleState::Empty => RleState::LoneVal(value.clone()),
+            RleState::LoneVal(other) => {
+                if &other == value {
+                    RleState::Run(value.clone(), 2)
+                } else {
+                    let mut v = Vec::with_capacity(2);
+                    v.push(other);
+                    RleState::LiteralRun(value.clone(), v)
+                }
+            }
+            RleState::Run(other, len) => {
+                if &other == value {
+                    RleState::Run(other, len + 1)
+                } else {
+                    self.flush_run(&other, len);
+                    RleState::LoneVal(value.clone())
+                }
+            }
+            RleState::LiteralRun(last, mut run) => {
+                if &last == value {
+                    self.flush_lit_run(run);
+                    RleState::Run(value.clone(), 2)
+                } else {
+                    run.push(last);
+                    RleState::LiteralRun(value.clone(), run)
+                }
+            }
+            RleState::NullRun(size) => {
+                self.flush_null_run(size);
+                RleState::LoneVal(value.clone())
+            }
+        }
+    }
+
+    pub fn append(&mut self, value: Option<&T>) {
+        match value {
+            Some(t) => self.append_value(t),
+            None => self.append_null(),
+        }
+    }
+
+    fn encode<V>(&mut self, val: &V)
+    where
+        V: Encodable,
+    {
+        self.written += val.encode(&mut self.buf);
+    }
+}
+
+enum RleState<T> {
+    Empty,
+    NullRun(usize),
+    LiteralRun(T, Vec<T>),
+    LoneVal(T),
+    Run(T, usize),
+}
+
+impl<'a, T: Clone + PartialEq + Encodable> From<&'a mut Vec<u8>> for RleEncoder<'a, T> {
+    fn from(output: &'a mut Vec<u8>) -> Self {
+        Self::new(output) 
+    }
+}
+
+/// See discussion on [`RleEncoder`] for the format data is stored in.
+#[derive(Clone, Debug)]
+pub(crate) struct RleDecoder<'a, T> {
+    pub decoder: RawDecoder<'a>,
+    last_value: Option<T>,
+    count: isize,
+    literal: bool,
+}
+
+impl<'a, T> RleDecoder<'a, T> {
+    fn empty() -> Self {
+        RleDecoder{
+            decoder: RawDecoder::from(&[] as &[u8]),
+            last_value: None,
+            count: 0,
+            literal: false,
+        }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.decoder.done() && self.count == 0
+    }
+}
+
+impl<'a, T: Clone + Debug + Encodable + Decodable + Eq> RleDecoder<'a, T> {
+
+    pub(crate) fn encode<I>(items: I, out: &'a mut Vec<u8>) -> Range<usize>
+    where
+        I: Iterator<Item=T>
+    {
+        let mut empty = RleDecoder::empty();
+        let range = empty.splice(0..0, items.map(Some), out);
+        range
+    }
+
+    pub(crate) fn splice<I: Iterator<Item=Option<TB>>, TB: Borrow<T>>(&mut self, replace: Range<usize>, mut replace_with: I, out: &mut Vec<u8>) -> Range<usize> {
+        let start = out.len();
+        let mut encoder = RleEncoder::new(out);
+        let mut idx = 0;
+        while idx < replace.start {
+            match self.next() {
+                Some(elem) => encoder.append(elem.as_ref()),
+                None => panic!("out of bounds"),
+            }
+            idx += 1;
+        }
+        for _ in 0..replace.len() {
+            self.next();
+            if let Some(next) = replace_with.next() {
+                encoder.append(next.as_ref().map(|n| n.borrow()));
+            }
+        }
+        while let Some(next) = replace_with.next() {
+            encoder.append(next.as_ref().map(|n| n.borrow()));
+        }
+        while let Some(next) = self.next() {
+            encoder.append(next.as_ref());
+        }
+        start..(start + encoder.finish())
+    }
+}
+
+impl<'a, T> From<Cow<'a, [u8]>> for RleDecoder<'a, T> {
+    fn from(bytes: Cow<'a, [u8]>) -> Self {
+        RleDecoder {
+            decoder: RawDecoder::from(bytes),
+            last_value: None,
+            count: 0,
+            literal: false,
+        }
+    }
+}
+
+impl<'a, T> From<&'a [u8]> for RleDecoder<'a, T> {
+    fn from(d: &'a [u8]) -> Self {
+        Cow::Borrowed(d).into()
+    }
+}
+
+// this decoder needs to be able to send type T or 'null'
+// it is an endless iterator that will return all 'null's
+// once input is exhausted
+impl<'a, T> Iterator for RleDecoder<'a, T>
+where
+    T: Clone + Debug + Decodable,
+{
+    type Item = Option<T>;
+
+    fn next(&mut self) -> Option<Option<T>> {
+        while self.count == 0 {
+            if self.decoder.done() {
+                return None;
+            }
+            match self.decoder.read::<i64>() {
+                Ok(count) if count > 0 => {
+                    // normal run
+                    self.count = count as isize;
+                    self.last_value = self.decoder.read().ok();
+                    self.literal = false;
+                }
+                Ok(count) if count < 0 => {
+                    // literal run
+                    self.count = count.abs() as isize;
+                    self.literal = true;
+                }
+                Ok(_) => {
+                    // null run
+                    // FIXME(jeffa5): handle usize > i64 here somehow
+                    self.count = self.decoder.read::<usize>().unwrap() as isize;
+                    self.last_value = None;
+                    self.literal = false;
+                }
+                Err(e) => {
+                    tracing::warn!(error=?e, "error during rle decoding");
+                    return None;
+                }
+            }
+        }
+        self.count -= 1;
+        if self.literal {
+            Some(self.decoder.read().ok())
+        } else {
+            Some(self.last_value.clone())
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+    use super::*;
+    use proptest::prelude::*;
+    use super::super::properties::splice_scenario;
+
+    #[test]
+    fn rle_int_round_trip() {
+        let vals = [1,1,2,2,3,2,3,1,3];
+        let mut buf = Vec::with_capacity(vals.len() * 3);
+        let mut encoder: RleEncoder<'_, u64> = RleEncoder::new(&mut buf);
+        for val in vals {
+            encoder.append_value(&val)
+        }
+        let total_slice_len = encoder.finish();
+        let mut decoder: RleDecoder<'_, u64> = RleDecoder::from(Cow::Borrowed(&buf[0..total_slice_len]));
+        let mut result = Vec::new();
+        while let Some(Some(val)) = decoder.next() {
+            result.push(val);
+        }
+        assert_eq!(result, vals);
+    }
+
+    #[test]
+    fn rle_int_insert() {
+        let vals = [1,1,2,2,3,2,3,1,3];
+        let mut buf = Vec::with_capacity(vals.len() * 3);
+        let mut encoder: RleEncoder<'_, u64> = RleEncoder::new(&mut buf);
+        for i in 0..4 {
+            encoder.append_value(&vals[i])
+        }
+        encoder.append_value(&5);
+        for i in 4..vals.len() {
+            encoder.append_value(&vals[i]);
+        }
+        let total_slice_len = encoder.finish();
+        let mut decoder: RleDecoder<'_, u64> = RleDecoder::from(Cow::Borrowed(&buf[0..total_slice_len]));
+        let mut result = Vec::new();
+        while let Some(Some(val)) = decoder.next() {
+            result.push(val);
+        }
+        let expected = [1,1,2,2,5,3,2,3,1,3];
+        assert_eq!(result, expected);
+    }
+
+    fn encode<T: Clone + Encodable + PartialEq>(vals: &[Option<T>]) -> Vec<u8> {
+        let mut buf = Vec::with_capacity(vals.len() * 3);
+        let mut encoder: RleEncoder<'_, T> = RleEncoder::new(&mut buf);
+        for val in vals {
+            encoder.append(val.as_ref())
+        }
+        encoder.finish();
+        buf
+    }
+
+    fn decode<T: Clone + Decodable + Debug>(buf: Vec<u8>) -> Vec<Option<T>> {
+        let decoder = RleDecoder::<'_, T>::from(&buf[..]);
+        decoder.collect()
+    }
+
+    proptest!{
+        #[test]
+        fn splice_ints(scenario in splice_scenario(any::<Option<i32>>())) {
+            let buf = encode(&scenario.initial_values);
+            let mut decoder = RleDecoder::<'_, i32>::from(&buf[..]);
+            let mut out = Vec::new();
+            decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter().cloned(), &mut out);
+            let result = decode::<i32>(out);
+            scenario.check(result)
+        }
+
+        #[test]
+        fn splice_strings(scenario in splice_scenario(any::<Option<String>>())) {
+            let buf = encode(&scenario.initial_values);
+            let mut decoder = RleDecoder::<'_, String>::from(&buf[..]);
+            let mut out = Vec::new();
+            decoder.splice(scenario.replace_range.clone(), scenario.replacements.iter().cloned(), &mut out);
+            let result = decode::<String>(out);
+            scenario.check(result)
+        }
+    }
+}

automerge/src/columnar_2/rowblock/encoding/value.rs

@@ -0,0 +1,475 @@
+use crate::columnar_2::rowblock::column_layout::ColumnSpliceError;
+use std::{borrow::Cow, convert::TryInto, ops::Range};
+
+use super::{DecodeColumnError, RawDecoder, RawEncoder, RleDecoder, RleEncoder, RawBytes};
+use crate::columnar_2::rowblock::value::PrimVal;
+
+#[derive(Clone)]
+pub(crate) struct ValueDecoder<'a> {
+    meta: RleDecoder<'a, u64>,
+    raw: RawDecoder<'a>,
+}
+
+impl<'a> ValueDecoder<'a> {
+    pub(crate) fn new(meta: RleDecoder<'a, u64>, raw: RawDecoder<'a>) -> ValueDecoder<'a> {
+        ValueDecoder { meta, raw }
+    }
+
+    pub(crate) fn done(&self) -> bool {
+        self.meta.done()
+    }
+
+    pub(crate) fn next(&mut self) -> Option<Result<PrimVal<'a>, DecodeColumnError>> {
+        match self.meta.next() {
+            Some(Some(next)) => {
+                let val_meta = ValueMeta::from(next);
+                #[allow(clippy::redundant_slicing)]
+                match val_meta.type_code() {
+                    ValueType::Null => Some(Ok(PrimVal::Null)),
+                    ValueType::True => Some(Ok(PrimVal::Bool(true))),
+                    ValueType::False => Some(Ok(PrimVal::Bool(false))),
+                    ValueType::Uleb => self.parse_raw(val_meta, |mut bytes| {
+                        let val = leb128::read::unsigned(&mut bytes).map_err(|e| {
+                            DecodeColumnError::InvalidValue {
+                                column: "value".to_string(),
+                                description: e.to_string(),
+                            }
+                        })?;
+                        Ok(PrimVal::Uint(val))
+                    }),
+                    ValueType::Leb => self.parse_raw(val_meta, |mut bytes| {
+                        let val = leb128::read::signed(&mut bytes).map_err(|e| {
+                            DecodeColumnError::InvalidValue {
+                                column: "value".to_string(),
+                                description: e.to_string(),
+                            }
+                        })?;
+                        Ok(PrimVal::Int(val))
+                    }),
+                    ValueType::String => self.parse_raw(val_meta, |bytes| {
+                        let val = Cow::Owned(
+                            std::str::from_utf8(bytes)
+                                .map_err(|e| DecodeColumnError::InvalidValue {
+                                    column: "value".to_string(),
+                                    description: e.to_string(),
+                                })?
+                                .into(),
+                        );
+                        Ok(PrimVal::String(val))
+                    }),
+                    ValueType::Float => self.parse_raw(val_meta, |bytes| {
+                        if val_meta.length() != 8 {
+                            return Err(DecodeColumnError::InvalidValue {
+                                column: "value".to_string(),
+                                description: format!(
+                                    "float should have length 8, had {0}",
+                                    val_meta.length()
+                                ),
+                            });
+                        }
+                        let raw: [u8; 8] = bytes
+                            .try_into()
+                            // SAFETY: parse_raw() calls read_bytes(val_meta.length()) and we have
+                            //         checked that val_meta.length() == 8
+                            .unwrap();
+                        let val = f64::from_le_bytes(raw);
+                        Ok(PrimVal::Float(val))
+                    }),
+                    ValueType::Counter => self.parse_raw(val_meta, |mut bytes| {
+                        let val = leb128::read::unsigned(&mut bytes).map_err(|e| {
+                            DecodeColumnError::InvalidValue {
+                                column: "value".to_string(),
+                                description: e.to_string(),
+                            }
+                        })?;
+                        Ok(PrimVal::Counter(val))
+                    }),
+                    ValueType::Timestamp => self.parse_raw(val_meta, |mut bytes| {
+                        let val = leb128::read::unsigned(&mut bytes).map_err(|e| {
+                            DecodeColumnError::InvalidValue {
+                                column: "value".to_string(),
+                                description: e.to_string(),
+                            }
+                        })?;
+                        Ok(PrimVal::Timestamp(val))
+                    }),
+                    ValueType::Unknown(code) => self.parse_raw(val_meta, |bytes| {
+                        Ok(PrimVal::Unknown {
+                            type_code: code,
+                            data: bytes.to_vec(),
+                        })
+                    }),
+                    ValueType::Bytes => match self.raw.read_bytes(val_meta.length()) {
+                        Err(e) => Some(Err(DecodeColumnError::InvalidValue {
+                            column: "value".to_string(),
+                            description: e.to_string(),
+                        })),
+                        Ok(bytes) => Some(Ok(PrimVal::Bytes(Cow::Owned(bytes.to_vec())))),
+                    },
+                }
+            }
+            Some(None) => Some(Err(DecodeColumnError::UnexpectedNull("meta".to_string()))),
+            None => None,
+        }
+    }
+
+    pub(crate) fn splice<'b, I>(
+        &'a mut self,
+        replace: Range<usize>,
+        replace_with: I,
+        out: &mut Vec<u8>,
+    ) -> (Range<usize>, Range<usize>)
+    where
+        I: Iterator<Item = PrimVal<'a>> + Clone,
+    {
+        // SAFETY: try_splice only fails if the iterator fails, and this iterator is infallible
+        self.try_splice(replace, replace_with.map(|i| Ok(i)), out).unwrap()
+    }
+
+    pub(crate) fn try_splice<'b, I>(
+        &'a mut self,
+        replace: Range<usize>,
+        mut replace_with: I,
+        out: &mut Vec<u8>,
+    ) -> Result<(Range<usize>, Range<usize>), ColumnSpliceError>
+    where
+        I: Iterator<Item = Result<PrimVal<'a>, ColumnSpliceError>> + Clone,
+    {
+        // Our semantics here are similar to those of Vec::splice. We can describe this
+        // imperatively like this:
+        //
+        // * First copy everything up to the start of `replace` into the output
+        // * For every index in `replace` skip that index from ourselves and if `replace_with`
+        //   returns `Some` then copy that value to the output
+        // * Once we have iterated past `replace.end` we continue to call `replace_with` until it
+        //   returns None, copying the results to the output
+        // * Finally we copy the remainder of our data into the output
+        //
+        // However, things are complicated by the fact that our data is stored in two columns. This
+        // means that we do this in two passes. First we execute the above logic for the metadata
+        // column. Then we do it all over again for the value column.
+
+        // First pass - metadata
+        //
+        // Copy the metadata decoder so we can iterate over it again when we read the values in the
+        // second pass
+        let start = out.len();
+        let mut meta_copy = self.meta.clone();
+        let mut meta_out = RleEncoder::from(&mut *out);
+        let mut idx = 0;
+        // Copy everything up to replace.start to the output
+        while idx < replace.start {
+            let val = meta_copy.next().unwrap_or(None);
+            meta_out.append(val.as_ref());
+            idx += 1;
+        }
+        // Now step through replace, skipping our data and inserting the replacement data (if there
+        // is any)
+        let mut meta_replace_with = replace_with.clone();
+        for _ in 0..replace.len() {
+            meta_copy.next();
+            if let Some(val) = meta_replace_with.next() {
+                let val = val?;
+                // Note that we are just constructing metadata values here.
+                let meta_val = &u64::from(ValueMeta::from(&val));
+                meta_out.append(Some(meta_val));
+            }
+            idx += 1;
+        }
+        // Copy any remaining input from the replacments to the output
+        while let Some(val) = meta_replace_with.next() {
+            let val = val?;
+            let meta_val = &u64::from(ValueMeta::from(&val));
+            meta_out.append(Some(meta_val));
+            idx += 1;
+        }
+        // Now copy any remaining data we have to the output
+        while !meta_copy.done() {
+            let val = meta_copy.next().unwrap_or(None);
+            meta_out.append(val.as_ref());
+        }
+        let meta_len = meta_out.finish();
+        let meta_range = start..(start + meta_len);
+
+        // Second pass, copying the values. For this pass we iterate over ourselves.
+        //
+        //
+        let mut value_range_len = 0;
+        let mut raw_encoder = RawEncoder::from(out);
+        idx = 0;
+        // Copy everything up to replace.start to the output
+        while idx < replace.start {
+            let val = self.next().unwrap().unwrap_or(PrimVal::Null);
+            value_range_len += encode_primval(&mut raw_encoder, &val);
+            idx += 1;
+        }
+
+        // Now step through replace, skipping our data and inserting the replacement data (if there
+        // is any)
+        for _ in 0..replace.len() {
+            self.next();
+            if let Some(val) = replace_with.next() {
+                let val = val?;
+                value_range_len += encode_primval(&mut raw_encoder, &val);
+            }
+            idx += 1;
+        }
+        // Copy any remaining input from the replacments to the output
+        while let Some(val) = replace_with.next() {
+            let val = val?;
+            value_range_len += encode_primval(&mut raw_encoder, &val);
+            idx += 1;
+        }
+        // Now copy any remaining data we have to the output
+        while !self.done() {
+            let val = self.next().unwrap().unwrap_or(PrimVal::Null);
+            value_range_len += encode_primval(&mut raw_encoder, &val);
+        }
+
+        let value_range = meta_range.end..(meta_range.end + value_range_len);
+
+        Ok((meta_range, value_range))
+    }
+
+    fn parse_raw<R, F: Fn(&[u8]) -> Result<R, DecodeColumnError>>(
+        &mut self,
+        meta: ValueMeta,
+        f: F,
+    ) -> Option<Result<R, DecodeColumnError>> {
+        let raw = match self.raw.read_bytes(meta.length()) {
+            Err(e) => {
+                return Some(Err(DecodeColumnError::InvalidValue {
+                    column: "value".to_string(),
+                    description: e.to_string(),
+                }))
+            }
+            Ok(bytes) => bytes,
+        };
+        let val = match f(&mut &raw[..]) {
+            Ok(v) => v,
+            Err(e) => return Some(Err(e)),
+        };
+        Some(Ok(val))
+    }
+}
+
+fn encode_primval(out: &mut RawEncoder, val: &PrimVal) -> usize {
+    match val {
+        PrimVal::Uint(i) => out.append(i),
+        PrimVal::Int(i) => out.append(i),
+        PrimVal::Null => 0,
+        PrimVal::Bool(_) => 0,
+        PrimVal::Timestamp(i) => out.append(i),
+        PrimVal::Float(f) => out.append(f),
+        PrimVal::Counter(i) => out.append(i),
+        PrimVal::String(s) => out.append(&RawBytes::from(s.as_bytes())),
+        PrimVal::Bytes(b) => out.append(&RawBytes::from(&b[..])),
+        PrimVal::Unknown { data, .. } => out.append(&RawBytes::from(&data[..])),
+    }
+}
+
+impl<'a> Iterator for ValueDecoder<'a> {
+    type Item = Result<PrimVal<'a>, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        ValueDecoder::next(self)
+    }
+}
+
+enum ValueType {
+    Null,
+    False,
+    True,
+    Uleb,
+    Leb,
+    Float,
+    String,
+    Bytes,
+    Counter,
+    Timestamp,
+    Unknown(u8),
+}
+
+#[derive(Copy, Clone)]
+struct ValueMeta(u64);
+
+impl ValueMeta {
+    fn type_code(&self) -> ValueType {
+        let low_byte = (self.0 & 0b00001111) as u8;
+        match low_byte {
+            0 => ValueType::Null,
+            1 => ValueType::False,
+            2 => ValueType::True,
+            3 => ValueType::Uleb,
+            4 => ValueType::Leb,
+            5 => ValueType::Float,
+            6 => ValueType::String,
+            7 => ValueType::Bytes,
+            8 => ValueType::Counter,
+            9 => ValueType::Timestamp,
+            other => ValueType::Unknown(other),
+        }
+    }
+
+    fn length(&self) -> usize {
+        (self.0 >> 4) as usize
+    }
+}
+
+impl<'a> From<&PrimVal<'a>> for ValueMeta {
+    fn from(p: &PrimVal<'a>) -> Self {
+        match p {
+            PrimVal::Uint(i) => Self((ulebsize(*i) << 4) | 3),
+            PrimVal::Int(i) => Self((lebsize(*i) << 4) | 4),
+            PrimVal::Null => Self(0),
+            PrimVal::Bool(b) => Self(match b {
+                false => 1,
+                true => 2,
+            }),
+            PrimVal::Timestamp(i) => Self((ulebsize(*i) << 4) | 9),
+            PrimVal::Float(_) => Self((8 << 4) | 5),
+            PrimVal::Counter(i) => Self((ulebsize(*i) << 4) | 8),
+            PrimVal::String(s) => Self(((s.as_bytes().len() as u64) << 4) | 6),
+            PrimVal::Bytes(b) => Self(((b.len() as u64) << 4) | 7),
+            PrimVal::Unknown { type_code, data } => {
+                Self(((data.len() as u64) << 4) | (*type_code as u64))
+            }
+        }
+    }
+}
+
+impl From<u64> for ValueMeta {
+    fn from(raw: u64) -> Self {
+        ValueMeta(raw)
+    }
+}
+
+impl From<ValueMeta> for u64 {
+    fn from(v: ValueMeta) -> Self {
+        v.0
+    }
+}
+
+impl<'a> From<&PrimVal<'a>> for ValueType {
+    fn from(p: &PrimVal) -> Self {
+        match p {
+            PrimVal::Uint(_) => ValueType::Uleb,
+            PrimVal::Int(_) => ValueType::Leb,
+            PrimVal::Null => ValueType::Null,
+            PrimVal::Bool(b) => match b {
+                true => ValueType::True,
+                false => ValueType::False,
+            },
+            PrimVal::Timestamp(_) => ValueType::Timestamp,
+            PrimVal::Float(_) => ValueType::Float,
+            PrimVal::Counter(_) => ValueType::Counter,
+            PrimVal::String(_) => ValueType::String,
+            PrimVal::Bytes(_) => ValueType::Bytes,
+            PrimVal::Unknown { type_code, .. } => ValueType::Unknown(*type_code),
+        }
+    }
+}
+
+impl From<ValueType> for u64 {
+    fn from(v: ValueType) -> Self {
+        match v {
+            ValueType::Null => 0,
+            ValueType::False => 1,
+            ValueType::True => 2,
+            ValueType::Uleb => 3,
+            ValueType::Leb => 4,
+            ValueType::Float => 5,
+            ValueType::String => 6,
+            ValueType::Bytes => 7,
+            ValueType::Counter => 8,
+            ValueType::Timestamp => 9,
+            ValueType::Unknown(other) => other as u64,
+        }
+    }
+}
+
+fn lebsize(val: i64) -> u64 {
+    if val == 0 {
+        return 1;
+    }
+    let numbits = (val as f64).abs().log2().ceil() as u64;
+    let mut numblocks = (numbits as f64 / 7.0).ceil() as u64;
+    // Make room for the sign bit
+    if numbits % 7 == 0 {
+        numblocks += 1;
+    }
+    return numblocks;
+}
+
+fn ulebsize(val: u64) -> u64 {
+    if val == 0 {
+        return 1;
+    }
+    let numbits = (val as f64).log2().ceil() as u64;
+    let numblocks = (numbits as f64 / 7.0).ceil() as u64;
+    return numblocks;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::columnar_2::rowblock::encoding::{
+        properties::{splice_scenario, value}, RawDecoder, RleDecoder,
+    };
+    use proptest::prelude::*;
+
+    fn encode_values(vals: &[PrimVal]) -> (Range<usize>, Range<usize>, Vec<u8>) {
+        let mut decoder = ValueDecoder {
+            meta: RleDecoder::from(&[] as &[u8]),
+            raw: RawDecoder::from(&[] as &[u8]),
+        };
+        let mut out = Vec::new();
+        let (meta_range, val_range) = decoder
+            .try_splice(0..0, vals.iter().map(|v| Ok(v.clone())), &mut out)
+            .unwrap();
+        (meta_range, val_range, out)
+    }
+
+    proptest! {
+        #[test]
+        fn test_initialize_splice(values in proptest::collection::vec(value(), 0..100)) {
+            let (meta_range, val_range, out) = encode_values(&values);
+            let mut decoder = ValueDecoder{
+                meta: RleDecoder::from(&out[meta_range]),
+                raw: RawDecoder::from(&out[val_range]),
+            };
+            let mut testvals = Vec::new();
+            while !decoder.done() {
+                testvals.push(decoder.next().unwrap().unwrap());
+            }
+            assert_eq!(values, testvals);
+        }
+
+        #[test]
+        fn test_splice_values(scenario in splice_scenario(value())){
+            let (meta_range, val_range, out) = encode_values(&scenario.initial_values);
+            let mut decoder = ValueDecoder{
+                meta: RleDecoder::from(&out[meta_range]),
+                raw: RawDecoder::from(&out[val_range]),
+            };
+            let mut spliced = Vec::new();
+            let (spliced_meta, spliced_val) = decoder
+                .try_splice(
+                    scenario.replace_range.clone(),
+                    scenario.replacements.clone().into_iter().map(|i| Ok(i)),
+                    &mut spliced,
+                ).unwrap();
+            let mut spliced_decoder = ValueDecoder{
+                meta: RleDecoder::from(&spliced[spliced_meta]),
+                raw: RawDecoder::from(&spliced[spliced_val]),
+            };
+            let mut result_values = Vec::new();
+            while !spliced_decoder.done() {
+                result_values.push(spliced_decoder.next().unwrap().unwrap());
+            }
+            let mut expected: Vec<_> = scenario.initial_values.clone();
+            expected.splice(scenario.replace_range, scenario.replacements);
+            assert_eq!(result_values, expected);
+        }
+    }
+}

automerge/src/columnar_2/rowblock/mod.rs

@@ -0,0 +1,143 @@
+use std::{borrow::Cow, convert::TryInto};
+
+use self::column_layout::DocOpColumns;
+
+use super::{ColumnId, ColumnSpec};
+
+mod column_layout;
+pub(crate) use column_layout::doc_change_columns;
+pub(crate) use column_layout::doc_op_columns;
+pub(crate) use column_layout::change_op_columns;
+pub(crate) use column_layout::{BadColumnLayout, ColumnLayout};
+mod column_range;
+mod encoding;
+pub(crate) use encoding::Key;
+use encoding::{DecodeColumnError, GenericColDecoder};
+mod value;
+pub(crate) use value::{CellValue, PrimVal};
+
+pub(crate) struct RowBlock<'a, C> {
+    columns: C,
+    data: Cow<'a, [u8]>,
+}
+
+impl<'a> RowBlock<'a, ColumnLayout> {
+    pub(crate) fn new<I: Iterator<Item = (ColumnSpec, std::ops::Range<usize>)>>(
+        cols: I,
+        data: Cow<'a, [u8]>,
+    ) -> Result<RowBlock<'a, ColumnLayout>, BadColumnLayout> {
+        let layout = ColumnLayout::parse(data.len(), cols)?;
+        Ok(RowBlock {
+            columns: layout,
+            data,
+        })
+    }
+
+    pub(crate) fn into_doc_ops(
+        self,
+    ) -> Result<RowBlock<'a, column_layout::DocOpColumns>, column_layout::ParseDocColumnError> {
+        let doc_cols: column_layout::DocOpColumns = self.columns.try_into()?;
+        Ok(RowBlock {
+            columns: doc_cols,
+            data: self.data,
+        })
+    }
+
+    pub(crate) fn into_doc_change(
+        self,
+    ) -> Result<
+        RowBlock<'a, column_layout::doc_change_columns::DocChangeColumns>,
+        column_layout::doc_change_columns::DecodeChangeError,
+    > {
+        let doc_cols: column_layout::doc_change_columns::DocChangeColumns =
+            self.columns.try_into()?;
+        Ok(RowBlock {
+            columns: doc_cols,
+            data: self.data,
+        })
+    }
+
+    pub(crate) fn into_change_ops(
+        self
+    ) -> Result<RowBlock<'a, change_op_columns::ChangeOpsColumns>, change_op_columns::ParseChangeColumnsError> {
+        let change_cols: change_op_columns::ChangeOpsColumns = self.columns.try_into()?;
+        Ok(RowBlock {
+            columns: change_cols,
+            data: self.data,
+        })
+    }
+}
+
+impl<'a, 'b> IntoIterator for &'a RowBlock<'b, ColumnLayout> {
+    type Item = Result<Vec<(usize, CellValue<'a>)>, DecodeColumnError>;
+    type IntoIter = RowBlockIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        RowBlockIter {
+            failed: false,
+            decoders: self
+                .columns
+                .iter()
+                .map(|c| (c.id(), c.decoder(&self.data)))
+                .collect(),
+        }
+    }
+}
+
+pub(crate) struct RowBlockIter<'a> {
+    failed: bool,
+    decoders: Vec<(ColumnId, GenericColDecoder<'a>)>,
+}
+
+impl<'a> Iterator for RowBlockIter<'a> {
+    type Item = Result<Vec<(usize, CellValue<'a>)>, DecodeColumnError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.failed {
+            return None;
+        }
+        if self.decoders.iter().all(|(_, d)| d.done()) {
+            None
+        } else {
+            let mut result = Vec::with_capacity(self.decoders.len());
+            for (col_index, (_, decoder)) in self.decoders.iter_mut().enumerate() {
+                match decoder.next() {
+                    Some(Ok(c)) => result.push((col_index, c)),
+                    Some(Err(e)) => {
+                        self.failed = true;
+                        return Some(Err(e));
+                    },
+                    None => {},
+                }
+            }
+            Some(Ok(result))
+        }
+    }
+}
+
+impl<'a> IntoIterator for &'a RowBlock<'a, DocOpColumns> {
+    type Item = Result<doc_op_columns::DocOp<'a>, column_layout::doc_op_columns::DecodeOpError>;
+    type IntoIter = column_layout::doc_op_columns::DocOpColumnIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.columns.iter(&self.data)
+    }
+}
+
+impl<'a> IntoIterator for &'a RowBlock<'a, doc_change_columns::DocChangeColumns> {
+    type Item = Result<doc_change_columns::ChangeMetadata<'a>, doc_change_columns::DecodeChangeError>;
+    type IntoIter = doc_change_columns::DocChangeColumnIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.columns.iter(&self.data)
+    }
+}
+
+impl<'a> IntoIterator for &'a RowBlock<'a, change_op_columns::ChangeOpsColumns> {
+    type Item = Result<change_op_columns::ChangeOp<'a>, change_op_columns::ReadChangeOpError>;
+    type IntoIter = change_op_columns::ChangeOpsIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.columns.iter(&self.data)
+    }
+}

automerge/src/columnar_2/rowblock/value.rs

@@ -0,0 +1,105 @@
+use crate::ScalarValue;
+
+use std::borrow::Cow;
+
+use smol_str::SmolStr;
+
+#[derive(Debug)]
+pub(crate) enum CellValue<'a> {
+    Uint(u64),
+    Int(i64),
+    Bool(bool),
+    String(Cow<'a, SmolStr>),
+    Value(PrimVal<'a>),
+    List(Vec<Vec<CellValue<'a>>>),
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) enum PrimVal<'a> {
+    Null,
+    Bool(bool),
+    Uint(u64),
+    Int(i64),
+    Float(f64),
+    String(Cow<'a, SmolStr>),
+    Bytes(Cow<'a, [u8]>),
+    Counter(u64),
+    Timestamp(u64),
+    Unknown { type_code: u8, data: Vec<u8> },
+}
+
+impl<'a> PrimVal<'a> {
+    pub(crate) fn into_owned(self) -> PrimVal<'static> {
+        match self {
+            PrimVal::String(s) => PrimVal::String(Cow::Owned(s.into_owned().into())),
+            PrimVal::Bytes(b) => PrimVal::Bytes(Cow::Owned(b.to_vec())),
+            PrimVal::Null => PrimVal::Null,
+            PrimVal::Bool(b) => PrimVal::Bool(b),
+            PrimVal::Uint(u) => PrimVal::Uint(u),
+            PrimVal::Int(i) => PrimVal::Int(i),
+            PrimVal::Float(f) => PrimVal::Float(f),
+            PrimVal::Counter(u) => PrimVal::Counter(u),
+            PrimVal::Timestamp(u) => PrimVal::Timestamp(u),
+            PrimVal::Unknown { type_code, data } => PrimVal::Unknown{ type_code, data},
+        }
+    }
+}
+
+impl<'a> From<PrimVal<'a>> for ScalarValue {
+    fn from(p: PrimVal) -> Self {
+        match p {
+            PrimVal::Null => Self::Null,
+            PrimVal::Bool(b) => Self::Boolean(b),
+            PrimVal::Uint(u) => Self::Uint(u),
+            PrimVal::Int(i) => Self::Int(i),
+            PrimVal::Float(f) => Self::F64(f),
+            PrimVal::String(s) => Self::Str(s.into_owned()),
+            PrimVal::Bytes(b) => Self::Bytes(b.to_vec()),
+            PrimVal::Counter(c) => Self::Counter((c as i64).into()),
+            PrimVal::Timestamp(t) => Self::Timestamp(t as i64),
+            PrimVal::Unknown { data, .. } => Self::Bytes(data),
+        }
+    }
+}
+
+impl<'a> From<ScalarValue> for PrimVal<'static> {
+    fn from(s: ScalarValue) -> Self {
+        match s {
+            ScalarValue::Null => PrimVal::Null,
+            ScalarValue::Boolean(b) => PrimVal::Bool(b),
+            ScalarValue::Uint(u) => PrimVal::Uint(u),
+            ScalarValue::Int(i) => PrimVal::Int(i),
+            ScalarValue::F64(f) => PrimVal::Float(f),
+            ScalarValue::Str(s) => PrimVal::String(Cow::Owned(s)),
+            // This is bad, if there was an unknown type code in the primval we have lost it on the
+            // round trip
+            ScalarValue::Bytes(b) => PrimVal::Bytes(Cow::Owned(b)),
+            ScalarValue::Counter(c) => PrimVal::Counter(c.current as u64),
+            ScalarValue::Timestamp(t) => PrimVal::Timestamp(t as u64),
+        }
+    }
+}
+
+impl<'a> From<&ScalarValue> for PrimVal<'static> {
+    fn from(s: &ScalarValue) -> Self {
+        match s {
+            ScalarValue::Null => PrimVal::Null,
+            ScalarValue::Boolean(b) => PrimVal::Bool(*b),
+            ScalarValue::Uint(u) => PrimVal::Uint(*u),
+            ScalarValue::Int(i) => PrimVal::Int(*i),
+            ScalarValue::F64(f) => PrimVal::Float(*f),
+            ScalarValue::Str(s) => PrimVal::String(Cow::Owned(s.clone())),
+            // This is bad, if there was an unknown type code in the primval we have lost it on the
+            // round trip
+            ScalarValue::Bytes(b) => PrimVal::Bytes(Cow::Owned(b.clone())),
+            ScalarValue::Counter(c) => PrimVal::Counter(c.current as u64),
+            ScalarValue::Timestamp(t) => PrimVal::Timestamp((*t) as u64),
+        }
+    }
+}
+
+impl<'a> From<&'a [u8]> for PrimVal<'a> {
+    fn from(d: &'a [u8]) -> Self {
+        PrimVal::Bytes(Cow::Borrowed(d))
+    }
+}

automerge/src/columnar_2/save.rs

@@ -0,0 +1,256 @@
+use std::{
+    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
+    iter::Iterator,
+};
+
+use itertools::Itertools;
+
+use crate::{
+    columnar_2::{
+        rowblock::{
+            change_op_columns::{ChangeOp, ChangeOpsColumns},
+            doc_change_columns::{ChangeMetadata, DocChangeColumns},
+            doc_op_columns::{DocOp, DocOpColumns},
+            Key as EncodedKey, PrimVal,
+        },
+        storage::{Chunk, Document},
+    },
+    indexed_cache::IndexedCache,
+    types::{ActorId, ElemId, Key, ObjId, Op, OpId, OpType},
+    Change, ChangeHash,
+};
+
+/// # Panics
+///
+/// * If any of the `heads` are not in `changes`
+/// * If any of ops in `ops` reference an actor which is not in `actors`
+/// * If any of ops in `ops` reference a property which is not in `props`
+/// * If any of the changes reference a dependency index which is not in `changes`
+pub(crate) fn save_document<'a, I, O>(
+    changes: I,
+    ops: O,
+    actors: &'a IndexedCache<ActorId>,
+    props: &IndexedCache<String>,
+    heads: &[ChangeHash],
+) -> Vec<u8>
+where
+    I: Iterator<Item = &'a Change> + Clone + 'a,
+    O: Iterator<Item = (&'a ObjId, &'a Op)> + Clone,
+{
+    let actor_lookup = actors.encode_index();
+    let doc_ops = ops.map(|(obj, op)| DocOp {
+        id: translate_opid(&op.id, &actor_lookup),
+        insert: op.insert,
+        object: translate_objid(obj, &actor_lookup),
+        key: translate_key(&op.key, props),
+        action: op.action.action_index() as usize,
+        value: match &op.action {
+            OpType::Set(v) => v.into(),
+            OpType::Inc(i) => PrimVal::Int(*i),
+            _ => PrimVal::Null,
+        },
+        succ: op
+            .succ
+            .iter()
+            .map(|o| translate_opid(o, &actor_lookup))
+            .collect(),
+    });
+    let mut ops_out = Vec::new();
+    let ops_meta = DocOpColumns::encode(doc_ops, &mut ops_out);
+
+    let mut change_out = Vec::new();
+    let hash_graph = HashGraph::new(changes.clone(), heads);
+    let cols = DocChangeColumns::encode(
+        changes.map(|c| hash_graph.construct_change(c, &actor_lookup, actors)),
+        &mut change_out,
+    );
+
+    let doc = Document {
+        actors: actors.sorted().cache,
+        heads: heads.to_vec(),
+        op_metadata: ops_meta.metadata(),
+        op_bytes: Cow::Owned(ops_out),
+        change_metadata: cols.metadata(),
+        change_bytes: Cow::Owned(change_out),
+        head_indices: hash_graph.head_indices,
+    };
+
+    let written = doc.write();
+    let chunk = Chunk::new_document(&written);
+    chunk.write()
+}
+
+pub(crate) fn encode_change_ops<'a, O>(
+    ops: O,
+    change_actor: ActorId,
+    actors: &IndexedCache<ActorId>,
+    props: &IndexedCache<String>,
+) -> (ChangeOpsColumns, Vec<u8>, Vec<ActorId>)
+where
+    O: Iterator<Item = (&'a ObjId, &'a Op)> + Clone,
+{
+    let encoded_actors = actor_ids_in_change(ops.clone(), change_actor.clone(), actors);
+    let actor_lookup = actors
+        .cache
+        .iter()
+        .map(|a| encoded_actors.iter().position(|r| r == a).unwrap())
+        .collect::<Vec<_>>();
+    let change_ops = ops.map(|(obj, op)| ChangeOp {
+        insert: op.insert,
+        obj: translate_objid(obj, &actor_lookup),
+        key: translate_key(&op.key, props),
+        action: op.action.action_index(),
+        val: match &op.action {
+            OpType::Set(v) => v.into(),
+            OpType::Inc(i) => PrimVal::Int(*i),
+            _ => PrimVal::Null,
+        },
+        pred: op
+            .pred
+            .iter()
+            .map(|o| translate_opid(o, &actor_lookup))
+            .collect(),
+    });
+    let mut out = Vec::new();
+    let cols = ChangeOpsColumns::empty().encode(change_ops, &mut out);
+    let other_actors = encoded_actors.into_iter().skip(1).collect();
+    (cols, out, other_actors)
+}
+
+/// When encoding a change chunk we take all the actor IDs referenced by a change and place them in
+/// an array. The array has the actor who authored the change as the first element and all
+/// remaining actors (i.e. those referenced in object IDs in the target of an operation or in the
+/// `pred` of an operation) lexicographically ordered following the change author.
+fn actor_ids_in_change<'a, I>(
+    ops: I,
+    change_actor: ActorId,
+    actors: &IndexedCache<ActorId>,
+) -> Vec<ActorId>
+where
+    I: Iterator<Item = (&'a ObjId, &'a Op)> + Clone,
+{
+    let mut other_ids: Vec<ActorId> = ops
+        .flat_map(|(obj, o)| opids_in_operation(&obj, &o, actors))
+        .filter(|a| *a != &change_actor)
+        .unique()
+        .cloned()
+        .collect();
+    other_ids.sort();
+    // Now prepend the change actor
+    std::iter::once(change_actor)
+        .chain(other_ids.into_iter())
+        .collect()
+}
+
+fn opids_in_operation<'a>(
+    obj: &'a ObjId,
+    op: &'a Op,
+    actors: &'a IndexedCache<ActorId>,
+) -> impl Iterator<Item = &'a ActorId> {
+    let obj_actor_id = if obj.is_root() {
+        None
+    } else {
+        Some(actors.get(obj.opid().actor()))
+    };
+    let pred_ids = op.pred.iter().filter_map(|a| {
+        if a.counter() != 0 {
+            Some(actors.get(a.actor()))
+        } else {
+            None
+        }
+    });
+    let key_actor = match &op.key {
+        Key::Seq(ElemId(op)) if !op.counter() == 0 => Some(actors.get(op.actor())),
+        _ => None,
+    };
+    obj_actor_id
+        .into_iter()
+        .chain(key_actor.into_iter())
+        .chain(pred_ids)
+}
+
+fn translate_key(k: &Key, props: &IndexedCache<String>) -> EncodedKey {
+    match k {
+        Key::Seq(e) => EncodedKey::Elem(*e),
+        Key::Map(idx) => EncodedKey::Prop(props.get(*idx).into()),
+    }
+}
+
+fn translate_objid(obj: &ObjId, actors: &[usize]) -> ObjId {
+    if obj.is_root() {
+        *obj
+    } else {
+        ObjId(translate_opid(&obj.opid(), actors))
+    }
+}
+
+fn translate_opid(id: &OpId, actors: &[usize]) -> OpId {
+    OpId::new(actors[id.actor()], id.counter())
+}
+
+fn find_head_indices<'a, I>(changes: I, heads: &[ChangeHash]) -> Vec<u64>
+where
+    I: Iterator<Item = &'a Change>,
+{
+    let heads_set: BTreeSet<ChangeHash> = heads.iter().copied().collect();
+    let mut head_indices = BTreeMap::new();
+    for (index, change) in changes.enumerate() {
+        if heads_set.contains(&change.hash()) {
+            head_indices.insert(change.hash(), index as u64);
+        }
+    }
+    heads.iter().map(|h| head_indices[h]).collect()
+}
+
+struct HashGraph {
+    head_indices: Vec<u64>,
+    index_by_hash: BTreeMap<ChangeHash, usize>,
+}
+
+impl HashGraph {
+    fn new<'a, I>(changes: I, heads: &[ChangeHash]) -> Self
+    where
+        I: Iterator<Item = &'a Change>,
+    {
+        let heads_set: BTreeSet<ChangeHash> = heads.iter().copied().collect();
+        let mut head_indices = BTreeMap::new();
+        let mut index_by_hash = BTreeMap::new();
+        for (index, change) in changes.enumerate() {
+            if heads_set.contains(&change.hash()) {
+                head_indices.insert(change.hash(), index as u64);
+            }
+            index_by_hash.insert(change.hash(), index);
+        }
+        let head_indices = heads.iter().map(|h| head_indices[h]).collect();
+        Self {
+            head_indices,
+            index_by_hash,
+        }
+    }
+
+    fn change_index(&self, hash: &ChangeHash) -> usize {
+        self.index_by_hash[hash]
+    }
+
+    fn construct_change(
+        &self,
+        c: &Change,
+        actor_lookup: &[usize],
+        actors: &IndexedCache<ActorId>,
+    ) -> ChangeMetadata<'static> {
+        ChangeMetadata {
+            actor: actor_lookup[actors.lookup(c.actor_id()).unwrap()],
+            seq: c.seq(),
+            max_op: c.max_op(),
+            timestamp: c.timestamp(),
+            message: c.message().map(|s| s.into()),
+            deps: c
+                .deps()
+                .iter()
+                .map(|d| self.change_index(d) as u64)
+                .collect(),
+            extra: Cow::Owned(c.extra_bytes().to_vec()),
+        }
+    }
+}

automerge/src/columnar_2/storage/change.rs

@@ -0,0 +1,114 @@
+use std::{borrow::Cow, io::Write};
+
+use crate::{ActorId, ChangeHash};
+
+use super::{parse, ColumnMetadata, Chunk};
+
+#[derive(Clone, Debug)]
+pub(crate) struct Change<'a> {
+    pub(crate) dependencies: Vec<ChangeHash>,
+    pub(crate) actor: ActorId,
+    pub(crate) other_actors: Vec<ActorId>,
+    pub(crate) seq: u64,
+    pub(crate) start_op: u64,
+    pub(crate) timestamp: i64,
+    pub(crate) message: Option<String>,
+    pub(crate) ops_meta: ColumnMetadata,
+    pub(crate) ops_data: Cow<'a, [u8]>,
+    pub(crate) extra_bytes: Cow<'a, [u8]>,
+}
+
+impl<'a> Change<'a> {
+    pub(crate) fn parse(input: &'a [u8]) -> parse::ParseResult<Change<'a>> {
+        let (i, deps) = parse::length_prefixed(parse::leb128_u64, parse::change_hash)(input)?;
+        let (i, actor) = parse::actor_id(i)?;
+        let (i, seq) = parse::leb128_u64(i)?;
+        let (i, start_op) = parse::leb128_u64(i)?;
+        let (i, timestamp) = parse::leb128_i64(i)?;
+        let (i, message_len) = parse::leb128_u64(i)?;
+        let (i, message) = parse::utf_8(message_len as usize, i)?;
+        let (i, other_actors) = parse::length_prefixed(parse::leb128_u64, parse::actor_id)(i)?;
+        let (i, ops_meta) = ColumnMetadata::parse(i)?;
+        let (i, ops_data) = parse::take_n(ops_meta.total_column_len(), i)?;
+        Ok((
+            &[],
+            Change {
+                dependencies: deps,
+                actor,
+                other_actors,
+                seq,
+                start_op,
+                timestamp,
+                message: if message.is_empty() {
+                    None
+                } else {
+                    Some(message)
+                },
+                ops_meta,
+                ops_data: Cow::Borrowed(ops_data),
+                extra_bytes: Cow::Borrowed(i),
+            },
+        ))
+    }
+
+    fn byte_len(&self) -> usize {
+        (self.dependencies.len() * 32)
+            + 8
+            + self.actor.to_bytes().len()
+            + 24 // seq, start op, timestamp
+            + 8
+            + self.message.as_ref().map(|m| m.as_bytes().len()).unwrap_or(0_usize)
+            + self.other_actors.iter().map(|a| a.to_bytes().len() + 8_usize).sum::<usize>()
+            + self.ops_meta.byte_len()
+            + self.ops_data.len()
+            + self.extra_bytes.len()
+    }
+
+    pub(crate) fn write(&self) -> Vec<u8> {
+        let mut out = Vec::with_capacity(self.byte_len());
+        leb128::write::unsigned(&mut out, self.dependencies.len() as u64).unwrap();
+        for dep in &self.dependencies {
+            out.write_all(dep.as_bytes()).unwrap();
+        }
+        length_prefixed_bytes(&self.actor, &mut out);
+        leb128::write::unsigned(&mut out, self.seq).unwrap();
+        leb128::write::unsigned(&mut out, self.start_op).unwrap();
+        leb128::write::signed(&mut out, self.timestamp).unwrap();
+        length_prefixed_bytes(self.message.as_ref().map(|m| m.as_bytes()).unwrap_or(&[]), &mut out);
+        leb128::write::unsigned(&mut out, self.other_actors.len() as u64).unwrap();
+        for actor in self.other_actors.iter() {
+            length_prefixed_bytes(&actor, &mut out);
+        }
+        self.ops_meta.write(&mut out);
+        out.write_all(self.ops_data.as_ref()).unwrap();
+        out.write_all(self.extra_bytes.as_ref()).unwrap();
+        out
+    }
+
+    pub(crate) fn hash(&self) -> ChangeHash {
+        let this = self.write();
+        let chunk = Chunk::new_change(&this);
+        chunk.hash()
+    }
+
+    pub(crate) fn into_owned(self) -> Change<'static> {
+        Change{
+            dependencies: self.dependencies,
+            actor: self.actor,
+            other_actors: self.other_actors,
+            seq: self.seq,
+            start_op: self.start_op,
+            timestamp: self.timestamp,
+            message: self.message,
+            ops_meta: self.ops_meta,
+            ops_data: Cow::Owned(self.ops_data.into_owned()),
+            extra_bytes: Cow::Owned(self.extra_bytes.into_owned()),
+        }
+    }
+}
+
+fn length_prefixed_bytes<B: AsRef<[u8]>>(b: B, out: &mut Vec<u8>) -> usize {
+    let prefix_len = leb128::write::unsigned(out, b.as_ref().len() as u64).unwrap();
+    out.write_all(b.as_ref()).unwrap();
+    prefix_len + b.as_ref().len()
+}

automerge/src/columnar_2/storage/chunk.rs

@@ -0,0 +1,163 @@
+use std::{borrow::Cow, convert::{TryFrom, TryInto}};
+
+use sha2::{Digest, Sha256};
+
+use crate::ChangeHash;
+use super::parse;
+
+
+const MAGIC_BYTES: [u8; 4] = [0x85, 0x6f, 0x4a, 0x83];
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum ChunkType {
+    Document,
+    Change,
+    Compressed,
+}
+
+impl TryFrom<u8> for ChunkType {
+    type Error = u8;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            0 => Ok(Self::Document),
+            1 => Ok(Self::Change),
+            2 => Ok(Self::Compressed),
+            other => Err(other),
+        }
+    }
+}
+
+impl From<ChunkType> for u8 {
+    fn from(ct: ChunkType) -> Self {
+        match ct {
+            ChunkType::Document => 0,
+            ChunkType::Change => 1,
+            ChunkType::Compressed => 2,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct CheckSum([u8; 4]);
+
+impl CheckSum {
+    fn bytes(&self) -> [u8; 4] {
+        self.0
+    }
+}
+
+impl From<[u8; 4]> for CheckSum {
+    fn from(raw: [u8; 4]) -> Self {
+        CheckSum(raw)
+    }
+}
+
+impl From<ChangeHash> for CheckSum {
+    fn from(h: ChangeHash) -> Self {
+        let bytes = h.as_bytes();
+        [bytes[0], bytes[1], bytes[2], bytes[3]].into()
+    }
+}
+
+
+#[derive(Debug)]
+pub(crate) struct Chunk<'a> {
+    typ: ChunkType,
+    checksum: CheckSum,
+    data: Cow<'a, [u8]>,
+}
+
+impl<'a> Chunk<'a> {
+    pub(crate) fn new_change(data: &'a [u8]) -> Chunk<'a> {
+        let hash_result = hash(ChunkType::Change, data);
+        Chunk{
+            typ: ChunkType::Change,
+            checksum: hash_result.into(),
+            data: Cow::Borrowed(data),
+        }
+    }
+
+    pub(crate) fn new_document(data: &'a [u8]) -> Chunk<'a> {
+        let hash_result = hash(ChunkType::Document, data);
+        Chunk{
+            typ: ChunkType::Document,
+            checksum: hash_result.into(),
+            data: Cow::Borrowed(data),
+        }
+    }
+
+    pub(crate) fn parse(input: &'a [u8]) -> parse::ParseResult<Chunk<'a>> {
+        let (i, magic) = parse::take4(input)?;
+        if magic != MAGIC_BYTES {
+            return Err(parse::ParseError::Error(
+                parse::ErrorKind::InvalidMagicBytes,
+            ));
+        }
+        let (i, checksum_bytes) = parse::take4(i)?;
+        let (i, raw_chunk_type) = parse::take1(i)?;
+        let chunk_type: ChunkType = raw_chunk_type
+            .try_into()
+            .map_err(|e| parse::ParseError::Error(parse::ErrorKind::UnknownChunkType(e)))?;
+        let (i, chunk_len) = parse::leb128_u64(i)?;
+        let (i, data) = parse::take_n(chunk_len as usize, i)?;
+        Ok((
+            i,
+            Chunk {
+                typ: chunk_type,
+                checksum: checksum_bytes.into(),
+                data: Cow::Borrowed(data),
+            },
+        ))
+    }
+
+    fn byte_len(&self) -> usize {
+        MAGIC_BYTES.len()
+        + 1   // chunk type
+        + 4 // checksum
+        + 5  //length
+        + self.data.len()
+    }
+
+    pub(crate) fn write(&self) -> Vec<u8> {
+        let mut out = Vec::with_capacity(self.byte_len());
+        out.extend(MAGIC_BYTES);
+        out.extend(self.checksum.bytes());
+        out.push(u8::from(self.typ));
+        leb128::write::unsigned(&mut out, self.data.len() as u64).unwrap();
+        out.extend(self.data.as_ref());
+        out
+    }
+
+    pub(crate) fn checksum_valid(&self) -> bool {
+        let hash = self.hash();
+        let checksum = CheckSum(hash.checksum());
+        checksum == self.checksum
+    }
+
+    pub(crate) fn hash(&self) -> ChangeHash {
+        hash(self.typ, self.data.as_ref())
+    }
+
+    pub(crate) fn typ(&self) -> ChunkType {
+        self.typ
+    }
+
+    pub(crate) fn checksum(&self) -> CheckSum {
+        self.checksum
+    }
+
+    pub(crate) fn data(&self) -> Cow<'a, [u8]> {
+        self.data.clone()
+    }
+}
+
+fn hash(typ: ChunkType, data: &[u8]) -> ChangeHash {
+    let mut out = Vec::new();
+    out.push(u8::from(typ));
+    leb128::write::unsigned(&mut out, data.len() as u64).unwrap();
+    out.extend(data.as_ref());
+    let hash_result = Sha256::digest(out);
+    let array: [u8; 32] = hash_result.into();
+    ChangeHash(array)
+}

automerge/src/columnar_2/storage/column_metadata.rs

@@ -0,0 +1,85 @@
+use std::ops::Range;
+
+use super::{super::ColumnSpec, parse};
+
+#[derive(Clone, Debug)]
+pub(crate) struct Column {
+    spec: ColumnSpec,
+    data: Range<usize>,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct ColumnMetadata(Vec<Column>);
+
+impl FromIterator<Column> for ColumnMetadata {
+    fn from_iter<T: IntoIterator<Item = Column>>(iter: T) -> Self {
+        Self(iter.into_iter().collect())
+    }
+}
+
+impl FromIterator<(ColumnSpec, Range<usize>)> for ColumnMetadata {
+    fn from_iter<T: IntoIterator<Item = (ColumnSpec, Range<usize>)>>(iter: T) -> Self {
+        Self(iter.into_iter().map(|(spec, data)| Column{spec, data}).collect())
+    }
+}
+
+impl ColumnMetadata {
+    pub(crate) fn parse(input: &[u8]) -> parse::ParseResult<ColumnMetadata> {
+        let i = input;
+        let (i, num_columns) = parse::leb128_u64(i)?;
+        let (i, specs_and_lens) = parse::apply_n(
+            num_columns as usize,
+            parse::tuple2(
+                parse::map(parse::leb128_u32, ColumnSpec::from),
+                parse::leb128_u64,
+            ),
+        )(i)?;
+        let columns = specs_and_lens
+            .into_iter()
+            .scan(0_usize, |offset, (spec, len)| {
+                let end = *offset + len as usize;
+                let data = *offset..end;
+                *offset = end;
+                Some(Column { spec, data })
+            })
+            .collect::<Vec<_>>();
+        if !are_normal_sorted(&columns) {
+            return Err(parse::ParseError::Error(
+                parse::ErrorKind::InvalidColumnMetadataSort,
+            ));
+        }
+        Ok((i, ColumnMetadata(columns)))
+    }
+
+    pub(crate) fn write(&self, out: &mut Vec<u8>) -> usize {
+        let mut written = leb128::write::unsigned(out, self.0.len() as u64).unwrap();
+        for col in &self.0 {
+            written += leb128::write::unsigned(out, u32::from(col.spec) as u64).unwrap();
+            written += leb128::write::unsigned(out, col.data.len() as u64).unwrap();
+        }
+        written
+    }
+
+    pub(crate) fn total_column_len(&self) -> usize {
+        self.0.iter().map(|c| c.data.len()).sum()
+    }
+
+    pub(crate) fn iter(&self) -> impl Iterator<Item = (ColumnSpec, Range<usize>)> + '_ {
+        self.0.iter().map(|c| (c.spec, c.data.clone()))
+    }
+
+    pub(crate) fn byte_len(&self) -> usize {
+        self.0.len() * 16
+    }
+}
+
+fn are_normal_sorted(cols: &[Column]) -> bool {
+    if cols.len() > 1 {
+        for (i, col) in cols[1..].iter().enumerate() {
+            if col.spec.normalize() < cols[i].spec.normalize() {
+                return false;
+            }
+        }
+    }
+    true
+}