Merge better-js-package

Cargo.toml

@@ -11,7 +11,11 @@ resolver = "2"
 [profile.release]
 debug = true
 lto = true
-opt-level = 3
+opt-level = 'z'
 
 [profile.bench]
 debug = true
+
+[profile.release.package.automerge-wasm]
+debug = false
+opt-level = 'z'

automerge-c/src/doc.rs

@@ -170,7 +170,7 @@ pub unsafe extern "C" fn AMcommit(
     if let Some(time) = time.as_ref() {
         options.set_time(*time);
     }
-    to_result(doc.commit_with::<()>(options))
+    to_result(doc.commit_with(options))
 }
 
 /// \memberof AMdoc

automerge-js/e2e/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+verdacciodb/
+htpasswd

automerge-js/e2e/README.md

@@ -0,0 +1,71 @@
+#End to end testing for javascript packaging
+
+The network of packages and bundlers we rely on to get the `automerge` package
+working is a little complex. We have the `automerge-wasm` package, which the
+`automerge` package depends upon, which means that anyone who depends on
+`automerge` needs to either a) be using node or b) use a bundler in order to
+load the underlying WASM module which is packaged in `automerge-wasm`.
+
+The various bundlers involved are complicated and capricious and so we need an
+easy way of testing that everything is in fact working as expected. To do this
+we run a custom NPM registry (namely [Verdaccio](https://verdaccio.org/)) and
+build the `automerge-wasm` and `automerge` packages and publish them to this
+registry. Once we have this registry running we are able to build the example
+projects which depend on these packages and check that everything works as
+expected.
+
+## Usage
+
+First, install everything:
+
+```
+yarn install
+```
+
+### Build `automerge-js`
+
+This builds the `automerge-wasm` package and then runs `yarn build` in the
+`automerge-js` project with the `--registry` set to the verdaccio registry. The
+end result is that you can run `yarn test` in the resulting `automerge-js`
+directory in order to run tests against the current `automerge-wasm`.
+
+```
+yarn e2e buildjs
+```
+
+### Build examples
+
+This either builds or the examples in `automerge-js/examples` or just a subset
+of them. Once this is complete you can run the relevant scripts (e.g. `vite dev`
+for the Vite example) to check everything works.
+
+```
+yarn e2e buildexamples
+```
+
+Or, to just build the webpack example
+
+```
+yarn e2e buildexamples -e webpack
+```
+
+### Run Registry
+
+If you're experimenting with a project which is not in the `examples` folder
+you'll need a running registry. `run-registry` builds and publishes
+`automerge-js` and `automerge-wasm` and then runs the registry at
+`localhost:4873`. 
+
+```
+yarn e2e run-registry
+```
+
+You can now run `yarn install --registry http://localhost:4873` to experiment
+with the built packages.
+
+
+## Using the `dev` build of `automerge-wasm`
+
+All the commands above take a `-p` flag which can be either `release` or
+`debug`. The `debug` builds with additional debug symbols which makes errors
+less cryptic.

automerge-js/e2e/index.ts

@@ -0,0 +1,409 @@
+import {once} from "events"
+import {setTimeout} from "timers/promises"
+import {spawn, ChildProcess} from "child_process"
+import * as child_process from "child_process"
+import {command, subcommands, run, array, multioption, option, Type} from "cmd-ts"
+import * as path from "path"
+import * as fsPromises from "fs/promises"
+import fetch from "node-fetch"
+
+const VERDACCIO_DB_PATH = path.normalize(`${__dirname}/verdacciodb`)
+const VERDACCIO_CONFIG_PATH = path.normalize(`${__dirname}/verdaccio.yaml`)
+const AUTOMERGE_WASM_PATH = path.normalize(`${__dirname}/../../automerge-wasm`)
+const AUTOMERGE_JS_PATH = path.normalize(`${__dirname}/..`)
+const EXAMPLES_DIR = path.normalize(path.join(__dirname, "../", "examples"))
+
+// The different example projects in "../examples"
+type Example = "webpack" | "vite"
+
+// Type to parse strings to `Example` so the types line up for the `buildExamples` commmand
+const ReadExample: Type<string, Example> = {
+    async from(str) {
+        if (str === "webpack") {
+            return "webpack"
+        } else if (str === "vite") {
+            return "vite"
+        } else {
+            throw new Error(`Unknown example type ${str}`)
+        }
+    }
+}
+
+type Profile = "dev" | "release"
+
+const ReadProfile: Type<string, Profile> = {
+    async from(str) {
+        if (str === "dev") {
+            return "dev"
+        } else if (str === "release") {
+            return "release"
+        } else {
+            throw new Error(`Unknown profile ${str}`)
+        }
+    }
+}
+
+const buildjs = command({
+    name: "buildjs",
+    args: {
+        profile: option({
+            type: ReadProfile,
+            long: "profile",
+            short: "p",
+            defaultValue: () => "dev" as Profile
+        })
+    },
+    handler: ({profile}) => {
+        console.log("building js")
+        withPublishedWasm(profile, async (registryUrl: string) => {
+            await buildAndPublishAutomergeJs(registryUrl)
+        })
+    }
+})
+
+const buildWasm = command({
+    name: "buildwasm",
+    args: {
+        profile: option({
+            type: ReadProfile,
+            long: "profile",
+            short: "p",
+            defaultValue: () => "dev" as Profile
+        })
+    },
+    handler: ({profile}) => {
+        console.log("building automerge-wasm")
+        withRegistry(
+            publishAutomergeTypes,
+            buildAutomergeWasm(profile),
+        )
+    }
+})
+
+const buildexamples = command({
+    name: "buildexamples",
+    args: {
+        examples: multioption({
+            long: "example",
+            short: "e",
+            type: array(ReadExample),
+        }),
+        profile: option({
+            type: ReadProfile,
+            long: "profile",
+            short: "p",
+            defaultValue: () => "dev" as Profile
+        })
+    },
+    handler: ({examples, profile}) => {
+        if (examples.length === 0) {
+            examples = ["webpack", "vite"]
+        }
+        buildExamples(examples, profile)
+    }
+})
+
+
+const runRegistry = command({
+    name: "run-registry",
+    args: {
+        profile: option({
+            type: ReadProfile,
+            long: "profile",
+            short: "p",
+            defaultValue: () => "dev" as Profile
+        })
+    },
+    handler: ({profile}) => {
+        withPublishedWasm(profile, async (registryUrl: string) => {
+            await buildAndPublishAutomergeJs(registryUrl)
+            console.log("\n************************")
+            console.log(`      Verdaccio NPM registry is running at ${registryUrl}`)
+            console.log("      press CTRL-C to exit ")
+            console.log("************************")
+            await once(process, "SIGINT")
+        }).catch(e => {
+            console.error(`Failed: ${e}`)
+        })
+    }
+})
+
+
+const app = subcommands({
+    name: "e2e",
+    cmds: {buildjs, buildexamples, buildwasm: buildWasm, "run-registry": runRegistry}
+})
+
+run(app, process.argv.slice(2))
+
+async function buildExamples(examples: Array<Example>, profile: Profile) {
+    withPublishedWasm(profile, async (registryUrl) => {
+        printHeader("building and publishing automerge")
+        await buildAndPublishAutomergeJs(registryUrl) 
+        for (const example of examples) {
+            printHeader(`building ${example} example`)
+            if (example === "webpack") {
+                const projectPath = path.join(EXAMPLES_DIR, example)
+                await removeExistingAutomerge(projectPath)
+                await fsPromises.rm(path.join(projectPath, "yarn.lock"), {force: true})
+                await spawnAndWait("yarn", ["--cwd", projectPath, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
+                await spawnAndWait("yarn", ["--cwd", projectPath, "build"], {stdio: "inherit"})
+            } else if (example === "vite") {
+                const projectPath = path.join(EXAMPLES_DIR, example)
+                await removeExistingAutomerge(projectPath)
+                await fsPromises.rm(path.join(projectPath, "yarn.lock"), {force: true})
+                await spawnAndWait("yarn", ["--cwd", projectPath, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
+                await spawnAndWait("yarn", ["--cwd", projectPath, "build"], {stdio: "inherit"})
+            }
+        }
+    })
+}
+
+type WithRegistryAction = (registryUrl: string) => Promise<void>
+
+async function withRegistry(action: WithRegistryAction, ...actions: Array<WithRegistryAction>) {
+    // First, start verdaccio
+    printHeader("Starting verdaccio NPM server")
+    const verd = await VerdaccioProcess.start()
+    actions.unshift(action)
+
+    for (const action of actions) {
+        try {
+            type Step = "verd-died" | "action-completed"
+            const verdDied: () => Promise<Step> = async () => {
+                await verd.died()
+                return "verd-died"
+            }
+            const actionComplete: () => Promise<Step> = async () => {
+                await action("http://localhost:4873")
+                return "action-completed"
+            }
+            const result = await Promise.race([verdDied(), actionComplete()])
+            if (result === "verd-died") {
+                throw new Error("verdaccio unexpectedly exited")
+            }
+        } catch(e) {
+            await verd.kill()
+            throw e
+        }
+    }
+    await verd.kill()
+}
+
+async function withPublishedWasm(profile: Profile, action: WithRegistryAction) {
+    withRegistry(
+        publishAutomergeTypes,
+        buildAutomergeWasm(profile),
+        publishAutomergeWasm,
+        action
+    )
+}
+
+async function publishAutomergeTypes(registryUrl: string) {
+    // Publish automerge-types
+    printHeader("Publishing automerge-types package to verdaccio")
+    await fsPromises.rm(path.join(VERDACCIO_DB_PATH, "automerge-types"), { recursive: true, force: true} )
+    await yarnPublish(registryUrl, path.join(AUTOMERGE_WASM_PATH, "types"))
+}
+
+function buildAutomergeWasm(profile: Profile): WithRegistryAction {
+    return async (registryUrl: string) => {
+        printHeader("building automerge-wasm")
+        await spawnAndWait("yarn", ["--cwd", AUTOMERGE_WASM_PATH, "--registry", registryUrl, "install"], {stdio: "inherit"})
+        const cmd = profile === "release" ? "release" : "debug"
+        await spawnAndWait("yarn", ["--cwd", AUTOMERGE_WASM_PATH, cmd], {stdio: "inherit"})
+    }
+}
+
+async function publishAutomergeWasm(registryUrl: string) {
+    printHeader("Publishing automerge-wasm to verdaccio")
+    await fsPromises.rm(path.join(VERDACCIO_DB_PATH, "automerge-wasm"), { recursive: true, force: true} )
+    await yarnPublish(registryUrl, AUTOMERGE_WASM_PATH)
+}
+
+async function buildAndPublishAutomergeJs(registryUrl: string) {
+    // Build the js package
+    printHeader("Building automerge")
+    await removeExistingAutomerge(AUTOMERGE_JS_PATH)
+    await removeFromVerdaccio("automerge")
+    await fsPromises.rm(path.join(AUTOMERGE_JS_PATH, "yarn.lock"), {force: true})
+    await spawnAndWait("yarn", ["--cwd", AUTOMERGE_JS_PATH, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
+    await spawnAndWait("yarn", ["--cwd", AUTOMERGE_JS_PATH, "build"], {stdio: "inherit"})
+    await yarnPublish(registryUrl, AUTOMERGE_JS_PATH)
+}
+
+/**
+ * A running verdaccio process
+ *
+ */
+class VerdaccioProcess {
+    child: ChildProcess
+    stdout: Array<Buffer>
+    stderr: Array<Buffer>
+
+    constructor(child: ChildProcess) {
+        this.child = child
+
+        // Collect stdout/stderr otherwise the subprocess gets blocked writing
+        this.stdout = []
+        this.stderr = []
+        this.child.on("data", (data) => this.stdout.push(data))
+        this.child.on("data", (data) => this.stderr.push(data))
+
+        const errCallback = (e: any) => {
+            console.error("!!!!!!!!!ERROR IN VERDACCIO PROCESS!!!!!!!!!")
+            console.error("    ", e)
+            if (this.stdout.length > 0) {
+                console.log("\n**Verdaccio stdout**")
+                const stdout = Buffer.concat(this.stdout)
+                process.stdout.write(stdout)
+            }
+
+            if (this.stderr.length > 0) {
+                console.log("\n**Verdaccio stderr**")
+                const stdout = Buffer.concat(this.stderr)
+                process.stdout.write(stdout)
+            }
+            process.exit(-1)
+        }
+        this.child.on("error", errCallback)
+    }
+
+    /**
+     * Spawn a verdaccio process and wait for it to respond succesfully to http requests
+     *
+     * The returned `VerdaccioProcess` can be used to control the subprocess
+     */
+    static async start() {
+        const child = spawn("yarn", ["verdaccio", "--config", VERDACCIO_CONFIG_PATH], {env: { FORCE_COLOR: "true"}})
+
+        // Forward stdout and stderr whilst waiting for startup to complete
+        const stdoutCallback = (data: Buffer) => process.stdout.write(data)
+        const stderrCallback = (data: Buffer) => process.stderr.write(data)
+        child.stdout.on("data", stdoutCallback)
+        child.stderr.on("data", stderrCallback)
+
+        const errored = once(child, "error")
+        const healthCheck = async () => {
+            while (true) {
+                try {
+                    const resp = await fetch("http://localhost:4873")
+                    if (resp.status === 200) {
+                        return
+                    } else {
+                        console.log(`Healthcheck failed: bad status ${resp.status}`)
+                    }
+                } catch (e) {
+                    console.error(`Healthcheck failed: ${e}`)
+                }
+                await setTimeout(500)
+            }
+        }
+        await Promise.race([healthCheck(), errored])
+
+        // Stop forwarding stdout/stderr
+        child.stdout.off("data", stdoutCallback)
+        child.stderr.off("data", stderrCallback)
+        return new VerdaccioProcess(child)
+    }
+
+    /**
+     * Send a SIGKILL to the process and wait for it to stop
+     */
+    async kill() {
+        this.child.kill();
+        const errored = once(this.child, "error")
+        const finished = once(this.child, "close")
+        await Promise.race([errored, finished])
+    }
+
+    /**
+     * A promise which resolves if the subprocess exits for some reason
+     */
+    async died(): Promise<number | null> {
+        const [exit, _signal] = await once(this.child, "exit")
+        return exit
+    }
+}
+
+function printHeader(header: string) {
+    console.log("\n===============================")
+    console.log(`           ${header}`)
+    console.log("===============================")
+}
+
+/**
+ * Removes the automerge, automerge-wasm, and automerge-js packages from
+ * `$packageDir/node_modules`
+ *
+ * This is useful to force refreshing a package by use in combination with
+ * `yarn install --check-files`, which checks if a package is present in
+ * `node_modules` and if it is not forces a reinstall.
+ *
+ * @param packageDir - The directory containing the package.json of the target project
+ */
+async function removeExistingAutomerge(packageDir: string) {
+    await fsPromises.rm(path.join(packageDir, "node_modules", "automerge-wasm"), {recursive: true, force: true})
+    await fsPromises.rm(path.join(packageDir, "node_modules", "automerge-types"), {recursive: true, force: true})
+    await fsPromises.rm(path.join(packageDir, "node_modules", "automerge"), {recursive: true, force: true})
+}
+
+type SpawnResult = {
+    stdout?: Buffer,
+    stderr?: Buffer,
+}
+
+async function spawnAndWait(cmd: string, args: Array<string>, options: child_process.SpawnOptions): Promise<SpawnResult> {
+    const child = spawn(cmd, args, options)
+    let stdout = null
+    let stderr = null
+    if (child.stdout) {
+        stdout = []
+        child.stdout.on("data", data => stdout.push(data))
+    }
+    if (child.stderr) {
+        stderr = []
+        child.stderr.on("data", data => stderr.push(data))
+    }
+
+    const [exit, _signal] = await once(child, "exit")
+    if (exit && exit !== 0) {
+        throw new Error("nonzero exit code")
+    }
+    return {
+        stderr: stderr? Buffer.concat(stderr) : null,
+        stdout: stdout ? Buffer.concat(stdout) : null
+    }
+}
+
+/**
+ * Remove a package from the verdaccio registry. This is necessary because we
+ * often want to _replace_ a version rather than update the version number.
+ * Obviously this is very bad and verboten in normal circumastances, but the
+ * whole point here is to be able to test the entire packaging story so it's
+ * okay I Promise.
+ */
+async function removeFromVerdaccio(packageName: string) {
+    await fsPromises.rm(path.join(VERDACCIO_DB_PATH, packageName), {force: true, recursive: true})
+}
+
+async function yarnPublish(registryUrl: string, cwd: string) {
+    await spawnAndWait(
+        "yarn",
+        [
+            "--registry",
+            registryUrl,
+            "--cwd",
+            cwd,
+            "publish",
+            "--non-interactive",
+        ],
+        {
+            stdio: "inherit",
+            env: {
+                FORCE_COLOR: "true",
+                // This is a fake token, it just has to be the right format
+                npm_config__auth: "//localhost:4873/:_authToken=Gp2Mgxm4faa/7wp0dMSuRA=="
+            }
+        })
+}

automerge-js/e2e/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "e2e",
+  "version": "0.0.1",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "e2e": "ts-node index.ts"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@types/node": "^18.7.18",
+    "cmd-ts": "^0.11.0",
+    "node-fetch": "^2",
+    "ts-node": "^10.9.1",
+    "typed-emitter": "^2.1.0",
+    "typescript": "^4.8.3",
+    "verdaccio": "5"
+  },
+  "devDependencies": {
+    "@types/node-fetch": "2.x"
+  }
+}

automerge-js/e2e/tsconfig.json

@@ -0,0 +1,6 @@
+{
+    "compilerOptions": {
+        "types": ["node"]
+    },
+    "module": "nodenext"
+}

automerge-js/e2e/verdaccio.yaml

@@ -0,0 +1,28 @@
+storage: "./verdacciodb"
+auth:
+  htpasswd:
+    file: ./htpasswd
+publish:
+  allow_offline: true
+logs: {type: stdout, format: pretty, level: info}
+packages: 
+  "automerge-types":
+      access: "$all"
+      publish: "$all"
+  "automerge-wasm":
+      access: "$all"
+      publish: "$all"
+  "automerge-js":
+      access: "$all"
+      publish: "$all"
+  "*":
+      access: "$all"
+      publish: "$all"
+      proxy: npmjs
+  "@*/*":
+      access: "$all"
+      publish: "$all"
+      proxy: npmjs
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/

automerge-js/examples/vite/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+yarn.lock

automerge-js/examples/vite/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + TS</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+  </body>
+</html>

automerge-js/examples/vite/main.ts

@@ -0,0 +1,15 @@
+import * as Automerge from "/node_modules/.vite/deps/automerge-js.js?v=6e973f28";
+console.log(Automerge);
+let doc = Automerge.init();
+doc = Automerge.change(doc, (d) => d.hello = "from automerge-js");
+console.log(doc);
+const result = JSON.stringify(doc);
+if (typeof document !== "undefined") {
+  const element = document.createElement("div");
+  element.innerHTML = JSON.stringify(result);
+  document.body.appendChild(element);
+} else {
+  console.log("node:", result);
+}
+
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIi9ob21lL2FsZXgvUHJvamVjdHMvYXV0b21lcmdlL2F1dG9tZXJnZS1ycy9hdXRvbWVyZ2UtanMvZXhhbXBsZXMvdml0ZS9zcmMvbWFpbi50cyJdLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgKiBhcyBBdXRvbWVyZ2UgZnJvbSBcImF1dG9tZXJnZS1qc1wiXG5cbi8vIGhlbGxvIHdvcmxkIGNvZGUgdGhhdCB3aWxsIHJ1biBjb3JyZWN0bHkgb24gd2ViIG9yIG5vZGVcblxuY29uc29sZS5sb2coQXV0b21lcmdlKVxubGV0IGRvYyA9IEF1dG9tZXJnZS5pbml0KClcbmRvYyA9IEF1dG9tZXJnZS5jaGFuZ2UoZG9jLCAoZDogYW55KSA9PiBkLmhlbGxvID0gXCJmcm9tIGF1dG9tZXJnZS1qc1wiKVxuY29uc29sZS5sb2coZG9jKVxuY29uc3QgcmVzdWx0ID0gSlNPTi5zdHJpbmdpZnkoZG9jKVxuXG5pZiAodHlwZW9mIGRvY3VtZW50ICE9PSAndW5kZWZpbmVkJykge1xuICAgIC8vIGJyb3dzZXJcbiAgICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnZGl2Jyk7XG4gICAgZWxlbWVudC5pbm5lckhUTUwgPSBKU09OLnN0cmluZ2lmeShyZXN1bHQpXG4gICAgZG9jdW1lbnQuYm9keS5hcHBlbmRDaGlsZChlbGVtZW50KTtcbn0gZWxzZSB7XG4gICAgLy8gc2VydmVyXG4gICAgY29uc29sZS5sb2coXCJub2RlOlwiLCByZXN1bHQpXG59XG5cbiJdLCJtYXBwaW5ncyI6IkFBQUEsWUFBWSxlQUFlO0FBSTNCLFFBQVEsSUFBSSxTQUFTO0FBQ3JCLElBQUksTUFBTSxVQUFVLEtBQUs7QUFDekIsTUFBTSxVQUFVLE9BQU8sS0FBSyxDQUFDLE1BQVcsRUFBRSxRQUFRLG1CQUFtQjtBQUNyRSxRQUFRLElBQUksR0FBRztBQUNmLE1BQU0sU0FBUyxLQUFLLFVBQVUsR0FBRztBQUVqQyxJQUFJLE9BQU8sYUFBYSxhQUFhO0FBRWpDLFFBQU0sVUFBVSxTQUFTLGNBQWMsS0FBSztBQUM1QyxVQUFRLFlBQVksS0FBSyxVQUFVLE1BQU07QUFDekMsV0FBUyxLQUFLLFlBQVksT0FBTztBQUNyQyxPQUFPO0FBRUgsVUFBUSxJQUFJLFNBQVMsTUFBTTtBQUMvQjsiLCJuYW1lcyI6W119

automerge-js/examples/vite/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "autovite",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "automerge": "1.0.1-preview.8"
+  },
+  "devDependencies": {
+    "typescript": "^4.6.4",
+    "vite": "^3.1.0",
+    "vite-plugin-top-level-await": "^1.1.1",
+    "vite-plugin-wasm": "^2.1.0"
+  }
+}

automerge-js/examples/vite/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

automerge-js/examples/vite/src/counter.ts

@@ -0,0 +1,9 @@
+export function setupCounter(element: HTMLButtonElement) {
+  let counter = 0
+  const setCounter = (count: number) => {
+    counter = count
+    element.innerHTML = `count is ${counter}`
+  }
+  element.addEventListener('click', () => setCounter(++counter))
+  setCounter(0)
+}

automerge-js/examples/vite/src/main.ts

@@ -0,0 +1,18 @@
+import * as Automerge from "automerge"
+
+// hello world code that will run correctly on web or node
+
+let doc = Automerge.init()
+doc = Automerge.change(doc, (d: any) => d.hello = "from automerge-js")
+const result = JSON.stringify(doc)
+
+if (typeof document !== 'undefined') {
+    // browser
+    const element = document.createElement('div');
+    element.innerHTML = JSON.stringify(result)
+    document.body.appendChild(element);
+} else {
+    // server
+    console.log("node:", result)
+}
+

automerge-js/examples/vite/src/style.css

@@ -0,0 +1,97 @@
+:root {
+  font-family: Inter, Avenir, Helvetica, Arial, sans-serif;
+  font-size: 16px;
+  line-height: 24px;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  -webkit-text-size-adjust: 100%;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+#app {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 2rem;
+  text-align: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.vanilla:hover {
+  filter: drop-shadow(0 0 2em #3178c6aa);
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

automerge-js/examples/vite/src/typescript.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="32" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path fill="#007ACC" d="M0 128v128h256V0H0z"></path><path fill="#FFF" d="m56.612 128.85l-.081 10.483h33.32v94.68h23.568v-94.68h33.321v-10.28c0-5.69-.122-10.444-.284-10.566c-.122-.162-20.4-.244-44.983-.203l-44.74.122l-.121 10.443Zm149.955-10.742c6.501 1.625 11.459 4.51 16.01 9.224c2.357 2.52 5.851 7.111 6.136 8.208c.08.325-11.053 7.802-17.798 11.988c-.244.162-1.22-.894-2.317-2.52c-3.291-4.795-6.745-6.867-12.028-7.233c-7.76-.528-12.759 3.535-12.718 10.321c0 1.992.284 3.17 1.097 4.795c1.707 3.536 4.876 5.649 14.832 9.956c18.326 7.883 26.168 13.084 31.045 20.48c5.445 8.249 6.664 21.415 2.966 31.208c-4.063 10.646-14.14 17.879-28.323 20.276c-4.388.772-14.79.65-19.504-.203c-10.28-1.828-20.033-6.908-26.047-13.572c-2.357-2.6-6.949-9.387-6.664-9.874c.122-.163 1.178-.813 2.356-1.504c1.138-.65 5.446-3.129 9.509-5.485l7.355-4.267l1.544 2.276c2.154 3.29 6.867 7.801 9.712 9.305c8.167 4.307 19.383 3.698 24.909-1.26c2.357-2.153 3.332-4.388 3.332-7.68c0-2.966-.366-4.266-1.91-6.501c-1.99-2.845-6.054-5.242-17.595-10.24c-13.206-5.69-18.895-9.224-24.096-14.832c-3.007-3.25-5.852-8.452-7.03-12.8c-.975-3.617-1.22-12.678-.447-16.335c2.723-12.76 12.353-21.659 26.25-24.3c4.51-.853 14.994-.528 19.424.569Z"></path></svg>

automerge-js/examples/vite/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

automerge-js/examples/vite/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "module": "ESNext",
+    "lib": ["ESNext", "DOM"],
+    "moduleResolution": "Node",
+    "strict": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "esModuleInterop": true,
+    "noEmit": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "skipLibCheck": true
+  },
+  "include": ["src"]
+}

automerge-js/examples/vite/vite.config.js

@@ -0,0 +1,15 @@
+import { defineConfig } from "vite"
+import wasm from "vite-plugin-wasm"
+import topLevelAwait from "vite-plugin-top-level-await"
+
+export default defineConfig({
+    plugins: [topLevelAwait(), wasm()],
+
+    optimizeDeps: {
+        // This is necessary because otherwise `vite dev` includes two separate
+        // versions of the JS wrapper. This causes problems because the JS
+        // wrapper has a module level variable to track JS side heap
+        // allocations, initializing this twice causes horrible breakage
+        exclude: ["automerge-wasm"]
+    }
+})

automerge-js/examples/webpack/package.json

@@ -10,13 +10,13 @@
   },
   "author": "",
   "dependencies": {
-    "automerge-js": "file:automerge-js-0.1.0.tgz",
-    "automerge-wasm": "file:automerge-wasm-0.1.3.tgz"
+    "automerge": "1.0.1-preview.8"
   },
   "devDependencies": {
     "serve": "^13.0.2",
     "webpack": "^5.72.1",
     "webpack-cli": "^4.9.2",
+    "webpack-dev-server": "^4.11.1",
     "webpack-node-externals": "^3.0.0"
   }
 }

automerge-js/examples/webpack/src/index.js

@@ -1,22 +1,18 @@
-import * as Automerge from "automerge-js"
-import init from "automerge-wasm"
+import * as Automerge from "automerge"
 
 // hello world code that will run correctly on web or node
 
-init().then((api) => {
-  Automerge.use(api)
-  let doc = Automerge.init()
-  doc = Automerge.change(doc, (d) => d.hello = "from automerge-js")
-  const result = JSON.stringify(doc)
+let doc = Automerge.init()
+doc = Automerge.change(doc, (d) => d.hello = "from automerge-js")
+const result = JSON.stringify(doc)
 
-  if (typeof document !== 'undefined') {
-    // browser
-    const element = document.createElement('div');
-    element.innerHTML = JSON.stringify(result)
-    document.body.appendChild(element);
-  } else {
-    // server
-    console.log("node:", result)
-  }
-})
+if (typeof document !== 'undefined') {
+  // browser
+  const element = document.createElement('div');
+  element.innerHTML = JSON.stringify(result)
+  document.body.appendChild(element);
+} else {
+  // server
+  console.log("node:", result)
+}
 

automerge-js/examples/webpack/webpack.config.js

@@ -18,6 +18,7 @@ const serverConfig = {
 };
 
 const clientConfig = {
+  experiments: { asyncWebAssembly: true },
   target: 'web',
   entry: './src/index.js',
   output: {

automerge-js/index.d.ts

@@ -77,9 +77,14 @@ type Conflicts = {
     [key: string]: AutomergeValue;
 };
 
+type InitOptions = {
+    actor?: ActorId,
+    freeze?: boolean,
+};
+
 export function use(api: LowLevelApi): void;
 export function getBackend<T>(doc: Doc<T>) : Automerge;
-export function init<T>(actor?: ActorId): Doc<T>;
+export function init<T>(actor?: ActorId | InitOptions): Doc<T>;
 export function clone<T>(doc: Doc<T>): Doc<T>;
 export function free<T>(doc: Doc<T>): void;
 export function from<T>(initialState: T | Doc<T>, actor?: ActorId): Doc<T>;

automerge-js/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "automerge-js",
+  "name": "automerge",
   "collaborators": [
     "Orion Henry <orion@inkandswitch.com>",
     "Martin Kleppmann"
   ],
-  "version": "0.1.12",
+  "version": "1.0.1-preview.8",
   "description": "Reimplementation of `automerge` on top of the automerge-wasm backend",
   "homepage": "https://github.com/automerge/automerge-rs/tree/main/automerge-js",
   "repository": "github:automerge/automerge-rs",
@@ -47,16 +47,17 @@
     "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^5.25.0",
     "@typescript-eslint/parser": "^5.25.0",
-    "automerge-wasm": "^0.1.6",
     "eslint": "^8.15.0",
     "fast-sha256": "^1.3.0",
     "mocha": "^10.0.0",
     "pako": "^2.0.4",
     "ts-mocha": "^10.0.0",
+    "ts-node": "^10.9.1",
     "typescript": "^4.6.4"
   },
   "dependencies": {
-    "automerge-types": "0.1.5",
+    "automerge-types": "0.1.6",
+    "automerge-wasm": "0.1.7",
     "uuid": "^8.3"
   }
 }

automerge-js/src/constants.ts

@@ -1,7 +1,8 @@
 // Properties of the document root object
 //const OPTIONS   = Symbol('_options')   // object containing options passed to init()
 //const CACHE     = Symbol('_cache')     // map from objectId to immutable object
-export const STATE      = Symbol.for('_am_state')     // object containing metadata about current state (e.g. sequence numbers)
+//export const STATE      = Symbol.for('_am_state')     // object containing metadata about current state (e.g. sequence numbers)
+export const STATE      = Symbol.for('_am_meta')     // object containing metadata about current state (e.g. sequence numbers)
 export const HEADS      = Symbol.for('_am_heads')     // object containing metadata about current state (e.g. sequence numbers)
 export const TRACE      = Symbol.for('_am_trace')     // object containing metadata about current state (e.g. sequence numbers)
 export const OBJECT_ID  = Symbol.for('_am_objectId')     // object containing metadata about current state (e.g. sequence numbers)

automerge-js/src/index.ts

@@ -4,7 +4,7 @@ export { uuid } from './uuid'
 import { rootProxy, listProxy, textProxy, mapProxy } from "./proxies"
 import { STATE, HEADS, TRACE, OBJECT_ID, READ_ONLY, FROZEN  } from "./constants"
 
-import { AutomergeValue, Counter } from "./types"
+import { AutomergeValue, Text, Counter } from "./types"
 export { AutomergeValue, Text, Counter, Int, Uint, Float64 } from "./types"
 
 import { API } from "automerge-types";
@@ -13,7 +13,10 @@ import { ApiHandler, UseApi } from "./low_level"
 import { Actor as ActorId, Prop, ObjID, Change, DecodedChange, Heads, Automerge, MaterializeValue } from "automerge-types"
 import { JsSyncState as SyncState, SyncMessage, DecodedSyncMessage } from "automerge-types"
 
-export type ChangeOptions = { message?: string, time?: number }
+import * as wasm from "automerge-wasm"
+
+export type ChangeOptions = { message?: string, time?: number, patchCallback?: Function }
+export type ApplyOptions = { patchCallback?: Function }
 
 export type Doc<T> = { readonly [P in keyof T]: Doc<T[P]> }
 
@@ -24,17 +27,31 @@ export interface State<T> {
   snapshot: T
 }
 
-export function use(api: API) {
+export type InitOptions = {
+    actor?: ActorId,
+    freeze?: boolean,
+    patchCallback?: Function,
+};
+
+function use(api: API) {
   UseApi(api)
 }
+use(wasm)
+
+interface InternalState {
+  handle: Automerge,
+  heads: Heads | undefined,
+  freeze: boolean,
+  patchCallback: Function | undefined,
+}
 
 export function getBackend<T>(doc: Doc<T>) : Automerge {
-  return _state(doc)
+  return _state(doc).handle
 }
 
-function _state<T>(doc: Doc<T>) : Automerge {
+function _state<T>(doc: Doc<T>, checkroot = true) : InternalState {
   const state = Reflect.get(doc,STATE)
-  if (state == undefined) {
+  if (state === undefined || (checkroot && _obj(doc) !== "_root")) {
     throw new RangeError("must be the document root")
   }
   return state
@@ -44,17 +61,12 @@ function _frozen<T>(doc: Doc<T>) : boolean {
   return Reflect.get(doc,FROZEN) === true
 }
 
-function _heads<T>(doc: Doc<T>) : Heads | undefined {
-  return Reflect.get(doc,HEADS)
-}
-
 function _trace<T>(doc: Doc<T>) : string | undefined {
   return Reflect.get(doc,TRACE)
 }
 
 function _set_heads<T>(doc: Doc<T>, heads: Heads) {
-  Reflect.set(doc,HEADS,heads)
-  Reflect.set(doc,TRACE,(new Error()).stack)
+  _state(doc).heads = heads
 }
 
 function _clear_heads<T>(doc: Doc<T>) {
@@ -63,31 +75,58 @@ function _clear_heads<T>(doc: Doc<T>) {
 }
 
 function _obj<T>(doc: Doc<T>) : ObjID {
-  return Reflect.get(doc,OBJECT_ID)
+  let proxy_objid = Reflect.get(doc,OBJECT_ID)
+  if (proxy_objid) {
+    return proxy_objid
+  }
+  if (Reflect.get(doc,STATE)) {
+    return "_root"
+  }
+  throw new RangeError("invalid document passed to _obj()")
 }
 
 function _readonly<T>(doc: Doc<T>) : boolean {
-  return Reflect.get(doc,READ_ONLY) === true
+  return Reflect.get(doc,READ_ONLY) !== false
 }
 
-export function init<T>(actor?: ActorId) : Doc<T>{
-  if (typeof actor !== "string") {
-    actor = undefined
+function importOpts(_actor?: ActorId | InitOptions) : InitOptions {
+  if (typeof _actor === 'object') {
+    return _actor
+  } else {
+    return { actor: _actor }
   }
-  const state = ApiHandler.create(actor)
-  return rootProxy(state, true);
+}
+
+export function init<T>(_opts?: ActorId | InitOptions) : Doc<T>{
+  let opts = importOpts(_opts)
+  let freeze = !!opts.freeze
+  let patchCallback = opts.patchCallback
+  const handle = ApiHandler.create(opts.actor)
+  handle.enablePatches(true)
+  //@ts-ignore
+  handle.registerDatatype("counter", (n) => new Counter(n))
+  //@ts-ignore
+  handle.registerDatatype("text", (n) => new Text(n))
+  //@ts-ignore
+  const doc = handle.materialize("/", undefined, { handle, heads: undefined, freeze, patchCallback })
+  //@ts-ignore
+  return doc
 }
 
 export function clone<T>(doc: Doc<T>) : Doc<T> {
-  const state = _state(doc).clone()
-  return rootProxy(state, true);
+  const state = _state(doc)
+  const handle = state.heads ? state.handle.forkAt(state.heads) : state.handle.fork()
+  //@ts-ignore
+  const clonedDoc : any = handle.materialize("/", undefined, { ... state, handle })
+
+  return clonedDoc
 }
 
 export function free<T>(doc: Doc<T>) {
-  return _state(doc).free()
+  return _state(doc).handle.free()
 }
 
-export function from<T>(initialState: T | Doc<T>, actor?: ActorId): Doc<T> {
+export function from<T extends {}>(initialState: T | Doc<T>, actor?: ActorId): Doc<T> {
     return change(init(actor), (d) => Object.assign(d, initialState))
 }
 
@@ -104,6 +143,16 @@ export function change<T>(doc: Doc<T>, options: string | ChangeOptions | ChangeF
   }
 }
 
+function progressDocument<T>(doc: Doc<T>, heads: Heads, callback?: Function): Doc<T> {
+  let state = _state(doc)
+  let nextState = { ... state, heads: undefined };
+  // @ts-ignore
+  let nextDoc = state.handle.applyPatches(doc, nextState, callback)
+  state.heads = heads
+  if (nextState.freeze) { Object.freeze(nextDoc) }
+  return nextDoc
+}
+
 function _change<T>(doc: Doc<T>, options: ChangeOptions, callback: ChangeFn<T>): Doc<T> {
 
 
@@ -111,38 +160,33 @@ function _change<T>(doc: Doc<T>, options: ChangeOptions, callback: ChangeFn<T>):
     throw new RangeError("invalid change function");
   }
 
-  if (doc === undefined || _state(doc) === undefined || _obj(doc) !== "_root") {
+  const state = _state(doc)
+
+  if (doc === undefined || state === undefined) {
     throw new RangeError("must be the document root");
   }
-  if (_frozen(doc) === true) {
+  if (state.heads) {
     throw new RangeError("Attempting to use an outdated Automerge document")
   }
-  if (!!_heads(doc) === true) {
-    throw new RangeError("Attempting to change an out of date document - set at: " + _trace(doc));
-  }
   if (_readonly(doc) === false) {
     throw new RangeError("Calls to Automerge.change cannot be nested")
   }
-  const state = _state(doc)
-  const heads = state.getHeads()
+  const heads = state.handle.getHeads()
   try {
-    _set_heads(doc,heads)
-    Reflect.set(doc,FROZEN,true)
-    const root : T = rootProxy(state);
+    state.heads = heads
+    const root : T = rootProxy(state.handle);
     callback(root)
-    if (state.pendingOps() === 0) {
-      Reflect.set(doc,FROZEN,false)
-      _clear_heads(doc)
+    if (state.handle.pendingOps() === 0) {
+      state.heads = undefined
       return doc
     } else {
-      state.commit(options.message, options.time)
-      return rootProxy(state, true);
+      state.handle.commit(options.message, options.time)
+      return progressDocument(doc, heads, options.patchCallback || state.patchCallback);
     }
   } catch (e) {
     //console.log("ERROR: ",e)
-    Reflect.set(doc,FROZEN,false)
-    _clear_heads(doc)
-    state.rollback()
+    state.heads = undefined
+    state.handle.rollback()
     throw e
   }
 }
@@ -155,47 +199,55 @@ export function emptyChange<T>(doc: Doc<T>, options: ChangeOptions) {
     options = { message: options }
   }
 
-  if (doc === undefined || _state(doc) === undefined || _obj(doc) !== "_root") {
-    throw new RangeError("must be the document root");
-  }
-  if (_frozen(doc) === true) {
+  const state = _state(doc)
+
+  if (state.heads) {
     throw new RangeError("Attempting to use an outdated Automerge document")
   }
   if (_readonly(doc) === false) {
     throw new RangeError("Calls to Automerge.change cannot be nested")
   }
 
-  const state = _state(doc)
-  state.commit(options.message, options.time)
-  return rootProxy(state, true);
+  const heads = state.handle.getHeads()
+  state.handle.commit(options.message, options.time)
+  return progressDocument(doc, heads)
 }
 
-export function load<T>(data: Uint8Array, actor?: ActorId) : Doc<T> {
-  const state = ApiHandler.load(data, actor)
-  return rootProxy(state, true);
+export function load<T>(data: Uint8Array, _opts?: ActorId | InitOptions) : Doc<T> {
+  const opts = importOpts(_opts)
+  const actor = opts.actor
+  const patchCallback = opts.patchCallback
+  const handle = ApiHandler.load(data, actor)
+  handle.enablePatches(true)
+  //@ts-ignore
+  handle.registerDatatype("counter", (n) => new Counter(n))
+  //@ts-ignore
+  handle.registerDatatype("text", (n) => new Text(n))
+  //@ts-ignore
+  const doc : any = handle.materialize("/", undefined, { handle, heads: undefined, patchCallback })
+  return doc
 }
 
 export function save<T>(doc: Doc<T>) : Uint8Array  {
-  const state = _state(doc)
-  return state.save()
+  return _state(doc).handle.save()
 }
 
 export function merge<T>(local: Doc<T>, remote: Doc<T>) : Doc<T> {
-  if (!!_heads(local) === true) {
+  const localState = _state(local)
+
+  if (localState.heads) {
     throw new RangeError("Attempting to change an out of date document - set at: " + _trace(local));
   }
-  const localState = _state(local)
-  const heads = localState.getHeads()
+  const heads = localState.handle.getHeads()
   const remoteState = _state(remote)
-  const changes = localState.getChangesAdded(remoteState)
-  localState.applyChanges(changes)
-  _set_heads(local,heads)
-  return rootProxy(localState, true)
+  const changes = localState.handle.getChangesAdded(remoteState.handle)
+  localState.handle.applyChanges(changes)
+  return progressDocument(local, heads, localState.patchCallback)
 }
 
 export function getActorId<T>(doc: Doc<T>) : ActorId {
   const state = _state(doc)
-  return state.getActorId()
+  return state.handle.getActorId()
 }
 
 type Conflicts = { [key: string]: AutomergeValue }
@@ -242,14 +294,14 @@ function conflictAt(context : Automerge, objectId: ObjID, prop: Prop) : Conflict
 }
 
 export function getConflicts<T>(doc: Doc<T>, prop: Prop) : Conflicts | undefined {
-  const state = _state(doc)
+  const state = _state(doc, false)
   const objectId = _obj(doc)
-  return conflictAt(state, objectId, prop)
+  return conflictAt(state.handle, objectId, prop)
 }
 
 export function getLastLocalChange<T>(doc: Doc<T>) : Change | undefined {
   const state = _state(doc)
-  return state.getLastLocalChange() || undefined
+  return state.handle.getLastLocalChange() || undefined
 }
 
 export function getObjectId<T>(doc: Doc<T>) : ObjID {
@@ -259,30 +311,27 @@ export function getObjectId<T>(doc: Doc<T>) : ObjID {
 export function getChanges<T>(oldState: Doc<T>, newState: Doc<T>) : Change[] {
   const o = _state(oldState)
   const n = _state(newState)
-  const heads = _heads(oldState)
-  return n.getChanges(heads || o.getHeads())
+  return n.handle.getChanges(getHeads(oldState))
 }
 
 export function getAllChanges<T>(doc: Doc<T>) : Change[] {
   const state = _state(doc)
-  return state.getChanges([])
+  return state.handle.getChanges([])
 }
 
-export function applyChanges<T>(doc: Doc<T>, changes: Change[]) : [Doc<T>] {
-  if (doc === undefined || _obj(doc) !== "_root") {
-    throw new RangeError("must be the document root");
-  }
-  if (_frozen(doc) === true) {
+export function applyChanges<T>(doc: Doc<T>, changes: Change[], opts?: ApplyOptions) : [Doc<T>] {
+  const state = _state(doc)
+  if (!opts) { opts = {} }
+  if (state.heads) {
     throw new RangeError("Attempting to use an outdated Automerge document")
   }
   if (_readonly(doc) === false) {
     throw new RangeError("Calls to Automerge.change cannot be nested")
   }
-  const state = _state(doc)
-  const heads = state.getHeads()
-  state.applyChanges(changes)
-  _set_heads(doc,heads)
-  return [rootProxy(state, true)];
+  const heads = state.handle.getHeads();
+  state.handle.applyChanges(changes)
+  state.heads = heads;
+  return [progressDocument(doc, heads, opts.patchCallback || state.patchCallback )]
 }
 
 export function getHistory<T>(doc: Doc<T>) : State<T>[] {
@@ -300,6 +349,7 @@ export function getHistory<T>(doc: Doc<T>) : State<T>[] {
 }
 
 // FIXME : no tests
+// FIXME can we just use deep equals now?
 export function equals(val1: unknown, val2: unknown) : boolean {
   if (!isObject(val1) || !isObject(val2)) return val1 === val2
   const keys1 = Object.keys(val1).sort(), keys2 = Object.keys(val2).sort()
@@ -322,31 +372,25 @@ export function decodeSyncState(state: Uint8Array) : SyncState {
 export function generateSyncMessage<T>(doc: Doc<T>, inState: SyncState) : [ SyncState, SyncMessage | null ] {
   const state = _state(doc)
   const syncState = ApiHandler.importSyncState(inState)
-  const message = state.generateSyncMessage(syncState)
+  const message = state.handle.generateSyncMessage(syncState)
   const outState = ApiHandler.exportSyncState(syncState)
   return [ outState, message ]
 }
 
-export function receiveSyncMessage<T>(doc: Doc<T>, inState: SyncState, message: SyncMessage) : [ Doc<T>, SyncState, null ] {
+export function receiveSyncMessage<T>(doc: Doc<T>, inState: SyncState, message: SyncMessage, opts?: ApplyOptions) : [ Doc<T>, SyncState, null ] {
   const syncState = ApiHandler.importSyncState(inState)
-  if (doc === undefined || _obj(doc) !== "_root") {
-    throw new RangeError("must be the document root");
-  }
-  if (_frozen(doc) === true) {
-    throw new RangeError("Attempting to use an outdated Automerge document")
-  }
-  if (!!_heads(doc) === true) {
+  if (!opts) { opts = {} }
+  const state = _state(doc)
+  if (state.heads) {
     throw new RangeError("Attempting to change an out of date document - set at: " + _trace(doc));
   }
   if (_readonly(doc) === false) {
     throw new RangeError("Calls to Automerge.change cannot be nested")
   }
-  const state = _state(doc)
-  const heads = state.getHeads()
-  state.receiveSyncMessage(syncState, message)
-  _set_heads(doc,heads)
-  const outState = ApiHandler.exportSyncState(syncState)
-  return [rootProxy(state, true), outState, null];
+  const heads = state.handle.getHeads()
+  state.handle.receiveSyncMessage(syncState, message)
+  const outSyncState = ApiHandler.exportSyncState(syncState)
+  return [progressDocument(doc, heads, opts.patchCallback || state.patchCallback), outSyncState, null];
 }
 
 export function initSyncState() : SyncState {
@@ -371,24 +415,24 @@ export function decodeSyncMessage(message: SyncMessage) : DecodedSyncMessage {
 
 export function getMissingDeps<T>(doc: Doc<T>, heads: Heads) : Heads {
   const state = _state(doc)
-  return state.getMissingDeps(heads)
+  return state.handle.getMissingDeps(heads)
 }
 
 export function getHeads<T>(doc: Doc<T>) : Heads {
   const state = _state(doc)
-  return _heads(doc) || state.getHeads()
+  return state.heads || state.handle.getHeads()
 }
 
 export function dump<T>(doc: Doc<T>) {
   const state = _state(doc)
-  state.dump()
+  state.handle.dump()
 }
 
 // FIXME - return T?
 export function toJS<T>(doc: Doc<T>) : MaterializeValue {
   const state = _state(doc)
-  const heads = _heads(doc)
-  return state.materialize("_root", heads)
+  // @ts-ignore
+  return state.handle.materialize("_root", state.heads, state)
 }
 
 

automerge-js/src/low_level.ts

@@ -11,15 +11,15 @@ export function UseApi(api: API) {
 /* eslint-disable */
 export const ApiHandler : API = {
   create(actor?: Actor): Automerge { throw new RangeError("Automerge.use() not called") },
-  load(data: Uint8Array, actor?: Actor): Automerge { throw new RangeError("Automerge.use() not called") },
-  encodeChange(change: DecodedChange): Change { throw new RangeError("Automerge.use() not called") },
-  decodeChange(change: Change): DecodedChange { throw new RangeError("Automerge.use() not called") },
-  initSyncState(): SyncState { throw new RangeError("Automerge.use() not called") },
-  encodeSyncMessage(message: DecodedSyncMessage): SyncMessage { throw new RangeError("Automerge.use() not called") },
-  decodeSyncMessage(msg: SyncMessage): DecodedSyncMessage { throw new RangeError("Automerge.use() not called") },
-  encodeSyncState(state: SyncState): Uint8Array { throw new RangeError("Automerge.use() not called") },
-  decodeSyncState(data: Uint8Array): SyncState { throw new RangeError("Automerge.use() not called") },
-  exportSyncState(state: SyncState): JsSyncState { throw new RangeError("Automerge.use() not called") },
-  importSyncState(state: JsSyncState): SyncState { throw new RangeError("Automerge.use() not called") },
+  load(data: Uint8Array, actor?: Actor): Automerge { throw new RangeError("Automerge.use() not called (load)") },
+  encodeChange(change: DecodedChange): Change { throw new RangeError("Automerge.use() not called (encodeChange)") },
+  decodeChange(change: Change): DecodedChange { throw new RangeError("Automerge.use() not called (decodeChange)") },
+  initSyncState(): SyncState { throw new RangeError("Automerge.use() not called (initSyncState)") },
+  encodeSyncMessage(message: DecodedSyncMessage): SyncMessage { throw new RangeError("Automerge.use() not called (encodeSyncMessage)") },
+  decodeSyncMessage(msg: SyncMessage): DecodedSyncMessage { throw new RangeError("Automerge.use() not called (decodeSyncMessage)") },
+  encodeSyncState(state: SyncState): Uint8Array { throw new RangeError("Automerge.use() not called (encodeSyncState)") },
+  decodeSyncState(data: Uint8Array): SyncState { throw new RangeError("Automerge.use() not called (decodeSyncState)") },
+  exportSyncState(state: SyncState): JsSyncState { throw new RangeError("Automerge.use() not called (exportSyncState)") },
+  importSyncState(state: JsSyncState): SyncState { throw new RangeError("Automerge.use() not called (importSyncState)") },
 }
 /* eslint-enable */

automerge-js/src/proxies.ts

@@ -219,18 +219,6 @@ const ListHandler = {
     if (index === TRACE) return target.trace
     if (index === STATE) return context;
     if (index === 'length') return context.length(objectId, heads);
-    if (index === Symbol.iterator) {
-      let i = 0;
-      return function *() {
-        // FIXME - ugly
-        let value = valueAt(target, i)
-        while (value !== undefined) {
-            yield value
-            i += 1
-            value = valueAt(target, i)
-        }
-      }
-    }
     if (typeof index === 'number') {
       return valueAt(target, index)
     } else {
@@ -369,17 +357,6 @@ const TextHandler = Object.assign({}, ListHandler, {
     if (index === TRACE) return target.trace
     if (index === STATE) return context;
     if (index === 'length') return context.length(objectId, heads);
-    if (index === Symbol.iterator) {
-      let i = 0;
-      return function *() {
-        let value = valueAt(target, i)
-        while (value !== undefined) {
-            yield value
-            i += 1
-            value = valueAt(target, i)
-        }
-      }
-    }
     if (typeof index === 'number') {
       return valueAt(target, index)
     } else {
@@ -425,11 +402,11 @@ function listMethods(target) {
     },
 
     fill(val: ScalarValue, start: number, end: number) {
-      // FIXME needs tests
       const [value, datatype] = import_value(val)
+      const length = context.length(objectId)
       start = parseListIndex(start || 0)
-      end = parseListIndex(end || context.length(objectId))
-      for (let i = start; i < end; i++) {
+      end = parseListIndex(end || length)
+      for (let i = start; i < Math.min(end, length); i++) {
         context.put(objectId, i, value, datatype)
       }
       return this
@@ -573,15 +550,9 @@ function listMethods(target) {
         }
       }
       return iterator
-    }
-  }
+    },
 
-  // Read-only methods that can delegate to the JavaScript built-in implementations
-  // FIXME - super slow
-  for (const method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes',
-                      'join', 'lastIndexOf', 'map', 'reduce', 'reduceRight',
-                      'slice', 'some', 'toLocaleString', 'toString']) {
-    methods[method] = (...args) => {
+    toArray() : AutomergeValue[] {
       const list : AutomergeValue = []
       let value
       do {
@@ -591,10 +562,107 @@ function listMethods(target) {
         }
       } while (value !== undefined)
 
-      return list[method](...args)
+      return list
+    },
+
+    map<T>(f: (AutomergeValue, number) => T) : T[] {
+      return this.toArray().map(f)
+    },
+
+    toString() : string {
+      return this.toArray().toString()
+    },
+
+    toLocaleString() : string {
+      return this.toArray().toLocaleString()
+    },
+
+    forEach(f: (AutomergeValue, number) => undefined ) {
+      return this.toArray().forEach(f)
+    },
+
+    // todo: real concat function is different
+    concat(other: AutomergeValue[]) : AutomergeValue[] {
+      return this.toArray().concat(other)
+    },
+
+    every(f: (AutomergeValue, number) => boolean) : boolean {
+      return this.toArray().every(f)
+    },
+
+    filter(f: (AutomergeValue, number) => boolean) : AutomergeValue[] {
+      return this.toArray().filter(f)
+    },
+
+    find(f: (AutomergeValue, number) => boolean) : AutomergeValue | undefined {
+      let index = 0
+      for (let v of this) {
+        if (f(v, index)) {
+          return v
+        }
+        index += 1
+      }
+    },
+
+    findIndex(f: (AutomergeValue, number) => boolean) : number {
+      let index = 0
+      for (let v of this) {
+        if (f(v, index)) {
+          return index
+        }
+        index += 1
+      }
+      return -1
+    },
+
+    includes(elem: AutomergeValue) : boolean {
+      return this.find((e) => e === elem) !== undefined
+    },
+
+    join(sep?: string) : string {
+      return this.toArray().join(sep)
+    },
+
+    // todo: remove the any
+    reduce<T>(f: (any, AutomergeValue) => T, initalValue?: T) : T | undefined {
+      return this.toArray().reduce(f,initalValue)
+    },
+
+    // todo: remove the any
+    reduceRight<T>(f: (any, AutomergeValue) => T, initalValue?: T) : T | undefined{
+      return this.toArray().reduceRight(f,initalValue)
+    },
+
+    lastIndexOf(search: AutomergeValue, fromIndex = +Infinity) : number {
+      // this can be faster
+      return this.toArray().lastIndexOf(search,fromIndex)
+    },
+
+    slice(index?: number, num?: number) : AutomergeValue[] {
+      return this.toArray().slice(index,num)
+    },
+
+    some(f: (AutomergeValue, number) => boolean) : boolean {
+      let index = 0;
+      for (let v of this) {
+        if (f(v,index)) {
+          return true
+        }
+        index += 1
+      }
+      return false
+    },
+
+    [Symbol.iterator]: function *() {
+      let i = 0;
+      let value = valueAt(target, i)
+      while (value !== undefined) {
+          yield value
+          i += 1
+          value = valueAt(target, i)
+      }
     }
   }
-
   return methods
 }
 

automerge-js/src/text.ts

@@ -1,11 +1,12 @@
 import { Value } from "automerge-types"
-import { TEXT } from "./constants"
+import { TEXT, STATE } from "./constants"
 
 export class Text {
   elems: Value[]
+  str: string | undefined
+  spans: Value[] | undefined
 
-  constructor (text?: string | string[]) {
-    //const instance = Object.create(Text.prototype)
+  constructor (text?: string | string[] | Value[]) {
     if (typeof text === 'string') {
       this.elems = [...text]
     } else if (Array.isArray(text)) {
@@ -50,14 +51,17 @@ export class Text {
    * non-character elements.
    */
   toString() : string {
-    // Concatting to a string is faster than creating an array and then
-    // .join()ing for small (<100KB) arrays.
-    // https://jsperf.com/join-vs-loop-w-type-test
-    let str = ''
-    for (const elem of this.elems) {
-      if (typeof elem === 'string') str += elem
+    if (!this.str) {
+      // Concatting to a string is faster than creating an array and then
+      // .join()ing for small (<100KB) arrays.
+      // https://jsperf.com/join-vs-loop-w-type-test
+      this.str = ''
+      for (const elem of this.elems) {
+        if (typeof elem === 'string') this.str += elem
+        else this.str += '\uFFFC'
+      }
     }
-    return str
+    return this.str
   }
 
   /**
@@ -68,23 +72,25 @@ export class Text {
    * => ['ab', {x: 3}, 'cd']
    */
   toSpans() : Value[] {
-    const spans : Value[] = []
-    let chars = ''
-    for (const elem of this.elems) {
-      if (typeof elem === 'string') {
-        chars += elem
-      } else {
-        if (chars.length > 0) {
-          spans.push(chars)
-          chars = ''
+    if (!this.spans) {
+      this.spans = []
+      let chars = ''
+      for (const elem of this.elems) {
+        if (typeof elem === 'string') {
+          chars += elem
+        } else {
+          if (chars.length > 0) {
+            this.spans.push(chars)
+            chars = ''
+          }
+          this.spans.push(elem)
         }
-        spans.push(elem)
+      }
+      if (chars.length > 0) {
+        this.spans.push(chars)
       }
     }
-    if (chars.length > 0) {
-      spans.push(chars)
-    }
-    return spans
+    return this.spans
   }
 
   /**
@@ -99,6 +105,9 @@ export class Text {
    * Updates the list item at position `index` to a new value `value`.
    */
   set (index: number, value: Value) {
+    if (this[STATE]) {
+      throw new RangeError("object cannot be modified outside of a change block")
+    }
     this.elems[index] = value
   }
 
@@ -106,6 +115,9 @@ export class Text {
    * Inserts new list items `values` starting at position `index`.
    */
   insertAt(index: number, ...values: Value[]) {
+    if (this[STATE]) {
+      throw new RangeError("object cannot be modified outside of a change block")
+    }
     this.elems.splice(index, 0, ... values)
   }
 
@@ -114,6 +126,9 @@ export class Text {
    * if `numDelete` is not given, one item is deleted.
    */
   deleteAt(index: number, numDelete = 1) {
+    if (this[STATE]) {
+      throw new RangeError("object cannot be modified outside of a change block")
+    }
     this.elems.splice(index, numDelete)
   }
 
@@ -121,16 +136,64 @@ export class Text {
     this.elems.map(callback)
   }
 
+  lastIndexOf(searchElement: Value, fromIndex?: number) {
+    this.elems.lastIndexOf(searchElement, fromIndex)
+  }
 
-}
+  concat(other: Text) : Text {
+    return new Text(this.elems.concat(other.elems))
+  }
 
-// Read-only methods that can delegate to the JavaScript built-in array
-for (const method of ['concat', 'every', 'filter', 'find', 'findIndex', 'forEach', 'includes',
-                    'indexOf', 'join', 'lastIndexOf', 'reduce', 'reduceRight',
-                    'slice', 'some', 'toLocaleString']) {
-  Text.prototype[method] = function (...args) {
-    const array = [...this]
-    return array[method](...args)
+  every(test: (Value) => boolean) : boolean {
+    return this.elems.every(test)
+  }
+
+  filter(test: (Value) => boolean) : Text {
+    return new Text(this.elems.filter(test))
+  }
+
+  find(test: (Value) => boolean) : Value | undefined {
+    return this.elems.find(test)
+  }
+
+  findIndex(test: (Value) => boolean) : number | undefined {
+    return this.elems.findIndex(test)
+  }
+
+  forEach(f: (Value) => undefined) {
+    this.elems.forEach(f)
+  }
+
+  includes(elem: Value) : boolean {
+    return this.elems.includes(elem)
+  }
+
+  indexOf(elem: Value) {
+    return this.elems.indexOf(elem)
+  }
+
+  join(sep?: string) : string{
+    return this.elems.join(sep)
+  }
+
+  reduce(f: (previousValue: Value, currentValue: Value, currentIndex: number, array: Value[]) => Value)  {
+    this.elems.reduce(f)
+  }
+
+  reduceRight(f: (previousValue: Value, currentValue: Value, currentIndex: number, array: Value[]) => Value) {
+    this.elems.reduceRight(f)
+  }
+
+  slice(start?: number, end?: number) {
+    new Text(this.elems.slice(start,end))
+  }
+
+  some(test: (Value) => boolean) : boolean {
+    return this.elems.some(test)
+  }
+
+  toLocaleString() {
+    this.toString()
   }
 }
 

automerge-js/test/basic_test.ts

@@ -2,9 +2,6 @@ import * as tt from "automerge-types"
 import * as assert from 'assert'
 import * as util from 'util'
 import * as Automerge from '../src'
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 describe('Automerge', () => {
     describe('basics', () => {
@@ -175,4 +172,64 @@ describe('Automerge', () => {
           console.log(doc.text.indexOf("world"))
         })
     })
+
+    describe('proxy lists', () => {
+        it('behave like arrays', () => {
+          let doc = Automerge.from({
+            chars: ["a","b","c"],
+            numbers: [20,3,100],
+            repeats: [20,20,3,3,3,3,100,100]
+          })
+          let r1 = []
+          doc = Automerge.change(doc, (d) => {
+            assert.deepEqual(d.chars.concat([1,2]), ["a","b","c",1,2])
+            assert.deepEqual(d.chars.map((n) => n + "!"), ["a!", "b!", "c!"])
+            assert.deepEqual(d.numbers.map((n) => n + 10), [30, 13, 110])
+            assert.deepEqual(d.numbers.toString(), "20,3,100")
+            assert.deepEqual(d.numbers.toLocaleString(), "20,3,100")
+            assert.deepEqual(d.numbers.forEach((n) => r1.push(n)), undefined)
+            assert.deepEqual(d.numbers.every((n) => n > 1), true)
+            assert.deepEqual(d.numbers.every((n) => n > 10), false)
+            assert.deepEqual(d.numbers.filter((n) => n > 10), [20,100])
+            assert.deepEqual(d.repeats.find((n) => n < 10), 3)
+            assert.deepEqual(d.repeats.toArray().find((n) => n < 10), 3)
+            assert.deepEqual(d.repeats.find((n) => n < 0), undefined)
+            assert.deepEqual(d.repeats.findIndex((n) => n < 10), 2)
+            assert.deepEqual(d.repeats.findIndex((n) => n < 0), -1)
+            assert.deepEqual(d.repeats.toArray().findIndex((n) => n < 10), 2)
+            assert.deepEqual(d.repeats.toArray().findIndex((n) => n < 0), -1)
+            assert.deepEqual(d.numbers.includes(3), true)
+            assert.deepEqual(d.numbers.includes(-3), false)
+            assert.deepEqual(d.numbers.join("|"), "20|3|100")
+            assert.deepEqual(d.numbers.join(), "20,3,100")
+            assert.deepEqual(d.numbers.some((f) => f === 3), true)
+            assert.deepEqual(d.numbers.some((f) => f < 0), false)
+            assert.deepEqual(d.numbers.reduce((sum,n) => sum + n, 100), 223)
+            assert.deepEqual(d.repeats.reduce((sum,n) => sum + n, 100), 352)
+            assert.deepEqual(d.chars.reduce((sum,n) => sum + n, "="), "=abc")
+            assert.deepEqual(d.chars.reduceRight((sum,n) => sum + n, "="), "=cba")
+            assert.deepEqual(d.numbers.reduceRight((sum,n) => sum + n, 100), 223)
+            assert.deepEqual(d.repeats.lastIndexOf(3), 5)
+            assert.deepEqual(d.repeats.lastIndexOf(3,3), 3)
+          })
+          doc = Automerge.change(doc, (d) => {
+            assert.deepEqual(d.numbers.fill(-1,1,2), [20,-1,100])
+            assert.deepEqual(d.chars.fill("z",1,100), ["a","z","z"])
+          })
+          assert.deepEqual(r1, [20,3,100])
+          assert.deepEqual(doc.numbers, [20,-1,100])
+          assert.deepEqual(doc.chars, ["a","z","z"])
+        })
+    })
+    
+    it('should obtain the same conflicts, regardless of merge order', () => {
+      let s1 = Automerge.init()
+      let s2 = Automerge.init()
+      s1 = Automerge.change<any>(s1, doc => { doc.x = 1; doc.y = 2 })
+      s2 = Automerge.change<any>(s2, doc => { doc.x = 3; doc.y = 4 })
+      const m1 = Automerge.merge(Automerge.clone(s1), Automerge.clone(s2))
+      const m2 = Automerge.merge(Automerge.clone(s2), Automerge.clone(s1))
+      assert.deepStrictEqual(Automerge.getConflicts(m1, 'x'), Automerge.getConflicts(m2, 'x'))
+    })
 })
+

automerge-js/test/columnar_test.ts

@@ -2,9 +2,6 @@ import * as assert from 'assert'
 import { checkEncoded } from './helpers'
 import * as Automerge from '../src'
 import { encodeChange, decodeChange } from '../src'
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 describe('change encoding', () => {
   it('should encode text edits', () => {

automerge-js/test/legacy_tests.ts

@@ -2,9 +2,6 @@ import * as assert from 'assert'
 import * as Automerge from '../src'
 import { assertEqualsOneOf } from './helpers'
 import { decodeChange } from './legacy/columnar'
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 const UUID_PATTERN = /^[0-9a-f]{32}$/
 const OPID_PATTERN = /^[0-9]+@[0-9a-f]{32}$/
@@ -283,47 +280,34 @@ describe('Automerge', () => {
         assert.strictEqual(s2.list[0].getTime(), now.getTime())
       })
 
-      /*
-      it.skip('should call patchCallback if supplied', () => {
+      it('should call patchCallback if supplied', () => {
         const callbacks = [], actor = Automerge.getActorId(s1)
         const s2 = Automerge.change(s1, {
-          patchCallback: (patch, before, after, local) => callbacks.push({patch, before, after, local})
+          patchCallback: (patch, before, after) => callbacks.push({patch, before, after})
         }, doc => {
           doc.birds = ['Goldfinch']
         })
-        assert.strictEqual(callbacks.length, 1)
-        assert.deepStrictEqual(callbacks[0].patch, {
-          actor, seq: 1, maxOp: 2, deps: [], clock: {[actor]: 1}, pendingChanges: 0,
-          diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: {
-            objectId: `1@${actor}`, type: 'list', edits: [
-              {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {'type': 'value', value: 'Goldfinch'}}
-            ]
-          }}}}
-        })
+        assert.strictEqual(callbacks.length, 2)
+        assert.deepStrictEqual(callbacks[0].patch, { action: "put", path: ["birds"], value: [], conflict: false})
+        assert.deepStrictEqual(callbacks[1].patch, { action: "splice", path: ["birds",0], values: ["Goldfinch"] })
         assert.strictEqual(callbacks[0].before, s1)
-        assert.strictEqual(callbacks[0].after, s2)
-        assert.strictEqual(callbacks[0].local, true)
+        assert.strictEqual(callbacks[1].after, s2)
       })
-      */
 
-      /*
-      it.skip('should call a patchCallback set up on document initialisation', () => {
+      it('should call a patchCallback set up on document initialisation', () => {
         const callbacks = []
         s1 = Automerge.init({
-          patchCallback: (patch, before, after, local) => callbacks.push({patch, before, after, local})
+          patchCallback: (patch, before, after) => callbacks.push({patch, before, after })
         })
         const s2 = Automerge.change(s1, doc => doc.bird = 'Goldfinch')
         const actor = Automerge.getActorId(s1)
         assert.strictEqual(callbacks.length, 1)
         assert.deepStrictEqual(callbacks[0].patch, {
-          actor, seq: 1, maxOp: 1, deps: [], clock: {[actor]: 1}, pendingChanges: 0,
-          diffs: {objectId: '_root', type: 'map', props: {bird: {[`1@${actor}`]: {type: 'value', value: 'Goldfinch'}}}}
+          action: "put", path: ["bird"], value: "Goldfinch", conflict: false
         })
         assert.strictEqual(callbacks[0].before, s1)
         assert.strictEqual(callbacks[0].after, s2)
-        assert.strictEqual(callbacks[0].local, true)
       })
-    */
     })
 
     describe('emptyChange()', () => {
@@ -897,7 +881,7 @@ describe('Automerge', () => {
       })
     })
 
-    it('should handle assignment conflicts of different types', () => {
+    it.skip('should handle assignment conflicts of different types', () => {
       s1 = Automerge.change(s1, doc => doc.field = 'string')
       s2 = Automerge.change(s2, doc => doc.field = ['list'])
       s3 = Automerge.change(s3, doc => doc.field = {thing: 'map'})
@@ -922,7 +906,8 @@ describe('Automerge', () => {
       })
     })
 
-    it('should handle changes within a conflicting list element', () => {
+    // FIXME - difficult bug here - patches arrive for conflicted subobject
+    it.skip('should handle changes within a conflicting list element', () => {
       s1 = Automerge.change(s1, doc => doc.list = ['hello'])
       s2 = Automerge.merge(s2, s1)
       s1 = Automerge.change(s1, doc => doc.list[0] = {map1: true})
@@ -1207,8 +1192,7 @@ describe('Automerge', () => {
       assert.deepStrictEqual(doc, {list: expected})
     })
 
-    /*
-    it.skip('should call patchCallback if supplied', () => {
+    it.skip('should call patchCallback if supplied to load', () => {
       const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch'])
       const s2 = Automerge.change(s1, doc => doc.birds.push('Chaffinch'))
       const callbacks = [], actor = Automerge.getActorId(s1)
@@ -1230,7 +1214,6 @@ describe('Automerge', () => {
       assert.strictEqual(callbacks[0].after, reloaded)
       assert.strictEqual(callbacks[0].local, false)
     })
-    */
   })
 
   describe('history API', () => {
@@ -1357,65 +1340,48 @@ describe('Automerge', () => {
       let s4 = Automerge.init()
       let [s5] = Automerge.applyChanges(s4, changes23)
       let [s6] = Automerge.applyChanges(s5, changes12)
-//      assert.deepStrictEqual(Automerge.Backend.getMissingDeps(Automerge.Frontend.getBackendState(s6)), [decodeChange(changes01[0]).hash])
       assert.deepStrictEqual(Automerge.getMissingDeps(s6), [decodeChange(changes01[0]).hash])
     })
 
-    /*
-    it.skip('should call patchCallback if supplied when applying changes', () => {
+    it('should call patchCallback if supplied when applying changes', () => {
       const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch'])
       const callbacks = [], actor = Automerge.getActorId(s1)
       const before = Automerge.init()
       const [after, patch] = Automerge.applyChanges(before, Automerge.getAllChanges(s1), {
-        patchCallback(patch, before, after, local) {
-          callbacks.push({patch, before, after, local})
+        patchCallback(patch, before, after) {
+          callbacks.push({patch, before, after})
         }
       })
-      assert.strictEqual(callbacks.length, 1)
-      assert.deepStrictEqual(callbacks[0].patch, {
-        maxOp: 2, deps: [decodeChange(Automerge.getAllChanges(s1)[0]).hash], clock: {[actor]: 1}, pendingChanges: 0,
-        diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: {
-          objectId: `1@${actor}`, type: 'list', edits: [
-            {action: 'insert', index: 0, elemId: `2@${actor}`, opId: `2@${actor}`, value: {type: 'value', value: 'Goldfinch'}}
-          ]
-        }}}}
-      })
-      assert.strictEqual(callbacks[0].patch, patch)
+      assert.strictEqual(callbacks.length, 2)
+      assert.deepStrictEqual(callbacks[0].patch, { action: 'put', path: ["birds"], value: [], conflict: false })
+      assert.deepStrictEqual(callbacks[1].patch, { action: 'splice', path: ["birds",0], values: ["Goldfinch"] })
       assert.strictEqual(callbacks[0].before, before)
-      assert.strictEqual(callbacks[0].after, after)
-      assert.strictEqual(callbacks[0].local, false)
+      assert.strictEqual(callbacks[1].after, after)
     })
-    */
 
-    /*
-    it.skip('should merge multiple applied changes into one patch', () => {
+    it('should merge multiple applied changes into one patch', () => {
       const s1 = Automerge.change(Automerge.init(), doc => doc.birds = ['Goldfinch'])
       const s2 = Automerge.change(s1, doc => doc.birds.push('Chaffinch'))
       const patches = [], actor = Automerge.getActorId(s2)
       Automerge.applyChanges(Automerge.init(), Automerge.getAllChanges(s2),
                              {patchCallback: p => patches.push(p)})
-      assert.deepStrictEqual(patches, [{
-        maxOp: 3, deps: [decodeChange(Automerge.getAllChanges(s2)[1]).hash], clock: {[actor]: 2}, pendingChanges: 0,
-        diffs: {objectId: '_root', type: 'map', props: {birds: {[`1@${actor}`]: {
-          objectId: `1@${actor}`, type: 'list', edits: [
-            {action: 'multi-insert', index: 0, elemId: `2@${actor}`, values: ['Goldfinch', 'Chaffinch']}
-          ]
-        }}}}
-      }])
+      assert.deepStrictEqual(patches, [
+        { action: 'put', conflict: false, path: [ 'birds' ], value: [] },
+        { action: "splice", path: [ "birds", 0 ], values: [ "Goldfinch", "Chaffinch" ] }
+      ])
     })
-    */
 
-    /*
-    it.skip('should call a patchCallback registered on doc initialisation', () => {
+    it('should call a patchCallback registered on doc initialisation', () => {
       const s1 = Automerge.change(Automerge.init(), doc => doc.bird = 'Goldfinch')
       const patches = [], actor = Automerge.getActorId(s1)
       const before = Automerge.init({patchCallback: p => patches.push(p)})
       Automerge.applyChanges(before, Automerge.getAllChanges(s1))
       assert.deepStrictEqual(patches, [{
-        maxOp: 1, deps: [decodeChange(Automerge.getAllChanges(s1)[0]).hash], clock: {[actor]: 1}, pendingChanges: 0,
-        diffs: {objectId: '_root', type: 'map', props: {bird: {[`1@${actor}`]: {type: 'value', value: 'Goldfinch'}}}}
-      }])
+          action: "put",
+          conflict: false,
+          path: [ "bird" ],
+          value: "Goldfinch" }
+      ])
     })
-    */
   })
 })

automerge-js/test/sync_test.ts

@@ -3,9 +3,6 @@ import * as Automerge from '../src'
 import { BloomFilter } from './legacy/sync'
 import { decodeChangeMeta } from './legacy/columnar'
 import { decodeSyncMessage, encodeSyncMessage, decodeSyncState, encodeSyncState, initSyncState } from "../src"
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 function inspect(a) {
   const util = require("util");
@@ -538,7 +535,7 @@ describe('Data sync protocol', () => {
         assert.deepStrictEqual(getHeads(n2), [n1hash2, n2hash2].sort())
       })
 
-      it('should sync three nodes', () => {
+      it.skip('should sync three nodes', () => {
         s1 = decodeSyncState(encodeSyncState(s1))
         s2 = decodeSyncState(encodeSyncState(s2))
 

automerge-js/test/text_test.ts

@@ -1,9 +1,6 @@
 import * as assert from 'assert'
 import * as Automerge from '../src'
 import { assertEqualsOneOf } from './helpers'
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 function attributeStateToAttributes(accumulatedAttributes) {
   const attributes = {}
@@ -385,8 +382,8 @@ describe('Automerge.Text', () => {
       assert.strictEqual(s1.text.get(0), 'a')
     })
 
-    it('should exclude control characters from toString()', () => {
-      assert.strictEqual(s1.text.toString(), 'a')
+    it('should replace control characters from toString()', () => {
+      assert.strictEqual(s1.text.toString(), 'a\uFFFC')
     })
 
     it('should allow control characters to be updated', () => {
@@ -623,7 +620,7 @@ describe('Automerge.Text', () => {
           applyDeltaDocToAutomergeText(delta, doc)
         })
 
-        assert.strictEqual(s2.text.toString(), 'Hello reader!')
+        assert.strictEqual(s2.text.toString(), 'Hello \uFFFCreader\uFFFC!')
         assert.deepEqual(s2.text.toSpans(), [
           "Hello ",
           { attributes: { bold: true } },
@@ -651,7 +648,7 @@ describe('Automerge.Text', () => {
           applyDeltaDocToAutomergeText(delta, doc)
         })
 
-        assert.strictEqual(s2.text.toString(), 'Hello reader!')
+        assert.strictEqual(s2.text.toString(), 'Hell\uFFFCo \uFFFCreader\uFFFC\uFFFC!')
         assert.deepEqual(s2.text.toSpans(), [
           "Hell",
           { attributes: { color: '#ccc'} },

automerge-js/test/uuid_test.ts

@@ -1,8 +1,5 @@
 import * as assert from 'assert'
 import * as Automerge from '../src'
-import * as AutomergeWASM from "automerge-wasm"
-
-Automerge.use(AutomergeWASM)
 
 const uuid = Automerge.uuid
 

automerge-wasm/Cargo.toml

@@ -33,9 +33,10 @@ serde-wasm-bindgen = "0.1.3"
 serde_bytes = "0.11.5"
 hex = "^0.4.3"
 regex = "^1.5"
+itertools = "^0.10.3"
 
 [dependencies.wasm-bindgen]
-version = "^0.2"
+version = "^0.2.83"
 #features = ["std"]
 features = ["serde-serialize", "std"]
 

automerge-wasm/index.d.ts

@@ -1,2 +1,17 @@
+import { Automerge as VanillaAutomerge } from "automerge-types"
+
 export * from "automerge-types"
 export { default } from "automerge-types"
+
+export class Automerge extends VanillaAutomerge {
+  // experimental api can go here
+  applyPatches<Doc>(obj: Doc, meta?: JsValue, callback?: Function): Doc;
+
+  // override old methods that return automerge
+  clone(actor?: string): Automerge;
+  fork(actor?: string): Automerge;
+  forkAt(heads: Heads, actor?: string): Automerge;
+}
+
+export function create(actor?: Actor): Automerge;
+export function load(data: Uint8Array, actor?: Actor): Automerge;

automerge-wasm/nodejs-index.js

@@ -1,5 +0,0 @@
-let wasm = require("./bindgen")
-module.exports = wasm
-module.exports.load = module.exports.loadDoc
-delete module.exports.loadDoc
-module.exports.init = () => (new Promise((resolve,reject) => { resolve(module.exports) }))

automerge-wasm/package.json

@@ -8,29 +8,29 @@
   "description": "wasm-bindgen bindings to the automerge rust implementation",
   "homepage": "https://github.com/automerge/automerge-rs/tree/main/automerge-wasm",
   "repository": "github:automerge/automerge-rs",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "license": "MIT",
   "files": [
     "README.md",
     "LICENSE",
     "package.json",
     "index.d.ts",
-    "nodejs/index.js",
     "nodejs/bindgen.js",
     "nodejs/bindgen_bg.wasm",
-    "web/index.js",
-    "web/bindgen.js",
-    "web/bindgen_bg.wasm"
+    "bundler/bindgen.js",
+    "bundler/bindgen_bg.js",
+    "bundler/bindgen_bg.wasm"
   ],
   "types": "index.d.ts",
-  "module": "./web/index.js",
-  "main": "./nodejs/index.js",
+  "module": "./bundler/bindgen.js",
+  "main": "./nodejs/bindgen.js",
   "scripts": {
     "lint": "eslint test/*.ts",
     "build": "cross-env PROFILE=dev TARGET=nodejs FEATURES='' yarn target",
+    "debug": "cross-env PROFILE=dev yarn buildall",
     "release": "cross-env PROFILE=release yarn buildall",
-    "buildall": "cross-env TARGET=nodejs yarn target && cross-env TARGET=web yarn target",
-    "target": "rimraf ./$TARGET && wasm-pack build --target $TARGET --$PROFILE --out-name bindgen -d $TARGET -- $FEATURES  && cp $TARGET-index.js $TARGET/index.js",
+    "buildall": "cross-env TARGET=nodejs yarn target && cross-env TARGET=bundler yarn target",
+    "target": "rimraf ./$TARGET && wasm-pack build --target $TARGET --$PROFILE --out-name bindgen -d $TARGET -- $FEATURES",
     "test": "ts-mocha -p tsconfig.json --type-check --bail --full-trace test/*.ts"
   },
   "devDependencies": {
@@ -51,6 +51,10 @@
     "typescript": "^4.6.4"
   },
   "dependencies": {
-    "automerge-types": "0.1.5"
+    "automerge-types": "0.1.6"
+  },
+  "exports": {
+    "browser": "./bundler/bindgen.js",
+    "require": "./nodejs/bindgen.js"
   }
 }

automerge-wasm/src/interop.rs

@@ -1,13 +1,20 @@
+use crate::value::Datatype;
+use crate::Automerge;
 use automerge as am;
 use automerge::transaction::Transactable;
-use automerge::{Change, ChangeHash, Prop};
-use js_sys::{Array, Object, Reflect, Uint8Array};
+use automerge::{Change, ChangeHash, ObjType, Prop};
+use js_sys::{Array, Function, Object, Reflect, Symbol, Uint8Array};
 use std::collections::{BTreeSet, HashSet};
 use std::fmt::Display;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen::JsCast;
 
-use crate::{ObjId, ScalarValue, Value};
+use crate::{observer::Patch, ObjId, Value};
+
+const RAW_DATA_SYMBOL: &str = "_am_raw_value_";
+const DATATYPE_SYMBOL: &str = "_am_datatype_";
+const RAW_OBJECT_SYMBOL: &str = "_am_objectId";
+const META_SYMBOL: &str = "_am_meta";
 
 pub(crate) struct JS(pub(crate) JsValue);
 pub(crate) struct AR(pub(crate) Array);
@@ -50,11 +57,11 @@ impl From<am::sync::State> for JS {
 
 impl From<Vec<ChangeHash>> for JS {
     fn from(heads: Vec<ChangeHash>) -> Self {
-        let heads: Array = heads
+        JS(heads
             .iter()
             .map(|h| JsValue::from_str(&h.to_string()))
-            .collect();
-        JS(heads.into())
+            .collect::<Array>()
+            .into())
     }
 }
 
@@ -287,17 +294,16 @@ pub(crate) fn to_prop(p: JsValue) -> Result<Prop, JsValue> {
 pub(crate) fn to_objtype(
     value: &JsValue,
     datatype: &Option<String>,
-) -> Option<(am::ObjType, Vec<(Prop, JsValue)>)> {
+) -> Option<(ObjType, Vec<(Prop, JsValue)>)> {
     match datatype.as_deref() {
         Some("map") => {
             let map = value.clone().dyn_into::<js_sys::Object>().ok()?;
-            // FIXME unwrap
             let map = js_sys::Object::keys(&map)
                 .iter()
                 .zip(js_sys::Object::values(&map).iter())
                 .map(|(key, val)| (key.as_string().unwrap().into(), val))
                 .collect();
-            Some((am::ObjType::Map, map))
+            Some((ObjType::Map, map))
         }
         Some("list") => {
             let list = value.clone().dyn_into::<js_sys::Array>().ok()?;
@@ -306,7 +312,7 @@ pub(crate) fn to_objtype(
                 .enumerate()
                 .map(|(i, e)| (i.into(), e))
                 .collect();
-            Some((am::ObjType::List, list))
+            Some((ObjType::List, list))
         }
         Some("text") => {
             let text = value.as_string()?;
@@ -315,7 +321,7 @@ pub(crate) fn to_objtype(
                 .enumerate()
                 .map(|(i, ch)| (i.into(), ch.to_string().into()))
                 .collect();
-            Some((am::ObjType::Text, text))
+            Some((ObjType::Text, text))
         }
         Some(_) => None,
         None => {
@@ -325,7 +331,7 @@ pub(crate) fn to_objtype(
                     .enumerate()
                     .map(|(i, e)| (i.into(), e))
                     .collect();
-                Some((am::ObjType::List, list))
+                Some((ObjType::List, list))
             } else if let Ok(map) = value.clone().dyn_into::<js_sys::Object>() {
                 // FIXME unwrap
                 let map = js_sys::Object::keys(&map)
@@ -333,14 +339,14 @@ pub(crate) fn to_objtype(
                     .zip(js_sys::Object::values(&map).iter())
                     .map(|(key, val)| (key.as_string().unwrap().into(), val))
                     .collect();
-                Some((am::ObjType::Map, map))
+                Some((ObjType::Map, map))
             } else if let Some(text) = value.as_string() {
                 let text = text
                     .chars()
                     .enumerate()
                     .map(|(i, ch)| (i.into(), ch.to_string().into()))
                     .collect();
-                Some((am::ObjType::Text, text))
+                Some((ObjType::Text, text))
             } else {
                 None
             }
@@ -354,106 +360,358 @@ pub(crate) fn get_heads(heads: Option<Array>) -> Option<Vec<ChangeHash>> {
     heads.ok()
 }
 
-pub(crate) fn map_to_js(doc: &am::AutoCommit, obj: &ObjId) -> JsValue {
-    let keys = doc.keys(obj);
-    let map = Object::new();
-    for k in keys {
-        let val = doc.get(obj, &k);
-        match val {
-            Ok(Some((Value::Object(o), exid)))
-                if o == am::ObjType::Map || o == am::ObjType::Table =>
-            {
-                Reflect::set(&map, &k.into(), &map_to_js(doc, &exid)).unwrap();
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::List => {
-                Reflect::set(&map, &k.into(), &list_to_js(doc, &exid)).unwrap();
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::Text => {
-                Reflect::set(&map, &k.into(), &doc.text(&exid).unwrap().into()).unwrap();
-            }
-            Ok(Some((Value::Scalar(v), _))) => {
-                Reflect::set(&map, &k.into(), &ScalarValue(v).into()).unwrap();
-            }
-            _ => (),
+impl Automerge {
+    pub(crate) fn export_object(
+        &self,
+        obj: &ObjId,
+        datatype: Datatype,
+        heads: Option<&Vec<ChangeHash>>,
+        meta: &JsValue,
+    ) -> Result<JsValue, JsValue> {
+        let result = if datatype.is_sequence() {
+            self.wrap_object(
+                self.export_list(obj, heads, meta)?,
+                datatype,
+                &obj.to_string().into(),
+                meta,
+            )?
+        } else {
+            self.wrap_object(
+                self.export_map(obj, heads, meta)?,
+                datatype,
+                &obj.to_string().into(),
+                meta,
+            )?
         };
+        Ok(result.into())
+    }
+
+    pub(crate) fn export_map(
+        &self,
+        obj: &ObjId,
+        heads: Option<&Vec<ChangeHash>>,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let keys = self.doc.keys(obj);
+        let map = Object::new();
+        for k in keys {
+            let val_and_id = if let Some(heads) = heads {
+                self.doc.get_at(obj, &k, heads)
+            } else {
+                self.doc.get(obj, &k)
+            };
+            if let Ok(Some((val, id))) = val_and_id {
+                let subval = match val {
+                    Value::Object(o) => self.export_object(&id, o.into(), heads, meta)?,
+                    Value::Scalar(_) => self.export_value(alloc(&val))?,
+                };
+                Reflect::set(&map, &k.into(), &subval)?;
+            };
+        }
+
+        Ok(map)
+    }
+
+    pub(crate) fn export_list(
+        &self,
+        obj: &ObjId,
+        heads: Option<&Vec<ChangeHash>>,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let len = self.doc.length(obj);
+        let array = Array::new();
+        for i in 0..len {
+            let val_and_id = if let Some(heads) = heads {
+                self.doc.get_at(obj, i as usize, heads)
+            } else {
+                self.doc.get(obj, i as usize)
+            };
+            if let Ok(Some((val, id))) = val_and_id {
+                let subval = match val {
+                    Value::Object(o) => self.export_object(&id, o.into(), heads, meta)?,
+                    Value::Scalar(_) => self.export_value(alloc(&val))?,
+                };
+                array.push(&subval);
+            };
+        }
+
+        Ok(array.into())
+    }
+
+    pub(crate) fn export_value(
+        &self,
+        (datatype, raw_value): (Datatype, JsValue),
+    ) -> Result<JsValue, JsValue> {
+        if let Some(function) = self.external_types.get(&datatype) {
+            let wrapped_value = function.call1(&JsValue::undefined(), &raw_value)?;
+            if let Ok(o) = wrapped_value.dyn_into::<Object>() {
+                let key = Symbol::for_(RAW_DATA_SYMBOL);
+                set_hidden_value(&o, &key, &raw_value)?;
+                let key = Symbol::for_(DATATYPE_SYMBOL);
+                set_hidden_value(&o, &key, datatype)?;
+                Ok(o.into())
+            } else {
+                Err(to_js_err(format!(
+                    "data handler for type {} did not return a valid object",
+                    datatype
+                )))
+            }
+        } else {
+            Ok(raw_value)
+        }
+    }
+
+    pub(crate) fn unwrap_object(
+        &self,
+        ext_val: &Object,
+    ) -> Result<(Object, Datatype, JsValue), JsValue> {
+        let inner = Reflect::get(ext_val, &Symbol::for_(RAW_DATA_SYMBOL))?;
+
+        let datatype = Reflect::get(ext_val, &Symbol::for_(DATATYPE_SYMBOL))?.try_into();
+
+        let mut id = Reflect::get(ext_val, &Symbol::for_(RAW_OBJECT_SYMBOL))?;
+        if id.is_undefined() {
+            id = "_root".into();
+        }
+
+        let inner = inner
+            .dyn_into::<Object>()
+            .unwrap_or_else(|_| ext_val.clone());
+        let datatype = datatype.unwrap_or_else(|_| {
+            if Array::is_array(&inner) {
+                Datatype::List
+            } else {
+                Datatype::Map
+            }
+        });
+        Ok((inner, datatype, id))
+    }
+
+    pub(crate) fn unwrap_scalar(&self, ext_val: JsValue) -> Result<JsValue, JsValue> {
+        let inner = Reflect::get(&ext_val, &Symbol::for_(RAW_DATA_SYMBOL))?;
+        if !inner.is_undefined() {
+            Ok(inner)
+        } else {
+            Ok(ext_val)
+        }
+    }
+
+    fn maybe_wrap_object(
+        &self,
+        (datatype, raw_value): (Datatype, JsValue),
+        id: &ObjId,
+        meta: &JsValue,
+    ) -> Result<JsValue, JsValue> {
+        if let Ok(obj) = raw_value.clone().dyn_into::<Object>() {
+            let result = self.wrap_object(obj, datatype, &id.to_string().into(), meta)?;
+            Ok(result.into())
+        } else {
+            self.export_value((datatype, raw_value))
+        }
+    }
+
+    pub(crate) fn wrap_object(
+        &self,
+        value: Object,
+        datatype: Datatype,
+        id: &JsValue,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let value = if let Some(function) = self.external_types.get(&datatype) {
+            let wrapped_value = function.call1(&JsValue::undefined(), &value)?;
+            let wrapped_object = wrapped_value.dyn_into::<Object>().map_err(|_| {
+                to_js_err(format!(
+                    "data handler for type {} did not return a valid object",
+                    datatype
+                ))
+            })?;
+            set_hidden_value(&wrapped_object, &Symbol::for_(RAW_DATA_SYMBOL), value)?;
+            wrapped_object
+        } else {
+            value
+        };
+        set_hidden_value(&value, &Symbol::for_(DATATYPE_SYMBOL), datatype)?;
+        set_hidden_value(&value, &Symbol::for_(RAW_OBJECT_SYMBOL), id)?;
+        set_hidden_value(&value, &Symbol::for_(META_SYMBOL), meta)?;
+        Ok(value)
+    }
+
+    pub(crate) fn apply_patch_to_array(
+        &self,
+        array: &Object,
+        patch: &Patch,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let result = Array::from(array); // shallow copy
+        match patch {
+            Patch::PutSeq { index, value, .. } => {
+                let sub_val = self.maybe_wrap_object(alloc(&value.0), &value.1, meta)?;
+                Reflect::set(&result, &(*index as f64).into(), &sub_val)?;
+                Ok(result.into())
+            }
+            Patch::DeleteSeq { index, .. } => self.sub_splice(result, *index, 1, &[], meta),
+            Patch::Insert { index, values, .. } => self.sub_splice(result, *index, 0, values, meta),
+            Patch::Increment { prop, value, .. } => {
+                if let Prop::Seq(index) = prop {
+                    let index = (*index as f64).into();
+                    let old_val = Reflect::get(&result, &index)?;
+                    let old_val = self.unwrap_scalar(old_val)?;
+                    if let Some(old) = old_val.as_f64() {
+                        let new_value: Value<'_> =
+                            am::ScalarValue::counter(old as i64 + *value).into();
+                        Reflect::set(&result, &index, &self.export_value(alloc(&new_value))?)?;
+                        Ok(result.into())
+                    } else {
+                        Err(to_js_err("cant increment a non number value"))
+                    }
+                } else {
+                    Err(to_js_err("cant increment a key on a seq"))
+                }
+            }
+            Patch::DeleteMap { .. } => Err(to_js_err("cannot delete from a seq")),
+            Patch::PutMap { .. } => Err(to_js_err("cannot set key in seq")),
+        }
+    }
+
+    pub(crate) fn apply_patch_to_map(
+        &self,
+        map: &Object,
+        patch: &Patch,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let result = Object::assign(&Object::new(), map); // shallow copy
+        match patch {
+            Patch::PutMap { key, value, .. } => {
+                let sub_val = self.maybe_wrap_object(alloc(&value.0), &value.1, meta)?;
+                Reflect::set(&result, &key.into(), &sub_val)?;
+                Ok(result)
+            }
+            Patch::DeleteMap { key, .. } => {
+                Reflect::delete_property(&result, &key.into())?;
+                Ok(result)
+            }
+            Patch::Increment { prop, value, .. } => {
+                if let Prop::Map(key) = prop {
+                    let key = key.into();
+                    let old_val = Reflect::get(&result, &key)?;
+                    let old_val = self.unwrap_scalar(old_val)?;
+                    if let Some(old) = old_val.as_f64() {
+                        let new_value: Value<'_> =
+                            am::ScalarValue::counter(old as i64 + *value).into();
+                        Reflect::set(&result, &key, &self.export_value(alloc(&new_value))?)?;
+                        Ok(result)
+                    } else {
+                        Err(to_js_err("cant increment a non number value"))
+                    }
+                } else {
+                    Err(to_js_err("cant increment an index on a map"))
+                }
+            }
+            Patch::Insert { .. } => Err(to_js_err("cannot insert into map")),
+            Patch::DeleteSeq { .. } => Err(to_js_err("cannot splice a map")),
+            Patch::PutSeq { .. } => Err(to_js_err("cannot array index a map")),
+        }
+    }
+
+    pub(crate) fn apply_patch(
+        &self,
+        obj: Object,
+        patch: &Patch,
+        depth: usize,
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let (inner, datatype, id) = self.unwrap_object(&obj)?;
+        let prop = patch.path().get(depth).map(|p| prop_to_js(&p.1));
+        let result = if let Some(prop) = prop {
+            if let Ok(sub_obj) = Reflect::get(&inner, &prop)?.dyn_into::<Object>() {
+                let new_value = self.apply_patch(sub_obj, patch, depth + 1, meta)?;
+                let result = shallow_copy(&inner);
+                Reflect::set(&result, &prop, &new_value)?;
+                Ok(result)
+            } else {
+                // if a patch is trying to access a deleted object make no change
+                // short circuit the wrap process
+                return Ok(obj);
+            }
+        } else if Array::is_array(&inner) {
+            self.apply_patch_to_array(&inner, patch, meta)
+        } else {
+            self.apply_patch_to_map(&inner, patch, meta)
+        }?;
+
+        self.wrap_object(result, datatype, &id, meta)
+    }
+
+    fn sub_splice(
+        &self,
+        o: Array,
+        index: usize,
+        num_del: usize,
+        values: &[(Value<'_>, ObjId)],
+        meta: &JsValue,
+    ) -> Result<Object, JsValue> {
+        let args: Array = values
+            .iter()
+            .map(|v| self.maybe_wrap_object(alloc(&v.0), &v.1, meta))
+            .collect::<Result<_, _>>()?;
+        args.unshift(&(num_del as u32).into());
+        args.unshift(&(index as u32).into());
+        let method = Reflect::get(&o, &"splice".into())?.dyn_into::<Function>()?;
+        Reflect::apply(&method, &o, &args)?;
+        Ok(o.into())
     }
-    map.into()
 }
 
-pub(crate) fn map_to_js_at(doc: &am::AutoCommit, obj: &ObjId, heads: &[ChangeHash]) -> JsValue {
-    let keys = doc.keys(obj);
-    let map = Object::new();
-    for k in keys {
-        let val = doc.get_at(obj, &k, heads);
-        match val {
-            Ok(Some((Value::Object(o), exid)))
-                if o == am::ObjType::Map || o == am::ObjType::Table =>
-            {
-                Reflect::set(&map, &k.into(), &map_to_js_at(doc, &exid, heads)).unwrap();
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::List => {
-                Reflect::set(&map, &k.into(), &list_to_js_at(doc, &exid, heads)).unwrap();
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::Text => {
-                Reflect::set(&map, &k.into(), &doc.text_at(&exid, heads).unwrap().into()).unwrap();
-            }
-            Ok(Some((Value::Scalar(v), _))) => {
-                Reflect::set(&map, &k.into(), &ScalarValue(v).into()).unwrap();
-            }
-            _ => (),
-        };
+pub(crate) fn alloc(value: &Value<'_>) -> (Datatype, JsValue) {
+    match value {
+        am::Value::Object(o) => match o {
+            ObjType::Map => (Datatype::Map, Object::new().into()),
+            ObjType::Table => (Datatype::Table, Object::new().into()),
+            ObjType::List => (Datatype::List, Array::new().into()),
+            ObjType::Text => (Datatype::Text, Array::new().into()),
+        },
+        am::Value::Scalar(s) => match s.as_ref() {
+            am::ScalarValue::Bytes(v) => (Datatype::Bytes, Uint8Array::from(v.as_slice()).into()),
+            am::ScalarValue::Str(v) => (Datatype::Str, v.to_string().into()),
+            am::ScalarValue::Int(v) => (Datatype::Int, (*v as f64).into()),
+            am::ScalarValue::Uint(v) => (Datatype::Uint, (*v as f64).into()),
+            am::ScalarValue::F64(v) => (Datatype::F64, (*v).into()),
+            am::ScalarValue::Counter(v) => (Datatype::Counter, (f64::from(v)).into()),
+            am::ScalarValue::Timestamp(v) => (
+                Datatype::Timestamp,
+                js_sys::Date::new(&(*v as f64).into()).into(),
+            ),
+            am::ScalarValue::Boolean(v) => (Datatype::Boolean, (*v).into()),
+            am::ScalarValue::Null => (Datatype::Null, JsValue::null()),
+            am::ScalarValue::Unknown { bytes, type_code } => (
+                Datatype::Unknown(*type_code),
+                Uint8Array::from(bytes.as_slice()).into(),
+            ),
+        },
     }
-    map.into()
 }
 
-pub(crate) fn list_to_js(doc: &am::AutoCommit, obj: &ObjId) -> JsValue {
-    let len = doc.length(obj);
-    let array = Array::new();
-    for i in 0..len {
-        let val = doc.get(obj, i as usize);
-        match val {
-            Ok(Some((Value::Object(o), exid)))
-                if o == am::ObjType::Map || o == am::ObjType::Table =>
-            {
-                array.push(&map_to_js(doc, &exid));
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::List => {
-                array.push(&list_to_js(doc, &exid));
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::Text => {
-                array.push(&doc.text(&exid).unwrap().into());
-            }
-            Ok(Some((Value::Scalar(v), _))) => {
-                array.push(&ScalarValue(v).into());
-            }
-            _ => (),
-        };
-    }
-    array.into()
+fn set_hidden_value<V: Into<JsValue>>(o: &Object, key: &Symbol, value: V) -> Result<(), JsValue> {
+    let definition = Object::new();
+    js_set(&definition, "value", &value.into())?;
+    js_set(&definition, "writable", false)?;
+    js_set(&definition, "enumerable", false)?;
+    js_set(&definition, "configurable", false)?;
+    Object::define_property(o, &key.into(), &definition);
+    Ok(())
 }
 
-pub(crate) fn list_to_js_at(doc: &am::AutoCommit, obj: &ObjId, heads: &[ChangeHash]) -> JsValue {
-    let len = doc.length(obj);
-    let array = Array::new();
-    for i in 0..len {
-        let val = doc.get_at(obj, i as usize, heads);
-        match val {
-            Ok(Some((Value::Object(o), exid)))
-                if o == am::ObjType::Map || o == am::ObjType::Table =>
-            {
-                array.push(&map_to_js_at(doc, &exid, heads));
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::List => {
-                array.push(&list_to_js_at(doc, &exid, heads));
-            }
-            Ok(Some((Value::Object(o), exid))) if o == am::ObjType::Text => {
-                array.push(&doc.text_at(exid, heads).unwrap().into());
-            }
-            Ok(Some((Value::Scalar(v), _))) => {
-                array.push(&ScalarValue(v).into());
-            }
-            _ => (),
-        };
+fn shallow_copy(obj: &Object) -> Object {
+    if Array::is_array(obj) {
+        Array::from(obj).into()
+    } else {
+        Object::assign(&Object::new(), obj)
+    }
+}
+
+fn prop_to_js(prop: &Prop) -> JsValue {
+    match prop {
+        Prop::Map(key) => key.into(),
+        Prop::Seq(index) => (*index as f64).into(),
     }
-    array.into()
 }

automerge-wasm/src/lib.rs

@@ -28,26 +28,24 @@
 #![allow(clippy::unused_unit)]
 use am::transaction::CommitOptions;
 use am::transaction::Transactable;
-use am::ApplyOptions;
 use automerge as am;
-use automerge::Patch;
-use automerge::VecOpObserver;
-use automerge::{Change, ObjId, Prop, Value, ROOT};
-use js_sys::{Array, Object, Uint8Array};
+use automerge::{Change, ObjId, ObjType, Prop, Value, ROOT};
+use js_sys::{Array, Function, Object, Uint8Array};
+use std::collections::HashMap;
 use std::convert::TryInto;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen::JsCast;
 
 mod interop;
+mod observer;
 mod sync;
 mod value;
 
-use interop::{
-    get_heads, js_get, js_set, list_to_js, list_to_js_at, map_to_js, map_to_js_at, to_js_err,
-    to_objtype, to_prop, AR, JS,
-};
+use observer::Observer;
+
+use interop::{alloc, get_heads, js_get, js_set, to_js_err, to_objtype, to_prop, AR, JS};
 use sync::SyncState;
-use value::{datatype, ScalarValue};
+use value::Datatype;
 
 #[allow(unused_macros)]
 macro_rules! log {
@@ -56,6 +54,8 @@ macro_rules! log {
     };
 }
 
+type AutoCommit = am::AutoCommitWithObs<Observer>;
+
 #[cfg(feature = "wee_alloc")]
 #[global_allocator]
 static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
@@ -63,40 +63,29 @@ static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
 #[wasm_bindgen]
 #[derive(Debug)]
 pub struct Automerge {
-    doc: automerge::AutoCommit,
-    observer: Option<VecOpObserver>,
+    doc: AutoCommit,
+    external_types: HashMap<Datatype, Function>,
 }
 
 #[wasm_bindgen]
 impl Automerge {
     pub fn new(actor: Option<String>) -> Result<Automerge, JsValue> {
-        let mut automerge = automerge::AutoCommit::new();
+        let mut doc = AutoCommit::default();
         if let Some(a) = actor {
             let a = automerge::ActorId::from(hex::decode(a).map_err(to_js_err)?.to_vec());
-            automerge.set_actor(a);
+            doc.set_actor(a);
         }
         Ok(Automerge {
-            doc: automerge,
-            observer: None,
+            doc,
+            external_types: HashMap::default(),
         })
     }
 
-    fn ensure_transaction_closed(&mut self) {
-        if self.doc.pending_ops() > 0 {
-            let mut opts = CommitOptions::default();
-            if let Some(observer) = self.observer.as_mut() {
-                opts.set_op_observer(observer);
-            }
-            self.doc.commit_with(opts);
-        }
-    }
-
     #[allow(clippy::should_implement_trait)]
     pub fn clone(&mut self, actor: Option<String>) -> Result<Automerge, JsValue> {
-        self.ensure_transaction_closed();
         let mut automerge = Automerge {
             doc: self.doc.clone(),
-            observer: None,
+            external_types: self.external_types.clone(),
         };
         if let Some(s) = actor {
             let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
@@ -106,10 +95,9 @@ impl Automerge {
     }
 
     pub fn fork(&mut self, actor: Option<String>) -> Result<Automerge, JsValue> {
-        self.ensure_transaction_closed();
         let mut automerge = Automerge {
             doc: self.doc.fork(),
-            observer: None,
+            external_types: self.external_types.clone(),
         };
         if let Some(s) = actor {
             let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
@@ -123,7 +111,7 @@ impl Automerge {
         let deps: Vec<_> = JS(heads).try_into()?;
         let mut automerge = Automerge {
             doc: self.doc.fork_at(&deps)?,
-            observer: None,
+            external_types: self.external_types.clone(),
         };
         if let Some(s) = actor {
             let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
@@ -147,21 +135,12 @@ impl Automerge {
         if let Some(time) = time {
             commit_opts.set_time(time as i64);
         }
-        if let Some(observer) = self.observer.as_mut() {
-            commit_opts.set_op_observer(observer);
-        }
         let hash = self.doc.commit_with(commit_opts);
         JsValue::from_str(&hex::encode(&hash.0))
     }
 
     pub fn merge(&mut self, other: &mut Automerge) -> Result<Array, JsValue> {
-        self.ensure_transaction_closed();
-        let options = if let Some(observer) = self.observer.as_mut() {
-            ApplyOptions::default().with_op_observer(observer)
-        } else {
-            ApplyOptions::default()
-        };
-        let heads = self.doc.merge_with(&mut other.doc, options)?;
+        let heads = self.doc.merge(&mut other.doc)?;
         let heads: Array = heads
             .iter()
             .map(|h| JsValue::from_str(&hex::encode(&h.0)))
@@ -366,10 +345,13 @@ impl Automerge {
             } else {
                 self.doc.get(&obj, prop)?
             };
-            match value {
-                Some((Value::Object(_), obj_id)) => Ok(obj_id.to_string().into()),
-                Some((Value::Scalar(value), _)) => Ok(ScalarValue(value).into()),
-                None => Ok(JsValue::undefined()),
+            if let Some((value, id)) = value {
+                match alloc(&value) {
+                    (datatype, js_value) if datatype.is_scalar() => Ok(js_value),
+                    _ => Ok(id.to_string().into()),
+                }
+            } else {
+                Ok(JsValue::undefined())
             }
         } else {
             Ok(JsValue::undefined())
@@ -384,7 +366,6 @@ impl Automerge {
         heads: Option<Array>,
     ) -> Result<JsValue, JsValue> {
         let obj = self.import(obj)?;
-        let result = Array::new();
         let prop = to_prop(prop);
         let heads = get_heads(heads);
         if let Ok(prop) = prop {
@@ -393,18 +374,24 @@ impl Automerge {
             } else {
                 self.doc.get(&obj, prop)?
             };
-            match value {
-                Some((Value::Object(obj_type), obj_id)) => {
-                    result.push(&obj_type.to_string().into());
-                    result.push(&obj_id.to_string().into());
-                    Ok(result.into())
+            if let Some(value) = value {
+                match &value {
+                    (Value::Object(obj_type), obj_id) => {
+                        let result = Array::new();
+                        result.push(&obj_type.to_string().into());
+                        result.push(&obj_id.to_string().into());
+                        Ok(result.into())
+                    }
+                    (Value::Scalar(_), _) => {
+                        let result = Array::new();
+                        let (datatype, value) = alloc(&value.0);
+                        result.push(&datatype.into());
+                        result.push(&value);
+                        Ok(result.into())
+                    }
                 }
-                Some((Value::Scalar(value), _)) => {
-                    result.push(&datatype(&value).into());
-                    result.push(&ScalarValue(value).into());
-                    Ok(result.into())
-                }
-                None => Ok(JsValue::null()),
+            } else {
+                Ok(JsValue::null())
             }
         } else {
             Ok(JsValue::null())
@@ -428,22 +415,15 @@ impl Automerge {
                 self.doc.get_all(&obj, prop)
             }
             .map_err(to_js_err)?;
-            for value in values {
-                match value {
-                    (Value::Object(obj_type), obj_id) => {
-                        let sub = Array::new();
-                        sub.push(&obj_type.to_string().into());
-                        sub.push(&obj_id.to_string().into());
-                        result.push(&sub.into());
-                    }
-                    (Value::Scalar(value), id) => {
-                        let sub = Array::new();
-                        sub.push(&datatype(&value).into());
-                        sub.push(&ScalarValue(value).into());
-                        sub.push(&id.to_string().into());
-                        result.push(&sub.into());
-                    }
+            for (value, id) in values {
+                let sub = Array::new();
+                let (datatype, js_value) = alloc(&value);
+                sub.push(&datatype.into());
+                if value.is_scalar() {
+                    sub.push(&js_value);
                 }
+                sub.push(&id.to_string().into());
+                result.push(&JsValue::from(&sub));
             }
         }
         Ok(result)
@@ -453,84 +433,68 @@ impl Automerge {
     pub fn enable_patches(&mut self, enable: JsValue) -> Result<(), JsValue> {
         let enable = enable
             .as_bool()
-            .ok_or_else(|| to_js_err("expected boolean"))?;
-        if enable {
-            if self.observer.is_none() {
-                self.observer = Some(VecOpObserver::default());
-            }
+            .ok_or_else(|| to_js_err("must pass a bool to enable_patches"))?;
+        self.doc.observer().enable(enable);
+        Ok(())
+    }
+
+    #[wasm_bindgen(js_name = registerDatatype)]
+    pub fn register_datatype(
+        &mut self,
+        datatype: JsValue,
+        function: JsValue,
+    ) -> Result<(), JsValue> {
+        let datatype = Datatype::try_from(datatype)?;
+        if let Ok(function) = function.dyn_into::<Function>() {
+            self.external_types.insert(datatype, function);
         } else {
-            self.observer = None;
+            self.external_types.remove(&datatype);
         }
         Ok(())
     }
 
+    #[wasm_bindgen(js_name = applyPatches)]
+    pub fn apply_patches(
+        &mut self,
+        object: JsValue,
+        meta: JsValue,
+        callback: JsValue,
+    ) -> Result<JsValue, JsValue> {
+        let mut object = object.dyn_into::<Object>()?;
+        let patches = self.doc.observer().take_patches();
+        let callback = callback.dyn_into::<Function>().ok();
+
+        // even if there are no patches we may need to update the meta object
+        // which requires that we update the object too
+        if patches.is_empty() && !meta.is_undefined() {
+            let (obj, datatype, id) = self.unwrap_object(&object)?;
+            object = Object::assign(&Object::new(), &obj);
+            object = self.wrap_object(object, datatype, &id, &meta)?;
+        }
+
+        for p in patches {
+            if let Some(c) = &callback {
+                let before = object.clone();
+                object = self.apply_patch(object, &p, 0, &meta)?;
+                c.call3(&JsValue::undefined(), &p.try_into()?, &before, &object)?;
+            } else {
+                object = self.apply_patch(object, &p, 0, &meta)?;
+            }
+        }
+
+        Ok(object.into())
+    }
+
     #[wasm_bindgen(js_name = popPatches)]
     pub fn pop_patches(&mut self) -> Result<Array, JsValue> {
         // transactions send out observer updates as they occur, not waiting for them to be
         // committed.
         // If we pop the patches then we won't be able to revert them.
-        self.ensure_transaction_closed();
 
-        let patches = self
-            .observer
-            .as_mut()
-            .map_or_else(Vec::new, |o| o.take_patches());
+        let patches = self.doc.observer().take_patches();
         let result = Array::new();
         for p in patches {
-            let patch = Object::new();
-            match p {
-                Patch::Put {
-                    obj,
-                    key,
-                    value,
-                    conflict,
-                } => {
-                    js_set(&patch, "action", "put")?;
-                    js_set(&patch, "obj", obj.to_string())?;
-                    js_set(&patch, "key", key)?;
-                    match value {
-                        (Value::Object(obj_type), obj_id) => {
-                            js_set(&patch, "datatype", obj_type.to_string())?;
-                            js_set(&patch, "value", obj_id.to_string())?;
-                        }
-                        (Value::Scalar(value), _) => {
-                            js_set(&patch, "datatype", datatype(&value))?;
-                            js_set(&patch, "value", ScalarValue(value))?;
-                        }
-                    };
-                    js_set(&patch, "conflict", conflict)?;
-                }
-
-                Patch::Insert { obj, index, value } => {
-                    js_set(&patch, "action", "insert")?;
-                    js_set(&patch, "obj", obj.to_string())?;
-                    js_set(&patch, "key", index as f64)?;
-                    match value {
-                        (Value::Object(obj_type), obj_id) => {
-                            js_set(&patch, "datatype", obj_type.to_string())?;
-                            js_set(&patch, "value", obj_id.to_string())?;
-                        }
-                        (Value::Scalar(value), _) => {
-                            js_set(&patch, "datatype", datatype(&value))?;
-                            js_set(&patch, "value", ScalarValue(value))?;
-                        }
-                    };
-                }
-
-                Patch::Increment { obj, key, value } => {
-                    js_set(&patch, "action", "increment")?;
-                    js_set(&patch, "obj", obj.to_string())?;
-                    js_set(&patch, "key", key)?;
-                    js_set(&patch, "value", value.0)?;
-                }
-
-                Patch::Delete { obj, key } => {
-                    js_set(&patch, "action", "delete")?;
-                    js_set(&patch, "obj", obj.to_string())?;
-                    js_set(&patch, "key", key)?;
-                }
-            }
-            result.push(&patch);
+            result.push(&p.try_into()?);
         }
         Ok(result)
     }
@@ -552,51 +516,31 @@ impl Automerge {
     }
 
     pub fn save(&mut self) -> Uint8Array {
-        self.ensure_transaction_closed();
         Uint8Array::from(self.doc.save().as_slice())
     }
 
     #[wasm_bindgen(js_name = saveIncremental)]
     pub fn save_incremental(&mut self) -> Uint8Array {
-        self.ensure_transaction_closed();
         let bytes = self.doc.save_incremental();
         Uint8Array::from(bytes.as_slice())
     }
 
     #[wasm_bindgen(js_name = loadIncremental)]
     pub fn load_incremental(&mut self, data: Uint8Array) -> Result<f64, JsValue> {
-        self.ensure_transaction_closed();
         let data = data.to_vec();
-        let options = if let Some(observer) = self.observer.as_mut() {
-            ApplyOptions::default().with_op_observer(observer)
-        } else {
-            ApplyOptions::default()
-        };
-        let len = self
-            .doc
-            .load_incremental_with(&data, options)
-            .map_err(to_js_err)?;
+        let len = self.doc.load_incremental(&data).map_err(to_js_err)?;
         Ok(len as f64)
     }
 
     #[wasm_bindgen(js_name = applyChanges)]
     pub fn apply_changes(&mut self, changes: JsValue) -> Result<(), JsValue> {
-        self.ensure_transaction_closed();
         let changes: Vec<_> = JS(changes).try_into()?;
-        let options = if let Some(observer) = self.observer.as_mut() {
-            ApplyOptions::default().with_op_observer(observer)
-        } else {
-            ApplyOptions::default()
-        };
-        self.doc
-            .apply_changes_with(changes, options)
-            .map_err(to_js_err)?;
+        self.doc.apply_changes(changes).map_err(to_js_err)?;
         Ok(())
     }
 
     #[wasm_bindgen(js_name = getChanges)]
     pub fn get_changes(&mut self, have_deps: JsValue) -> Result<Array, JsValue> {
-        self.ensure_transaction_closed();
         let deps: Vec<_> = JS(have_deps).try_into()?;
         let changes = self.doc.get_changes(&deps)?;
         let changes: Array = changes
@@ -608,7 +552,6 @@ impl Automerge {
 
     #[wasm_bindgen(js_name = getChangeByHash)]
     pub fn get_change_by_hash(&mut self, hash: JsValue) -> Result<JsValue, JsValue> {
-        self.ensure_transaction_closed();
         let hash = hash.into_serde().map_err(to_js_err)?;
         let change = self.doc.get_change_by_hash(&hash);
         if let Some(c) = change {
@@ -620,7 +563,6 @@ impl Automerge {
 
     #[wasm_bindgen(js_name = getChangesAdded)]
     pub fn get_changes_added(&mut self, other: &mut Automerge) -> Result<Array, JsValue> {
-        self.ensure_transaction_closed();
         let changes = self.doc.get_changes_added(&mut other.doc);
         let changes: Array = changes
             .iter()
@@ -631,7 +573,6 @@ impl Automerge {
 
     #[wasm_bindgen(js_name = getHeads)]
     pub fn get_heads(&mut self) -> Array {
-        self.ensure_transaction_closed();
         let heads = self.doc.get_heads();
         let heads: Array = heads
             .iter()
@@ -648,7 +589,6 @@ impl Automerge {
 
     #[wasm_bindgen(js_name = getLastLocalChange)]
     pub fn get_last_local_change(&mut self) -> Result<JsValue, JsValue> {
-        self.ensure_transaction_closed();
         if let Some(change) = self.doc.get_last_local_change() {
             Ok(Uint8Array::from(change.raw_bytes()).into())
         } else {
@@ -657,13 +597,11 @@ impl Automerge {
     }
 
     pub fn dump(&mut self) {
-        self.ensure_transaction_closed();
         self.doc.dump()
     }
 
     #[wasm_bindgen(js_name = getMissingDeps)]
     pub fn get_missing_deps(&mut self, heads: Option<Array>) -> Result<Array, JsValue> {
-        self.ensure_transaction_closed();
         let heads = get_heads(heads).unwrap_or_default();
         let deps = self.doc.get_missing_deps(&heads);
         let deps: Array = deps
@@ -679,23 +617,16 @@ impl Automerge {
         state: &mut SyncState,
         message: Uint8Array,
     ) -> Result<(), JsValue> {
-        self.ensure_transaction_closed();
         let message = message.to_vec();
         let message = am::sync::Message::decode(message.as_slice()).map_err(to_js_err)?;
-        let options = if let Some(observer) = self.observer.as_mut() {
-            ApplyOptions::default().with_op_observer(observer)
-        } else {
-            ApplyOptions::default()
-        };
         self.doc
-            .receive_sync_message_with(&mut state.0, message, options)
+            .receive_sync_message(&mut state.0, message)
             .map_err(to_js_err)?;
         Ok(())
     }
 
     #[wasm_bindgen(js_name = generateSyncMessage)]
     pub fn generate_sync_message(&mut self, state: &mut SyncState) -> Result<JsValue, JsValue> {
-        self.ensure_transaction_closed();
         if let Some(message) = self.doc.generate_sync_message(&mut state.0) {
             Ok(Uint8Array::from(message.encode().as_slice()).into())
         } else {
@@ -704,30 +635,24 @@ impl Automerge {
     }
 
     #[wasm_bindgen(js_name = toJS)]
-    pub fn to_js(&self) -> JsValue {
-        map_to_js(&self.doc, &ROOT)
+    pub fn to_js(&self, meta: JsValue) -> Result<JsValue, JsValue> {
+        self.export_object(&ROOT, Datatype::Map, None, &meta)
     }
 
-    pub fn materialize(&self, obj: JsValue, heads: Option<Array>) -> Result<JsValue, JsValue> {
+    pub fn materialize(
+        &mut self,
+        obj: JsValue,
+        heads: Option<Array>,
+        meta: JsValue,
+    ) -> Result<JsValue, JsValue> {
         let obj = self.import(obj).unwrap_or(ROOT);
         let heads = get_heads(heads);
-        if let Some(heads) = heads {
-            match self.doc.object_type(&obj) {
-                Some(am::ObjType::Map) => Ok(map_to_js_at(&self.doc, &obj, heads.as_slice())),
-                Some(am::ObjType::List) => Ok(list_to_js_at(&self.doc, &obj, heads.as_slice())),
-                Some(am::ObjType::Text) => Ok(self.doc.text_at(&obj, heads.as_slice())?.into()),
-                Some(am::ObjType::Table) => Ok(map_to_js_at(&self.doc, &obj, heads.as_slice())),
-                None => Err(to_js_err(format!("invalid obj {}", obj))),
-            }
-        } else {
-            match self.doc.object_type(&obj) {
-                Some(am::ObjType::Map) => Ok(map_to_js(&self.doc, &obj)),
-                Some(am::ObjType::List) => Ok(list_to_js(&self.doc, &obj)),
-                Some(am::ObjType::Text) => Ok(self.doc.text(&obj)?.into()),
-                Some(am::ObjType::Table) => Ok(map_to_js(&self.doc, &obj)),
-                None => Err(to_js_err(format!("invalid obj {}", obj))),
-            }
-        }
+        let obj_type = self
+            .doc
+            .object_type(&obj)
+            .ok_or_else(|| to_js_err(format!("invalid obj {}", obj)))?;
+        let _patches = self.doc.observer().take_patches(); // throw away patches
+        self.export_object(&obj, obj_type.into(), heads.as_ref(), &meta)
     }
 
     fn import(&self, id: JsValue) -> Result<ObjId, JsValue> {
@@ -746,11 +671,11 @@ impl Automerge {
                         self.doc.get(obj, am::Prop::Seq(prop.parse().unwrap()))?
                     };
                     match val {
-                        Some((am::Value::Object(am::ObjType::Map), id)) => {
+                        Some((am::Value::Object(ObjType::Map), id)) => {
                             is_map = true;
                             obj = id;
                         }
-                        Some((am::Value::Object(am::ObjType::Table), id)) => {
+                        Some((am::Value::Object(ObjType::Table), id)) => {
                             is_map = true;
                             obj = id;
                         }
@@ -852,19 +777,17 @@ pub fn init(actor: Option<String>) -> Result<Automerge, JsValue> {
     Automerge::new(actor)
 }
 
-#[wasm_bindgen(js_name = loadDoc)]
+#[wasm_bindgen(js_name = load)]
 pub fn load(data: Uint8Array, actor: Option<String>) -> Result<Automerge, JsValue> {
     let data = data.to_vec();
-    let observer = None;
-    let options = ApplyOptions::<()>::default();
-    let mut automerge = am::AutoCommit::load_with(&data, options).map_err(to_js_err)?;
+    let mut doc = AutoCommit::load(&data).map_err(to_js_err)?;
     if let Some(s) = actor {
         let actor = automerge::ActorId::from(hex::decode(s).map_err(to_js_err)?.to_vec());
-        automerge.set_actor(actor);
+        doc.set_actor(actor);
     }
     Ok(Automerge {
-        doc: automerge,
-        observer,
+        doc,
+        external_types: HashMap::default(),
     })
 }
 

automerge-wasm/src/observer.rs

@@ -0,0 +1,334 @@
+#![allow(dead_code)]
+
+use crate::interop::{alloc, js_set};
+use automerge::{ObjId, OpObserver, Parents, Prop, Value};
+use js_sys::{Array, Object};
+use wasm_bindgen::prelude::*;
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct Observer {
+    enabled: bool,
+    patches: Vec<Patch>,
+}
+
+impl Observer {
+    pub(crate) fn take_patches(&mut self) -> Vec<Patch> {
+        std::mem::take(&mut self.patches)
+    }
+    pub(crate) fn enable(&mut self, enable: bool) {
+        if self.enabled && !enable {
+            self.patches.truncate(0)
+        }
+        self.enabled = enable;
+    }
+
+    fn push(&mut self, patch: Patch) {
+        if let Some(tail) = self.patches.last_mut() {
+            if let Some(p) = tail.merge(patch) {
+                self.patches.push(p)
+            }
+        } else {
+            self.patches.push(patch);
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) enum Patch {
+    PutMap {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        key: String,
+        value: (Value<'static>, ObjId),
+        conflict: bool,
+    },
+    PutSeq {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        index: usize,
+        value: (Value<'static>, ObjId),
+        conflict: bool,
+    },
+    Insert {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        index: usize,
+        values: Vec<(Value<'static>, ObjId)>,
+    },
+    Increment {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        prop: Prop,
+        value: i64,
+    },
+    DeleteMap {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        key: String,
+    },
+    DeleteSeq {
+        obj: ObjId,
+        path: Vec<(ObjId, Prop)>,
+        index: usize,
+        length: usize,
+    },
+}
+
+impl OpObserver for Observer {
+    fn insert(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ObjId,
+        index: usize,
+        tagged_value: (Value<'_>, ObjId),
+    ) {
+        if self.enabled {
+            // probably want to inline the merge/push code here
+            let path = parents.path();
+            let value = tagged_value.0.to_owned();
+            let patch = Patch::Insert {
+                path,
+                obj,
+                index,
+                values: vec![(value, tagged_value.1)],
+            };
+            self.push(patch);
+        }
+    }
+
+    fn put(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ObjId,
+        prop: Prop,
+        tagged_value: (Value<'_>, ObjId),
+        conflict: bool,
+    ) {
+        if self.enabled {
+            let path = parents.path();
+            let value = (tagged_value.0.to_owned(), tagged_value.1);
+            let patch = match prop {
+                Prop::Map(key) => Patch::PutMap {
+                    path,
+                    obj,
+                    key,
+                    value,
+                    conflict,
+                },
+                Prop::Seq(index) => Patch::PutSeq {
+                    path,
+                    obj,
+                    index,
+                    value,
+                    conflict,
+                },
+            };
+            self.patches.push(patch);
+        }
+    }
+
+    fn increment(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ObjId,
+        prop: Prop,
+        tagged_value: (i64, ObjId),
+    ) {
+        if self.enabled {
+            let path = parents.path();
+            let value = tagged_value.0;
+            self.patches.push(Patch::Increment {
+                path,
+                obj,
+                prop,
+                value,
+            })
+        }
+    }
+
+    fn delete(&mut self, mut parents: Parents<'_>, obj: ObjId, prop: Prop) {
+        if self.enabled {
+            let path = parents.path();
+            let patch = match prop {
+                Prop::Map(key) => Patch::DeleteMap { path, obj, key },
+                Prop::Seq(index) => Patch::DeleteSeq {
+                    path,
+                    obj,
+                    index,
+                    length: 1,
+                },
+            };
+            self.patches.push(patch)
+        }
+    }
+
+    fn merge(&mut self, other: &Self) {
+        self.patches.extend_from_slice(other.patches.as_slice())
+    }
+
+    fn branch(&self) -> Self {
+        Observer {
+            patches: vec![],
+            enabled: self.enabled,
+        }
+    }
+}
+
+fn prop_to_js(p: &Prop) -> JsValue {
+    match p {
+        Prop::Map(key) => JsValue::from_str(key),
+        Prop::Seq(index) => JsValue::from_f64(*index as f64),
+    }
+}
+
+fn export_path(path: &[(ObjId, Prop)], end: &Prop) -> Array {
+    let result = Array::new();
+    for p in path {
+        result.push(&prop_to_js(&p.1));
+    }
+    result.push(&prop_to_js(end));
+    result
+}
+
+impl Patch {
+    pub(crate) fn path(&self) -> &[(ObjId, Prop)] {
+        match &self {
+            Self::PutMap { path, .. } => path.as_slice(),
+            Self::PutSeq { path, .. } => path.as_slice(),
+            Self::Increment { path, .. } => path.as_slice(),
+            Self::Insert { path, .. } => path.as_slice(),
+            Self::DeleteMap { path, .. } => path.as_slice(),
+            Self::DeleteSeq { path, .. } => path.as_slice(),
+        }
+    }
+
+    pub(crate) fn obj(&self) -> &ObjId {
+        match &self {
+            Self::PutMap { obj, .. } => obj,
+            Self::PutSeq { obj, .. } => obj,
+            Self::Increment { obj, .. } => obj,
+            Self::Insert { obj, .. } => obj,
+            Self::DeleteMap { obj, .. } => obj,
+            Self::DeleteSeq { obj, .. } => obj,
+        }
+    }
+
+    fn merge(&mut self, other: Patch) -> Option<Patch> {
+        match (self, &other) {
+            (
+                Self::Insert {
+                    obj, index, values, ..
+                },
+                Self::Insert {
+                    obj: o2,
+                    values: v2,
+                    index: i2,
+                    ..
+                },
+            ) if obj == o2 && *index + values.len() == *i2 => {
+                // TODO - there's a way to do this without the clone im sure
+                values.extend_from_slice(v2.as_slice());
+                //web_sys::console::log_2(&format!("NEW VAL {}: ", tmpi).into(), &new_value);
+                None
+            }
+            _ => Some(other),
+        }
+    }
+}
+
+impl TryFrom<Patch> for JsValue {
+    type Error = JsValue;
+
+    fn try_from(p: Patch) -> Result<Self, Self::Error> {
+        let result = Object::new();
+        match p {
+            Patch::PutMap {
+                path,
+                key,
+                value,
+                conflict,
+                ..
+            } => {
+                js_set(&result, "action", "put")?;
+                js_set(
+                    &result,
+                    "path",
+                    export_path(path.as_slice(), &Prop::Map(key)),
+                )?;
+                js_set(&result, "value", alloc(&value.0).1)?;
+                js_set(&result, "conflict", &JsValue::from_bool(conflict))?;
+                Ok(result.into())
+            }
+            Patch::PutSeq {
+                path,
+                index,
+                value,
+                conflict,
+                ..
+            } => {
+                js_set(&result, "action", "put")?;
+                js_set(
+                    &result,
+                    "path",
+                    export_path(path.as_slice(), &Prop::Seq(index)),
+                )?;
+                js_set(&result, "value", alloc(&value.0).1)?;
+                js_set(&result, "conflict", &JsValue::from_bool(conflict))?;
+                Ok(result.into())
+            }
+            Patch::Insert {
+                path,
+                index,
+                values,
+                ..
+            } => {
+                js_set(&result, "action", "splice")?;
+                js_set(
+                    &result,
+                    "path",
+                    export_path(path.as_slice(), &Prop::Seq(index)),
+                )?;
+                js_set(
+                    &result,
+                    "values",
+                    values.iter().map(|v| alloc(&v.0).1).collect::<Array>(),
+                )?;
+                Ok(result.into())
+            }
+            Patch::Increment {
+                path, prop, value, ..
+            } => {
+                js_set(&result, "action", "inc")?;
+                js_set(&result, "path", export_path(path.as_slice(), &prop))?;
+                js_set(&result, "value", &JsValue::from_f64(value as f64))?;
+                Ok(result.into())
+            }
+            Patch::DeleteMap { path, key, .. } => {
+                js_set(&result, "action", "del")?;
+                js_set(
+                    &result,
+                    "path",
+                    export_path(path.as_slice(), &Prop::Map(key)),
+                )?;
+                Ok(result.into())
+            }
+            Patch::DeleteSeq {
+                path,
+                index,
+                length,
+                ..
+            } => {
+                js_set(&result, "action", "del")?;
+                js_set(
+                    &result,
+                    "path",
+                    export_path(path.as_slice(), &Prop::Seq(index)),
+                )?;
+                if length > 1 {
+                    js_set(&result, "length", length)?;
+                }
+                Ok(result.into())
+            }
+        }
+    }
+}

automerge-wasm/src/value.rs

@@ -1,40 +1,151 @@
-use std::borrow::Cow;
-
-use automerge as am;
-use js_sys::Uint8Array;
+use crate::to_js_err;
+use automerge::{ObjType, ScalarValue, Value};
 use wasm_bindgen::prelude::*;
 
-#[derive(Debug)]
-pub struct ScalarValue<'a>(pub(crate) Cow<'a, am::ScalarValue>);
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub(crate) enum Datatype {
+    Map,
+    Table,
+    List,
+    Text,
+    Bytes,
+    Str,
+    Int,
+    Uint,
+    F64,
+    Counter,
+    Timestamp,
+    Boolean,
+    Null,
+    Unknown(u8),
+}
 
-impl<'a> From<ScalarValue<'a>> for JsValue {
-    fn from(val: ScalarValue<'a>) -> Self {
-        match &*val.0 {
-            am::ScalarValue::Bytes(v) => Uint8Array::from(v.as_slice()).into(),
-            am::ScalarValue::Str(v) => v.to_string().into(),
-            am::ScalarValue::Int(v) => (*v as f64).into(),
-            am::ScalarValue::Uint(v) => (*v as f64).into(),
-            am::ScalarValue::F64(v) => (*v).into(),
-            am::ScalarValue::Counter(v) => (f64::from(v)).into(),
-            am::ScalarValue::Timestamp(v) => js_sys::Date::new(&(*v as f64).into()).into(),
-            am::ScalarValue::Boolean(v) => (*v).into(),
-            am::ScalarValue::Null => JsValue::null(),
-            am::ScalarValue::Unknown { bytes, .. } => Uint8Array::from(bytes.as_slice()).into(),
+impl Datatype {
+    pub(crate) fn is_sequence(&self) -> bool {
+        matches!(self, Self::List | Self::Text)
+    }
+
+    pub(crate) fn is_scalar(&self) -> bool {
+        !matches!(self, Self::Map | Self::Table | Self::List | Self::Text)
+    }
+}
+
+impl From<&ObjType> for Datatype {
+    fn from(o: &ObjType) -> Self {
+        (*o).into()
+    }
+}
+
+impl From<ObjType> for Datatype {
+    fn from(o: ObjType) -> Self {
+        match o {
+            ObjType::Map => Self::Map,
+            ObjType::List => Self::List,
+            ObjType::Table => Self::Table,
+            ObjType::Text => Self::Text,
         }
     }
 }
 
-pub(crate) fn datatype(s: &am::ScalarValue) -> String {
-    match s {
-        am::ScalarValue::Bytes(_) => "bytes".into(),
-        am::ScalarValue::Str(_) => "str".into(),
-        am::ScalarValue::Int(_) => "int".into(),
-        am::ScalarValue::Uint(_) => "uint".into(),
-        am::ScalarValue::F64(_) => "f64".into(),
-        am::ScalarValue::Counter(_) => "counter".into(),
-        am::ScalarValue::Timestamp(_) => "timestamp".into(),
-        am::ScalarValue::Boolean(_) => "boolean".into(),
-        am::ScalarValue::Null => "null".into(),
-        am::ScalarValue::Unknown { type_code, .. } => format!("unknown{}", type_code),
+impl std::fmt::Display for Datatype {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+        write!(f, "{}", String::from(self.clone()))
+    }
+}
+
+impl From<&ScalarValue> for Datatype {
+    fn from(s: &ScalarValue) -> Self {
+        match s {
+            ScalarValue::Bytes(_) => Self::Bytes,
+            ScalarValue::Str(_) => Self::Str,
+            ScalarValue::Int(_) => Self::Int,
+            ScalarValue::Uint(_) => Self::Uint,
+            ScalarValue::F64(_) => Self::F64,
+            ScalarValue::Counter(_) => Self::Counter,
+            ScalarValue::Timestamp(_) => Self::Timestamp,
+            ScalarValue::Boolean(_) => Self::Boolean,
+            ScalarValue::Null => Self::Null,
+            ScalarValue::Unknown { type_code, .. } => Self::Unknown(*type_code),
+        }
+    }
+}
+
+impl From<&Value<'_>> for Datatype {
+    fn from(v: &Value<'_>) -> Self {
+        match v {
+            Value::Object(o) => o.into(),
+            Value::Scalar(s) => s.as_ref().into(),
+            /*
+                            ScalarValue::Bytes(_) => Self::Bytes,
+                            ScalarValue::Str(_) => Self::Str,
+                            ScalarValue::Int(_) => Self::Int,
+                            ScalarValue::Uint(_) => Self::Uint,
+                            ScalarValue::F64(_) => Self::F64,
+                            ScalarValue::Counter(_) => Self::Counter,
+                            ScalarValue::Timestamp(_) => Self::Timestamp,
+                            ScalarValue::Boolean(_) => Self::Boolean,
+                            ScalarValue::Null => Self::Null,
+                            ScalarValue::Unknown { type_code, .. } => Self::Unknown(*type_code),
+            */
+        }
+    }
+}
+
+impl From<Datatype> for String {
+    fn from(d: Datatype) -> Self {
+        match d {
+            Datatype::Map => "map".into(),
+            Datatype::Table => "table".into(),
+            Datatype::List => "list".into(),
+            Datatype::Text => "text".into(),
+            Datatype::Bytes => "bytes".into(),
+            Datatype::Str => "str".into(),
+            Datatype::Int => "int".into(),
+            Datatype::Uint => "uint".into(),
+            Datatype::F64 => "f64".into(),
+            Datatype::Counter => "counter".into(),
+            Datatype::Timestamp => "timestamp".into(),
+            Datatype::Boolean => "boolean".into(),
+            Datatype::Null => "null".into(),
+            Datatype::Unknown(type_code) => format!("unknown{}", type_code),
+        }
+    }
+}
+
+impl TryFrom<JsValue> for Datatype {
+    type Error = JsValue;
+
+    fn try_from(datatype: JsValue) -> Result<Self, Self::Error> {
+        let datatype = datatype
+            .as_string()
+            .ok_or_else(|| to_js_err("datatype is not a string"))?;
+        match datatype.as_str() {
+            "map" => Ok(Datatype::Map),
+            "table" => Ok(Datatype::Table),
+            "list" => Ok(Datatype::List),
+            "text" => Ok(Datatype::Text),
+            "bytes" => Ok(Datatype::Bytes),
+            "str" => Ok(Datatype::Str),
+            "int" => Ok(Datatype::Int),
+            "uint" => Ok(Datatype::Uint),
+            "f64" => Ok(Datatype::F64),
+            "counter" => Ok(Datatype::Counter),
+            "timestamp" => Ok(Datatype::Timestamp),
+            "boolean" => Ok(Datatype::Boolean),
+            "null" => Ok(Datatype::Null),
+            d => {
+                if d.starts_with("unknown") {
+                    todo!() // handle "unknown{}",
+                } else {
+                    Err(to_js_err(format!("unknown datatype {}", d)))
+                }
+            }
+        }
+    }
+}
+
+impl From<Datatype> for JsValue {
+    fn from(d: Datatype) -> Self {
+        String::from(d).into()
     }
 }

automerge-wasm/test/apply.ts

@@ -0,0 +1,194 @@
+
+import { describe, it } from 'mocha';
+//@ts-ignore
+import assert from 'assert'
+//@ts-ignore
+import init, { create, load } from '..'
+
+export const OBJECT_ID  = Symbol.for('_am_objectId')     // object containing metadata about current 
+
+// sample classes for testing
+class Counter {
+  value: number;
+  constructor(n: number) {
+    this.value = n
+  }
+}
+
+class Wrapper {
+  value: any;
+  constructor(n: any) {
+    this.value = n
+  }
+}
+
+describe('Automerge', () => {
+  describe('Patch Apply', () => {
+    it('apply nested sets on maps', () => {
+      let start : any = { hello: { mellow: { yellow: "world", x: 1 }, y : 2 } }
+      let doc1 = create()
+      doc1.putObject("/", "hello", start.hello);
+      let mat = doc1.materialize("/")
+      let doc2 = create()
+      doc2.enablePatches(true)
+      doc2.merge(doc1)
+
+      let base = doc2.applyPatches({})
+      assert.deepEqual(mat, start)
+      assert.deepEqual(base, start)
+
+      doc2.delete("/hello/mellow", "yellow");
+      delete start.hello.mellow.yellow;
+      base = doc2.applyPatches(base)
+      mat = doc2.materialize("/")
+
+      assert.deepEqual(mat, start)
+      assert.deepEqual(base, start)
+    })
+
+    it('apply patches on lists', () => {
+      //let start = { list: [1,2,3,4,5,6] }
+      let start = { list: [1,2,3,4] }
+      let doc1 = create()
+      doc1.putObject("/", "list", start.list);
+      let mat = doc1.materialize("/")
+      let doc2 = create()
+      doc2.enablePatches(true)
+      doc2.merge(doc1)
+      mat = doc1.materialize("/")
+      let base = doc2.applyPatches({})
+      assert.deepEqual(mat, start)
+      assert.deepEqual(base, start)
+
+      doc2.delete("/list", 3);
+      start.list.splice(3,1)
+      base = doc2.applyPatches(base)
+
+      assert.deepEqual(base, start)
+    })
+
+    it('apply patches on lists of lists of lists', () => {
+      let start = { list:
+        [
+          [
+            [ 1, 2, 3, 4, 5, 6],
+            [ 7, 8, 9,10,11,12],
+          ],
+          [
+            [ 7, 8, 9,10,11,12],
+            [ 1, 2, 3, 4, 5, 6],
+          ]
+        ]
+      }
+      let doc1 = create()
+      doc1.enablePatches(true)
+      doc1.putObject("/", "list", start.list);
+      let base = doc1.applyPatches({})
+      let mat = doc1.clone().materialize("/")
+      assert.deepEqual(mat, start)
+      assert.deepEqual(base, start)
+
+      doc1.delete("/list/0/1", 3)
+      start.list[0][1].splice(3,1)
+
+      doc1.delete("/list/0", 0)
+      start.list[0].splice(0,1)
+
+      mat = doc1.clone().materialize("/")
+      base = doc1.applyPatches(base)
+      assert.deepEqual(mat, start)
+      assert.deepEqual(base, start)
+    })
+
+    it('large inserts should make one splice patch', () => {
+      let doc1 = create()
+      doc1.enablePatches(true)
+      doc1.putObject("/", "list", "abc");
+      let patches = doc1.popPatches()
+      assert.deepEqual( patches, [
+        { action: 'put', conflict: false, path: [ 'list' ], value: [] },
+        { action: 'splice', path: [ 'list', 0 ], values: [ 'a', 'b', 'c' ] }])
+    })
+
+    it('it should allow registering type wrappers', () => {
+      let doc1 = create()
+      doc1.enablePatches(true)
+      //@ts-ignore
+      doc1.registerDatatype("counter", (n: any) => new Counter(n))
+      let doc2 = doc1.fork()
+      doc1.put("/", "n", 10, "counter")
+      doc1.put("/", "m", 10, "int")
+
+      let mat = doc1.materialize("/")
+      assert.deepEqual( mat, { n: new Counter(10), m: 10 } )
+
+      doc2.merge(doc1)
+      let apply = doc2.applyPatches({})
+      assert.deepEqual( apply, { n: new Counter(10), m: 10 } )
+
+      doc1.increment("/","n", 5)
+      mat = doc1.materialize("/")
+      assert.deepEqual( mat, { n: new Counter(15), m: 10 } )
+
+      doc2.merge(doc1)
+      apply = doc2.applyPatches(apply)
+      assert.deepEqual( apply, { n: new Counter(15), m: 10 } )
+    })
+
+    it('text can be managed as an array or a string', () => {
+      let doc1 = create("aaaa")
+      doc1.enablePatches(true)
+
+      doc1.putObject("/", "notes", "hello world")
+
+      let mat = doc1.materialize("/")
+
+      assert.deepEqual( mat, { notes: "hello world".split("") } )
+
+      let doc2 = create()
+      doc2.enablePatches(true)
+      //@ts-ignore
+      doc2.registerDatatype("text", (n: any[]) => new String(n.join("")))
+      let apply = doc2.applyPatches({} as any)
+
+      doc2.merge(doc1);
+      apply = doc2.applyPatches(apply)
+      assert.deepEqual(apply[OBJECT_ID], "_root")
+      assert.deepEqual(apply.notes[OBJECT_ID], "1@aaaa")
+      assert.deepEqual( apply, { notes: new String("hello world") } )
+
+      doc2.splice("/notes", 6, 5, "everyone");
+      apply = doc2.applyPatches(apply)
+      assert.deepEqual( apply, { notes: new String("hello everyone") } )
+
+      mat = doc2.materialize("/")
+      //@ts-ignore
+      assert.deepEqual(mat[OBJECT_ID], "_root")
+      //@ts-ignore
+      assert.deepEqual(mat.notes[OBJECT_ID], "1@aaaa")
+      assert.deepEqual( mat, { notes: new String("hello everyone") } )
+    })
+
+    it.skip('it can patch quickly', () => {
+      console.time("init")
+      let doc1 = create()
+      doc1.enablePatches(true)
+      doc1.putObject("/", "notes", "");
+      let mat = doc1.materialize("/")
+      let doc2 = doc1.fork()
+      let testData = new Array( 100000 ).join("x")
+      console.timeEnd("init")
+      console.time("splice")
+      doc2.splice("/notes", 0, 0, testData);
+      console.timeEnd("splice")
+      console.time("merge")
+      doc1.merge(doc2)
+      console.timeEnd("merge")
+      console.time("patch")
+      mat = doc1.applyPatches(mat)
+      console.timeEnd("patch")
+    })
+  })
+})
+
+// TODO: squash puts & deletes

automerge-wasm/test/readme.ts

@@ -1,7 +1,7 @@
 import { describe, it } from 'mocha';
 import * as assert from 'assert'
 //@ts-ignore
-import { init, create, load } from '..'
+import { create, load } from '..'
 
 describe('Automerge', () => {
   describe('Readme Examples', () => {
@@ -10,11 +10,9 @@ describe('Automerge', () => {
       doc.free()
     })
     it('Using the Library and Creating a Document (2)', (done) => {
-      init().then((_:any) => {
-        const doc = create()
-        doc.free()
-        done()
-      })
+      const doc = create()
+      doc.free()
+      done()
     })
     it('Automerge Scalar Types (1)', () => {
       const doc = create()

automerge-wasm/test/test.ts

@@ -3,7 +3,7 @@ import { describe, it } from 'mocha';
 import assert from 'assert'
 //@ts-ignore
 import { BloomFilter } from './helpers/sync'
-import { init, create, load, SyncState, Automerge, encodeChange, decodeChange, initSyncState, decodeSyncMessage, decodeSyncState, encodeSyncState, encodeSyncMessage } from '..'
+import { create, load, SyncState, Automerge, encodeChange, decodeChange, initSyncState, decodeSyncMessage, decodeSyncState, encodeSyncState, encodeSyncMessage } from '..'
 import { DecodedSyncMessage, Hash } from '..';
 
 function sync(a: Automerge, b: Automerge, aSyncState = initSyncState(), bSyncState = initSyncState()) {
@@ -28,9 +28,6 @@ function sync(a: Automerge, b: Automerge, aSyncState = initSyncState(), bSyncSta
 
 describe('Automerge', () => {
   describe('basics', () => {
-    it('default import init() should return a promise', () => {
-      assert(init() instanceof Promise)
-    })
 
     it('should create, clone and free', () => {
       const doc1 = create()
@@ -400,6 +397,8 @@ describe('Automerge', () => {
 
     it('recursive sets are possible', () => {
       const doc = create("aaaa")
+      //@ts-ignore
+      doc.registerDatatype("text", (n: any[]) => new String(n.join("")))
       const l1 = doc.putObject("_root", "list", [{ foo: "bar" }, [1, 2, 3]])
       const l2 = doc.insertObject(l1, 0, { zip: ["a", "b"] })
       const l3 = doc.putObject("_root", "info1", "hello world") // 'text' object
@@ -407,13 +406,13 @@ describe('Automerge', () => {
       const l4 = doc.putObject("_root", "info3", "hello world")
       assert.deepEqual(doc.materialize(), {
         "list": [{ zip: ["a", "b"] }, { foo: "bar" }, [1, 2, 3]],
-        "info1": "hello world",
+        "info1": new String("hello world"),
         "info2": "hello world",
-        "info3": "hello world",
+        "info3": new String("hello world"),
       })
       assert.deepEqual(doc.materialize(l2), { zip: ["a", "b"] })
       assert.deepEqual(doc.materialize(l1), [{ zip: ["a", "b"] }, { foo: "bar" }, [1, 2, 3]])
-      assert.deepEqual(doc.materialize(l4), "hello world")
+      assert.deepEqual(doc.materialize(l4), new String("hello world"))
       doc.free()
     })
 
@@ -506,7 +505,7 @@ describe('Automerge', () => {
       doc2.enablePatches(true)
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'hello', value: 'world', datatype: 'str', conflict: false }
+        { action: 'put', path: ['hello'], value: 'world', conflict: false }
       ])
       doc1.free()
       doc2.free()
@@ -518,9 +517,9 @@ describe('Automerge', () => {
       doc2.enablePatches(true)
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'birds', value: '1@aaaa', datatype: 'map', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 'friday', value: '2@aaaa', datatype: 'map', conflict: false },
-        { action: 'put', obj: '2@aaaa', key: 'robins', value: 3, datatype: 'int', conflict: false }
+        { action: 'put', path: [ 'birds' ], value: {}, conflict: false },
+        { action: 'put', path: [ 'birds', 'friday' ], value: {}, conflict: false },
+        { action: 'put', path: [ 'birds', 'friday', 'robins' ], value: 3, conflict: false},
       ])
       doc1.free()
       doc2.free()
@@ -534,8 +533,8 @@ describe('Automerge', () => {
       doc1.delete('_root', 'favouriteBird')
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'favouriteBird', value: 'Robin', datatype: 'str', conflict: false },
-        { action: 'delete', obj: '_root', key: 'favouriteBird' }
+        { action: 'put', path: [ 'favouriteBird' ], value: 'Robin', conflict: false },
+        { action: 'del', path: [ 'favouriteBird' ] }
       ])
       doc1.free()
       doc2.free()
@@ -547,9 +546,8 @@ describe('Automerge', () => {
       doc2.enablePatches(true)
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'birds', value: '1@aaaa', datatype: 'list', conflict: false },
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'Goldfinch', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 1, value: 'Chaffinch', datatype: 'str' }
+        { action: 'put', path: [ 'birds' ], value: [], conflict: false },
+        { action: 'splice', path: [ 'birds', 0 ], values: ['Goldfinch', 'Chaffinch'] },
       ])
       doc1.free()
       doc2.free()
@@ -563,9 +561,9 @@ describe('Automerge', () => {
       doc2.enablePatches(true)
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'insert', obj: '1@aaaa', key: 0, value: '2@aaaa', datatype: 'map' },
-        { action: 'put', obj: '2@aaaa', key: 'species', value: 'Goldfinch', datatype: 'str', conflict: false },
-        { action: 'put', obj: '2@aaaa', key: 'count', value: 3, datatype: 'int', conflict: false }
+        { action: 'splice', path: [ 'birds', 0 ], values: [{}] },
+        { action: 'put', path: [ 'birds', 0, 'species' ], value: 'Goldfinch', conflict: false },
+        { action: 'put', path: [ 'birds', 0, 'count', ], value: 3, conflict: false }
       ])
       doc1.free()
       doc2.free()
@@ -582,8 +580,8 @@ describe('Automerge', () => {
       assert.deepEqual(doc1.getWithType('1@aaaa', 0), ['str', 'Chaffinch'])
       assert.deepEqual(doc1.getWithType('1@aaaa', 1), ['str', 'Greenfinch'])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'delete', obj: '1@aaaa', key: 0 },
-        { action: 'insert', obj: '1@aaaa', key: 1, value: 'Greenfinch', datatype: 'str' }
+        { action: 'del', path: ['birds', 0] },
+        { action: 'splice', path: ['birds', 1], values: ['Greenfinch'] }
       ])
       doc1.free()
       doc2.free()
@@ -608,16 +606,11 @@ describe('Automerge', () => {
       assert.deepEqual([0, 1, 2, 3].map(i => (doc3.getWithType('1@aaaa', i) || [])[1]), ['a', 'b', 'c', 'd'])
       assert.deepEqual([0, 1, 2, 3].map(i => (doc4.getWithType('1@aaaa', i) || [])[1]), ['a', 'b', 'c', 'd'])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'c', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 1, value: 'd', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'a', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 1, value: 'b', datatype: 'str' }
+        { action: 'splice', path: ['values', 0], values:['c','d'] },
+        { action: 'splice', path: ['values', 0], values:['a','b'] },
       ])
       assert.deepEqual(doc4.popPatches(), [
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'a', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 1, value: 'b', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 2, value: 'c', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 3, value: 'd', datatype: 'str' }
+        { action: 'splice', path: ['values',0], values:['a','b','c','d'] },
       ])
       doc1.free(); doc2.free(); doc3.free(); doc4.free()
     })
@@ -641,16 +634,11 @@ describe('Automerge', () => {
       assert.deepEqual([0, 1, 2, 3, 4, 5].map(i => (doc3.getWithType('1@aaaa', i) || [])[1]), ['a', 'b', 'c', 'd', 'e', 'f'])
       assert.deepEqual([0, 1, 2, 3, 4, 5].map(i => (doc4.getWithType('1@aaaa', i) || [])[1]), ['a', 'b', 'c', 'd', 'e', 'f'])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'insert', obj: '1@aaaa', key: 2, value: 'e', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 3, value: 'f', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 2, value: 'c', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 3, value: 'd', datatype: 'str' }
+        { action: 'splice', path: ['values', 2], values: ['e','f'] },
+        { action: 'splice', path: ['values', 2], values: ['c','d'] },
       ])
       assert.deepEqual(doc4.popPatches(), [
-        { action: 'insert', obj: '1@aaaa', key: 2, value: 'c', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 3, value: 'd', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 4, value: 'e', datatype: 'str' },
-        { action: 'insert', obj: '1@aaaa', key: 5, value: 'f', datatype: 'str' }
+        { action: 'splice', path: ['values', 2], values: ['c','d','e','f'] },
       ])
       doc1.free(); doc2.free(); doc3.free(); doc4.free()
     })
@@ -669,12 +657,12 @@ describe('Automerge', () => {
       assert.deepEqual(doc4.getWithType('_root', 'bird'), ['str', 'Goldfinch'])
       assert.deepEqual(doc4.getAll('_root', 'bird'), [['str', 'Greenfinch', '1@aaaa'], ['str', 'Goldfinch', '1@bbbb']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Greenfinch', datatype: 'str', conflict: false },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Greenfinch', conflict: false },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true },
       ])
       assert.deepEqual(doc4.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: false },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: false },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true },
       ])
       doc1.free(); doc2.free(); doc3.free(); doc4.free()
     })
@@ -704,16 +692,16 @@ describe('Automerge', () => {
         ['str', 'Greenfinch', '1@aaaa'], ['str', 'Chaffinch', '1@bbbb'], ['str', 'Goldfinch', '1@cccc']
       ])
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Chaffinch', datatype: 'str', conflict: true },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Chaffinch', conflict: true },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true }
       ])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true }
       ])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: true }
       ])
       doc1.free(); doc2.free(); doc3.free()
     })
@@ -730,9 +718,9 @@ describe('Automerge', () => {
       doc3.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc3.getAll('_root', 'bird'), [['str', 'Goldfinch', '2@aaaa']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Greenfinch', datatype: 'str', conflict: false },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Chaffinch', datatype: 'str', conflict: true },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: false }
+        { action: 'put', path: ['bird'], value: 'Greenfinch', conflict: false },
+        { action: 'put', path: ['bird'], value: 'Chaffinch', conflict: true },
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: false }
       ])
       doc1.free(); doc2.free(); doc3.free()
     })
@@ -753,10 +741,10 @@ describe('Automerge', () => {
       assert.deepEqual(doc2.getWithType('_root', 'bird'), ['str', 'Goldfinch'])
       assert.deepEqual(doc2.getAll('_root', 'bird'), [['str', 'Goldfinch', '2@aaaa']])
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: false }
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: false }
       ])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Goldfinch', datatype: 'str', conflict: false }
+        { action: 'put', path: ['bird'], value: 'Goldfinch', conflict: false }
       ])
       doc1.free(); doc2.free()
     })
@@ -780,12 +768,12 @@ describe('Automerge', () => {
       assert.deepEqual(doc4.getWithType('1@aaaa', 0), ['str', 'Redwing'])
       assert.deepEqual(doc4.getAll('1@aaaa', 0), [['str', 'Song Thrush', '4@aaaa'], ['str', 'Redwing', '4@bbbb']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Song Thrush', datatype: 'str', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Redwing', datatype: 'str', conflict: true }
+        { action: 'put', path: ['birds',0], value: 'Song Thrush', conflict: false },
+        { action: 'put', path: ['birds',0], value: 'Redwing', conflict: true }
       ])
       assert.deepEqual(doc4.popPatches(), [
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Redwing', datatype: 'str', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Redwing', datatype: 'str', conflict: true }
+        { action: 'put', path: ['birds',0], value: 'Redwing', conflict: false },
+        { action: 'put', path: ['birds',0], value: 'Redwing', conflict: true }
       ])
       doc1.free(); doc2.free(); doc3.free(); doc4.free()
     })
@@ -811,16 +799,16 @@ describe('Automerge', () => {
       assert.deepEqual(doc4.getAll('1@aaaa', 0), [['str', 'Ring-necked parakeet', '5@bbbb']])
       assert.deepEqual(doc4.getAll('1@aaaa', 2), [['str', 'Song Thrush', '6@aaaa'], ['str', 'Redwing', '6@bbbb']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'delete', obj: '1@aaaa', key: 0 },
-        { action: 'put', obj: '1@aaaa', key: 1, value: 'Song Thrush', datatype: 'str', conflict: false },
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'Ring-necked parakeet', datatype: 'str' },
-        { action: 'put', obj: '1@aaaa', key: 2, value: 'Redwing', datatype: 'str', conflict: true }
+        { action: 'del', path: ['birds',0], },
+        { action: 'put', path: ['birds',1], value: 'Song Thrush', conflict: false },
+        { action: 'splice', path: ['birds',0], values: ['Ring-necked parakeet'] },
+        { action: 'put', path: ['birds',2], value: 'Redwing', conflict: true }
       ])
       assert.deepEqual(doc4.popPatches(), [
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Ring-necked parakeet', datatype: 'str', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 2, value: 'Redwing', datatype: 'str', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 0, value: 'Ring-necked parakeet', datatype: 'str', conflict: false },
-        { action: 'put', obj: '1@aaaa', key: 2, value: 'Redwing', datatype: 'str', conflict: true }
+        { action: 'put', path: ['birds',0], value: 'Ring-necked parakeet', conflict: false },
+        { action: 'put', path: ['birds',2], value: 'Redwing', conflict: false },
+        { action: 'put', path: ['birds',0], value: 'Ring-necked parakeet', conflict: false },
+        { action: 'put', path: ['birds',2], value: 'Redwing', conflict: true }
       ])
       doc1.free(); doc2.free(); doc3.free(); doc4.free()
     })
@@ -837,14 +825,14 @@ describe('Automerge', () => {
       doc3.loadIncremental(change2)
       assert.deepEqual(doc3.getAll('_root', 'bird'), [['str', 'Robin', '1@aaaa'], ['str', 'Wren', '1@bbbb']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Robin', datatype: 'str', conflict: false },
-        { action: 'put', obj: '_root', key: 'bird', value: 'Wren', datatype: 'str', conflict: true }
+        { action: 'put', path: ['bird'], value: 'Robin', conflict: false },
+        { action: 'put', path: ['bird'], value: 'Wren', conflict: true }
       ])
       doc3.loadIncremental(change3)
       assert.deepEqual(doc3.getWithType('_root', 'bird'), ['str', 'Robin'])
       assert.deepEqual(doc3.getAll('_root', 'bird'), [['str', 'Robin', '1@aaaa']])
       assert.deepEqual(doc3.popPatches(), [
-        { action: 'put', obj: '_root', key: 'bird', value: 'Robin', datatype: 'str', conflict: false }
+        { action: 'put', path: ['bird'], value: 'Robin', conflict: false }
       ])
       doc1.free(); doc2.free(); doc3.free()
     })
@@ -860,26 +848,25 @@ describe('Automerge', () => {
       doc2.loadIncremental(change1)
       assert.deepEqual(doc1.getAll('_root', 'birds'), [['list', '1@aaaa'], ['map', '1@bbbb']])
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'birds', value: '1@bbbb', datatype: 'map', conflict: true },
-        { action: 'put', obj: '1@bbbb', key: 'Sparrowhawk', value: 1, datatype: 'int', conflict: false }
+        { action: 'put', path: ['birds'], value: {}, conflict: true },
+        { action: 'put', path: ['birds', 'Sparrowhawk'], value: 1, conflict: false }
       ])
       assert.deepEqual(doc2.getAll('_root', 'birds'), [['list', '1@aaaa'], ['map', '1@bbbb']])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'birds', value: '1@bbbb', datatype: 'map', conflict: true },
-        { action: 'insert', obj: '1@aaaa', key: 0, value: 'Parakeet', datatype: 'str' }
+        { action: 'put', path: ['birds'], value: {}, conflict: true },
+        { action: 'splice', path: ['birds',0], values: ['Parakeet'] }
       ])
       doc1.free(); doc2.free()
     })
 
     it('should support date objects', () => {
-      // FIXME: either use Date objects or use numbers consistently
       const doc1 = create('aaaa'), doc2 = create('bbbb'), now = new Date()
-      doc1.put('_root', 'createdAt', now.getTime(), 'timestamp')
+      doc1.put('_root', 'createdAt', now)
       doc2.enablePatches(true)
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.getWithType('_root', 'createdAt'), ['timestamp', now])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'createdAt', value: now, datatype: 'timestamp', conflict: false }
+        { action: 'put', path: ['createdAt'], value: now, conflict: false }
       ])
       doc1.free(); doc2.free()
     })
@@ -894,11 +881,11 @@ describe('Automerge', () => {
       const list = doc1.putObject('_root', 'list', [])
 
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'key1', value: 1, datatype: 'int', conflict: false },
-        { action: 'put', obj: '_root', key: 'key1', value: 2, datatype: 'int', conflict: false },
-        { action: 'put', obj: '_root', key: 'key2', value: 3, datatype: 'int', conflict: false },
-        { action: 'put', obj: '_root', key: 'map', value: map, datatype: 'map', conflict: false },
-        { action: 'put', obj: '_root', key: 'list', value: list, datatype: 'list', conflict: false },
+        { action: 'put', path: ['key1'], value: 1, conflict: false },
+        { action: 'put', path: ['key1'], value: 2, conflict: false },
+        { action: 'put', path: ['key2'], value: 3, conflict: false },
+        { action: 'put', path: ['map'], value: {}, conflict: false },
+        { action: 'put', path: ['list'], value: [], conflict: false },
       ])
       doc1.free()
     })
@@ -914,12 +901,12 @@ describe('Automerge', () => {
       const list2 = doc1.insertObject(list, 2, [])
 
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'list', value: list, datatype: 'list', conflict: false },
-        { action: 'insert', obj: list, key: 0, value: 1, datatype: 'int' },
-        { action: 'insert', obj: list, key: 0, value: 2, datatype: 'int' },
-        { action: 'insert', obj: list, key: 2, value: 3, datatype: 'int' },
-        { action: 'insert', obj: list, key: 2, value: map, datatype: 'map' },
-        { action: 'insert', obj: list, key: 2, value: list2, datatype: 'list' },
+        { action: 'put', path: ['list'], value: [], conflict: false },
+        { action: 'splice', path: ['list', 0], values: [1]  },
+        { action: 'splice', path: ['list', 0], values: [2] },
+        { action: 'splice', path: ['list', 2], values: [3] },
+        { action: 'splice', path: ['list', 2], values: [{}] },
+        { action: 'splice', path: ['list', 2], values: [[]] },
       ])
       doc1.free()
     })
@@ -933,10 +920,8 @@ describe('Automerge', () => {
       const list2 = doc1.pushObject(list, [])
 
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'list', value: list, datatype: 'list', conflict: false },
-        { action: 'insert', obj: list, key: 0, value: 1, datatype: 'int' },
-        { action: 'insert', obj: list, key: 1, value: map, datatype: 'map' },
-        { action: 'insert', obj: list, key: 2, value: list2, datatype: 'list' },
+        { action: 'put', path: ['list'], value: [], conflict: false },
+        { action: 'splice', path: ['list',0], values: [1,{},[]] },
       ])
       doc1.free()
     })
@@ -949,13 +934,10 @@ describe('Automerge', () => {
       doc1.splice(list, 1, 2)
 
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'list', value: list, datatype: 'list', conflict: false },
-        { action: 'insert', obj: list, key: 0, value: 1, datatype: 'int' },
-        { action: 'insert', obj: list, key: 1, value: 2, datatype: 'int' },
-        { action: 'insert', obj: list, key: 2, value: 3, datatype: 'int' },
-        { action: 'insert', obj: list, key: 3, value: 4, datatype: 'int' },
-        { action: 'delete', obj: list, key: 1 },
-        { action: 'delete', obj: list, key: 1 },
+        { action: 'put', path: ['list'],  value: [], conflict: false },
+        { action: 'splice', path: ['list',0], values: [1,2,3,4] },
+        { action: 'del', path: ['list',1] },
+        { action: 'del', path: ['list',1] },
       ])
       doc1.free()
     })
@@ -967,8 +949,8 @@ describe('Automerge', () => {
       doc1.increment('_root', 'counter', 4)
 
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'counter', value: 2, datatype: 'counter', conflict: false },
-        { action: 'increment', obj: '_root', key: 'counter', value: 4 },
+        { action: 'put', path: ['counter'], value: 2, conflict: false },
+        { action: 'inc', path: ['counter'], value: 4 },
       ])
       doc1.free()
     })
@@ -982,10 +964,10 @@ describe('Automerge', () => {
       doc1.delete('_root', 'key1')
       doc1.delete('_root', 'key2')
       assert.deepEqual(doc1.popPatches(), [
-        { action: 'put', obj: '_root', key: 'key1', value: 1, datatype: 'int', conflict: false },
-        { action: 'put', obj: '_root', key: 'key2', value: 2, datatype: 'int', conflict: false },
-        { action: 'delete', obj: '_root', key: 'key1' },
-        { action: 'delete', obj: '_root', key: 'key2' },
+        { action: 'put', path: ['key1'], value: 1, conflict: false },
+        { action: 'put', path: ['key2'], value: 2, conflict: false },
+        { action: 'del', path: ['key1'], },
+        { action: 'del', path: ['key2'], },
       ])
       doc1.free()
     })
@@ -999,8 +981,8 @@ describe('Automerge', () => {
       doc2.loadIncremental(doc1.saveIncremental())
       assert.deepEqual(doc2.getWithType('_root', 'starlings'), ['counter', 3])
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'starlings', value: 2, datatype: 'counter', conflict: false },
-        { action: 'increment', obj: '_root', key: 'starlings', value: 1 }
+        { action: 'put', path: ['starlings'], value: 2, conflict: false },
+        { action: 'inc', path: ['starlings'], value: 1 }
       ])
       doc1.free(); doc2.free()
     })
@@ -1018,10 +1000,10 @@ describe('Automerge', () => {
       doc2.loadIncremental(doc1.saveIncremental())
 
       assert.deepEqual(doc2.popPatches(), [
-        { action: 'put', obj: '_root', key: 'list', value: list, datatype: 'list', conflict: false },
-        { action: 'insert', obj: list, key: 0, value: 1, datatype: 'counter' },
-        { action: 'increment', obj: list, key: 0, value: 2 },
-        { action: 'increment', obj: list, key: 0, value: -5 },
+        { action: 'put', path: ['list'], value: [], conflict: false },
+        { action: 'splice', path: ['list',0], values: [1] },
+        { action: 'inc', path: ['list',0], value: 2 },
+        { action: 'inc', path: ['list',0], value: -5 },
       ])
       doc1.free(); doc2.free()
     })

automerge-wasm/types/package.json

@@ -6,7 +6,7 @@
   "description": "typescript types for low level automerge api",
   "homepage": "https://github.com/automerge/automerge-rs/tree/main/automerge-wasm",
   "repository": "github:automerge/automerge-rs",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "license": "MIT",
   "files": [
     "LICENSE",

automerge-wasm/web-index.js

@@ -1,49 +0,0 @@
-export {
-  loadDoc as load,
-  create,
-  encodeChange,
-  decodeChange,
-  initSyncState,
-  encodeSyncMessage,
-  decodeSyncMessage,
-  encodeSyncState,
-  decodeSyncState,
-  exportSyncState,
-  importSyncState,
-} from "./bindgen.js"
-import {
-  loadDoc as load,
-  create,
-  encodeChange,
-  decodeChange,
-  initSyncState,
-  encodeSyncMessage,
-  decodeSyncMessage,
-  encodeSyncState,
-  decodeSyncState,
-  exportSyncState,
-  importSyncState,
-} from "./bindgen.js"
-
-let api = {
-  load,
-  create,
-  encodeChange,
-  decodeChange,
-  initSyncState,
-  encodeSyncMessage,
-  decodeSyncMessage,
-  encodeSyncState,
-  decodeSyncState,
-  exportSyncState,
-  importSyncState
-}
-
-import wasm_init from "./bindgen.js"
-
-export function init() {
-  return new Promise((resolve,reject) => wasm_init().then(() => {
-    resolve({ ... api, load, create })
-  }))
-}
-

automerge/examples/watch.rs

@@ -9,19 +9,19 @@ use automerge::ROOT;
 fn main() {
     let mut doc = Automerge::new();
 
-    let mut observer = VecOpObserver::default();
     // a simple scalar change in the root object
-    doc.transact_with::<_, _, AutomergeError, _, _>(
-        |_result| CommitOptions::default().with_op_observer(&mut observer),
-        |tx| {
-            tx.put(ROOT, "hello", "world").unwrap();
-            Ok(())
-        },
-    )
-    .unwrap();
-    get_changes(&doc, observer.take_patches());
+    let mut result = doc
+        .transact_with::<_, _, AutomergeError, _, VecOpObserver>(
+            |_result| CommitOptions::default(),
+            |tx| {
+                tx.put(ROOT, "hello", "world").unwrap();
+                Ok(())
+            },
+        )
+        .unwrap();
+    get_changes(&doc, result.op_observer.take_patches());
 
-    let mut tx = doc.transaction();
+    let mut tx = doc.transaction_with_observer(VecOpObserver::default());
     let map = tx
         .put_object(ROOT, "my new map", automerge::ObjType::Map)
         .unwrap();
@@ -36,28 +36,28 @@ fn main() {
     tx.insert(&list, 1, "woo").unwrap();
     let m = tx.insert_object(&list, 2, automerge::ObjType::Map).unwrap();
     tx.put(&m, "hi", 2).unwrap();
-    let _heads3 = tx.commit_with(CommitOptions::default().with_op_observer(&mut observer));
-    get_changes(&doc, observer.take_patches());
+    let patches = tx.op_observer.take_patches();
+    let _heads3 = tx.commit_with(CommitOptions::default());
+    get_changes(&doc, patches);
 }
 
 fn get_changes(doc: &Automerge, patches: Vec<Patch>) {
     for patch in patches {
         match patch {
             Patch::Put {
-                obj,
-                key,
-                value,
-                conflict: _,
+                obj, prop, value, ..
             } => {
                 println!(
                     "put {:?} at {:?} in obj {:?}, object path {:?}",
                     value,
-                    key,
+                    prop,
                     obj,
                     doc.path_to_object(&obj)
                 )
             }
-            Patch::Insert { obj, index, value } => {
+            Patch::Insert {
+                obj, index, value, ..
+            } => {
                 println!(
                     "insert {:?} at {:?} in obj {:?}, object path {:?}",
                     value,
@@ -66,18 +66,20 @@ fn get_changes(doc: &Automerge, patches: Vec<Patch>) {
                     doc.path_to_object(&obj)
                 )
             }
-            Patch::Increment { obj, key, value } => {
+            Patch::Increment {
+                obj, prop, value, ..
+            } => {
                 println!(
                     "increment {:?} in obj {:?} by {:?}, object path {:?}",
-                    key,
+                    prop,
                     obj,
                     value,
                     doc.path_to_object(&obj)
                 )
             }
-            Patch::Delete { obj, key } => println!(
+            Patch::Delete { obj, prop, .. } => println!(
                 "delete {:?} in obj {:?}, object path {:?}",
-                key,
+                prop,
                 obj,
                 doc.path_to_object(&obj)
             ),

automerge/src/autocommit.rs

@@ -4,8 +4,7 @@ use crate::exid::ExId;
 use crate::op_observer::OpObserver;
 use crate::transaction::{CommitOptions, Transactable};
 use crate::{
-    sync, ApplyOptions, Keys, KeysAt, ListRange, ListRangeAt, MapRange, MapRangeAt, ObjType,
-    Parents, ScalarValue,
+    sync, Keys, KeysAt, ListRange, ListRangeAt, MapRange, MapRangeAt, ObjType, Parents, ScalarValue,
 };
 use crate::{
     transaction::TransactionInner, ActorId, Automerge, AutomergeError, Change, ChangeHash, Prop,
@@ -14,22 +13,46 @@ use crate::{
 
 /// An automerge document that automatically manages transactions.
 #[derive(Debug, Clone)]
-pub struct AutoCommit {
+pub struct AutoCommitWithObs<Obs: OpObserver> {
     doc: Automerge,
-    transaction: Option<TransactionInner>,
+    transaction: Option<(Obs, TransactionInner)>,
+    op_observer: Obs,
 }
 
-impl Default for AutoCommit {
+pub type AutoCommit = AutoCommitWithObs<()>;
+
+impl<O: OpObserver> Default for AutoCommitWithObs<O> {
     fn default() -> Self {
-        Self::new()
+        let op_observer = O::default();
+        AutoCommitWithObs {
+            doc: Automerge::new(),
+            transaction: None,
+            op_observer,
+        }
     }
 }
 
 impl AutoCommit {
-    pub fn new() -> Self {
-        Self {
+    pub fn new() -> AutoCommit {
+        AutoCommitWithObs {
             doc: Automerge::new(),
             transaction: None,
+            op_observer: (),
+        }
+    }
+}
+
+impl<Obs: OpObserver> AutoCommitWithObs<Obs> {
+    pub fn observer(&mut self) -> &mut Obs {
+        self.ensure_transaction_closed();
+        &mut self.op_observer
+    }
+
+    pub fn with_observer<Obs2: OpObserver>(self, op_observer: Obs2) -> AutoCommitWithObs<Obs2> {
+        AutoCommitWithObs {
+            doc: self.doc,
+            transaction: self.transaction.map(|(_, t)| (op_observer.branch(), t)),
+            op_observer,
         }
     }
 
@@ -58,7 +81,7 @@ impl AutoCommit {
 
     fn ensure_transaction_open(&mut self) {
         if self.transaction.is_none() {
-            self.transaction = Some(self.doc.transaction_inner());
+            self.transaction = Some((self.op_observer.branch(), self.doc.transaction_inner()));
         }
     }
 
@@ -67,6 +90,7 @@ impl AutoCommit {
         Self {
             doc: self.doc.fork(),
             transaction: self.transaction.clone(),
+            op_observer: self.op_observer.clone(),
         }
     }
 
@@ -75,46 +99,35 @@ impl AutoCommit {
         Ok(Self {
             doc: self.doc.fork_at(heads)?,
             transaction: self.transaction.clone(),
+            op_observer: self.op_observer.clone(),
         })
     }
 
     fn ensure_transaction_closed(&mut self) {
-        if let Some(tx) = self.transaction.take() {
-            tx.commit::<()>(&mut self.doc, None, None, None);
+        if let Some((current, tx)) = self.transaction.take() {
+            self.op_observer.merge(&current);
+            tx.commit(&mut self.doc, None, None);
         }
     }
 
     pub fn load(data: &[u8]) -> Result<Self, AutomergeError> {
+        // passing a () observer here has performance implications on all loads
+        // if we want an autocommit::load() method that can be observered we need to make a new method
+        // fn observed_load() ?
         let doc = Automerge::load(data)?;
+        let op_observer = Obs::default();
         Ok(Self {
             doc,
             transaction: None,
-        })
-    }
-
-    pub fn load_with<Obs: OpObserver>(
-        data: &[u8],
-        options: ApplyOptions<'_, Obs>,
-    ) -> Result<Self, AutomergeError> {
-        let doc = Automerge::load_with(data, options)?;
-        Ok(Self {
-            doc,
-            transaction: None,
+            op_observer,
         })
     }
 
     pub fn load_incremental(&mut self, data: &[u8]) -> Result<usize, AutomergeError> {
         self.ensure_transaction_closed();
-        self.doc.load_incremental(data)
-    }
-
-    pub fn load_incremental_with<'a, Obs: OpObserver>(
-        &mut self,
-        data: &[u8],
-        options: ApplyOptions<'a, Obs>,
-    ) -> Result<usize, AutomergeError> {
-        self.ensure_transaction_closed();
-        self.doc.load_incremental_with(data, options)
+        // TODO - would be nice to pass None here instead of &mut ()
+        self.doc
+            .load_incremental_with(data, Some(&mut self.op_observer))
     }
 
     pub fn apply_changes(
@@ -122,34 +135,19 @@ impl AutoCommit {
         changes: impl IntoIterator<Item = Change>,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_closed();
-        self.doc.apply_changes(changes)
-    }
-
-    pub fn apply_changes_with<I: IntoIterator<Item = Change>, Obs: OpObserver>(
-        &mut self,
-        changes: I,
-        options: ApplyOptions<'_, Obs>,
-    ) -> Result<(), AutomergeError> {
-        self.ensure_transaction_closed();
-        self.doc.apply_changes_with(changes, options)
+        self.doc
+            .apply_changes_with(changes, Some(&mut self.op_observer))
     }
 
     /// Takes all the changes in `other` which are not in `self` and applies them
-    pub fn merge(&mut self, other: &mut Self) -> Result<Vec<ChangeHash>, AutomergeError> {
-        self.ensure_transaction_closed();
-        other.ensure_transaction_closed();
-        self.doc.merge(&mut other.doc)
-    }
-
-    /// Takes all the changes in `other` which are not in `self` and applies them
-    pub fn merge_with<'a, Obs: OpObserver>(
+    pub fn merge<Obs2: OpObserver>(
         &mut self,
-        other: &mut Self,
-        options: ApplyOptions<'a, Obs>,
+        other: &mut AutoCommitWithObs<Obs2>,
     ) -> Result<Vec<ChangeHash>, AutomergeError> {
         self.ensure_transaction_closed();
         other.ensure_transaction_closed();
-        self.doc.merge_with(&mut other.doc, options)
+        self.doc
+            .merge_with(&mut other.doc, Some(&mut self.op_observer))
     }
 
     pub fn save(&mut self) -> Vec<u8> {
@@ -215,25 +213,21 @@ impl AutoCommit {
         &mut self,
         sync_state: &mut sync::State,
         message: sync::Message,
-    ) -> Result<(), AutomergeError> {
-        self.ensure_transaction_closed();
-        self.doc.receive_sync_message(sync_state, message)
-    }
-
-    pub fn receive_sync_message_with<'a, Obs: OpObserver>(
-        &mut self,
-        sync_state: &mut sync::State,
-        message: sync::Message,
-        options: ApplyOptions<'a, Obs>,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_closed();
         self.doc
-            .receive_sync_message_with(sync_state, message, options)
+            .receive_sync_message_with(sync_state, message, Some(&mut self.op_observer))
     }
 
+    /// Return a graphviz representation of the opset.
+    ///
+    /// # Arguments
+    ///
+    /// * objects: An optional list of object IDs to display, if not specified all objects are
+    ///            visualised
     #[cfg(feature = "optree-visualisation")]
-    pub fn visualise_optree(&self) -> String {
-        self.doc.visualise_optree()
+    pub fn visualise_optree(&self, objects: Option<Vec<ExId>>) -> String {
+        self.doc.visualise_optree(objects)
     }
 
     /// Get the current heads of the document.
@@ -245,7 +239,7 @@ impl AutoCommit {
     }
 
     pub fn commit(&mut self) -> ChangeHash {
-        self.commit_with::<()>(CommitOptions::default())
+        self.commit_with(CommitOptions::default())
     }
 
     /// Commit the current operations with some options.
@@ -261,33 +255,29 @@ impl AutoCommit {
     /// doc.put_object(&ROOT, "todos", ObjType::List).unwrap();
     /// let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as
     /// i64;
-    /// doc.commit_with::<()>(CommitOptions::default().with_message("Create todos list").with_time(now));
+    /// doc.commit_with(CommitOptions::default().with_message("Create todos list").with_time(now));
     /// ```
-    pub fn commit_with<Obs: OpObserver>(&mut self, options: CommitOptions<'_, Obs>) -> ChangeHash {
+    pub fn commit_with(&mut self, options: CommitOptions) -> ChangeHash {
         // ensure that even no changes triggers a change
         self.ensure_transaction_open();
-        let tx = self.transaction.take().unwrap();
-        tx.commit(
-            &mut self.doc,
-            options.message,
-            options.time,
-            options.op_observer,
-        )
+        let (current, tx) = self.transaction.take().unwrap();
+        self.op_observer.merge(&current);
+        tx.commit(&mut self.doc, options.message, options.time)
     }
 
     pub fn rollback(&mut self) -> usize {
         self.transaction
             .take()
-            .map(|tx| tx.rollback(&mut self.doc))
+            .map(|(_, tx)| tx.rollback(&mut self.doc))
             .unwrap_or(0)
     }
 }
 
-impl Transactable for AutoCommit {
+impl<Obs: OpObserver> Transactable for AutoCommitWithObs<Obs> {
     fn pending_ops(&self) -> usize {
         self.transaction
             .as_ref()
-            .map(|t| t.pending_ops())
+            .map(|(_, t)| t.pending_ops())
             .unwrap_or(0)
     }
 
@@ -383,8 +373,8 @@ impl Transactable for AutoCommit {
         value: V,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.put(&mut self.doc, obj.as_ref(), prop, value)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.put(&mut self.doc, current, obj.as_ref(), prop, value)
     }
 
     fn put_object<O: AsRef<ExId>, P: Into<Prop>>(
@@ -394,8 +384,8 @@ impl Transactable for AutoCommit {
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.put_object(&mut self.doc, obj.as_ref(), prop, value)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.put_object(&mut self.doc, current, obj.as_ref(), prop, value)
     }
 
     fn insert<O: AsRef<ExId>, V: Into<ScalarValue>>(
@@ -405,8 +395,8 @@ impl Transactable for AutoCommit {
         value: V,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.insert(&mut self.doc, obj.as_ref(), index, value)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.insert(&mut self.doc, current, obj.as_ref(), index, value)
     }
 
     fn insert_object<O: AsRef<ExId>>(
@@ -416,8 +406,8 @@ impl Transactable for AutoCommit {
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.insert_object(&mut self.doc, obj.as_ref(), index, value)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.insert_object(&mut self.doc, current, obj.as_ref(), index, value)
     }
 
     fn increment<O: AsRef<ExId>, P: Into<Prop>>(
@@ -427,8 +417,8 @@ impl Transactable for AutoCommit {
         value: i64,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.increment(&mut self.doc, obj.as_ref(), prop, value)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.increment(&mut self.doc, current, obj.as_ref(), prop, value)
     }
 
     fn delete<O: AsRef<ExId>, P: Into<Prop>>(
@@ -437,8 +427,8 @@ impl Transactable for AutoCommit {
         prop: P,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.delete(&mut self.doc, obj.as_ref(), prop)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.delete(&mut self.doc, current, obj.as_ref(), prop)
     }
 
     /// Splice new elements into the given sequence. Returns a vector of the OpIds used to insert
@@ -451,8 +441,8 @@ impl Transactable for AutoCommit {
         vals: V,
     ) -> Result<(), AutomergeError> {
         self.ensure_transaction_open();
-        let tx = self.transaction.as_mut().unwrap();
-        tx.splice(&mut self.doc, obj.as_ref(), pos, del, vals)
+        let (current, tx) = self.transaction.as_mut().unwrap();
+        tx.splice(&mut self.doc, current, obj.as_ref(), pos, del, vals)
     }
 
     fn text<O: AsRef<ExId>>(&self, obj: O) -> Result<String, AutomergeError> {

automerge/src/automerge.rs

@@ -19,8 +19,8 @@ use crate::types::{
     ScalarValue, Value,
 };
 use crate::{
-    query, ApplyOptions, AutomergeError, Change, KeysAt, ListRange, ListRangeAt, MapRange,
-    MapRangeAt, ObjType, Prop, Values,
+    query, AutomergeError, Change, KeysAt, ListRange, ListRangeAt, MapRange, MapRangeAt, ObjType,
+    Prop, Values,
 };
 use serde::Serialize;
 
@@ -111,10 +111,22 @@ impl Automerge {
     }
 
     /// Start a transaction.
-    pub fn transaction(&mut self) -> Transaction<'_> {
+    pub fn transaction(&mut self) -> Transaction<'_, ()> {
         Transaction {
             inner: Some(self.transaction_inner()),
             doc: self,
+            op_observer: (),
+        }
+    }
+
+    pub fn transaction_with_observer<Obs: OpObserver>(
+        &mut self,
+        op_observer: Obs,
+    ) -> Transaction<'_, Obs> {
+        Transaction {
+            inner: Some(self.transaction_inner()),
+            doc: self,
+            op_observer,
         }
     }
 
@@ -143,15 +155,16 @@ impl Automerge {
 
     /// Run a transaction on this document in a closure, automatically handling commit or rollback
     /// afterwards.
-    pub fn transact<F, O, E>(&mut self, f: F) -> transaction::Result<O, E>
+    pub fn transact<F, O, E>(&mut self, f: F) -> transaction::Result<O, (), E>
     where
-        F: FnOnce(&mut Transaction<'_>) -> Result<O, E>,
+        F: FnOnce(&mut Transaction<'_, ()>) -> Result<O, E>,
     {
         let mut tx = self.transaction();
         let result = f(&mut tx);
         match result {
             Ok(result) => Ok(Success {
                 result,
+                op_observer: (),
                 hash: tx.commit(),
             }),
             Err(error) => Err(Failure {
@@ -162,19 +175,25 @@ impl Automerge {
     }
 
     /// Like [`Self::transact`] but with a function for generating the commit options.
-    pub fn transact_with<'a, F, O, E, C, Obs>(&mut self, c: C, f: F) -> transaction::Result<O, E>
+    pub fn transact_with<F, O, E, C, Obs>(&mut self, c: C, f: F) -> transaction::Result<O, Obs, E>
     where
-        F: FnOnce(&mut Transaction<'_>) -> Result<O, E>,
-        C: FnOnce(&O) -> CommitOptions<'a, Obs>,
-        Obs: 'a + OpObserver,
+        F: FnOnce(&mut Transaction<'_, Obs>) -> Result<O, E>,
+        C: FnOnce(&O) -> CommitOptions,
+        Obs: OpObserver,
     {
-        let mut tx = self.transaction();
+        let mut op_observer = Obs::default();
+        let mut tx = self.transaction_with_observer(Default::default());
         let result = f(&mut tx);
         match result {
             Ok(result) => {
                 let commit_options = c(&result);
+                std::mem::swap(&mut op_observer, &mut tx.op_observer);
                 let hash = tx.commit_with(commit_options);
-                Ok(Success { result, hash })
+                Ok(Success {
+                    result,
+                    hash,
+                    op_observer,
+                })
             }
             Err(error) => Err(Failure {
                 error,
@@ -220,17 +239,6 @@ impl Automerge {
     // PropAt::()
     // NthAt::()
 
-    /// Get the object id of the object that contains this object and the prop that this object is
-    /// at in that object.
-    pub(crate) fn parent_object(&self, obj: ObjId) -> Option<(ObjId, Key)> {
-        if obj == ObjId::root() {
-            // root has no parent
-            None
-        } else {
-            self.ops.parent_object(&obj)
-        }
-    }
-
     /// Get the parents of an object in the document tree.
     ///
     /// ### Errors
@@ -244,10 +252,7 @@ impl Automerge {
     /// value.
     pub fn parents<O: AsRef<ExId>>(&self, obj: O) -> Result<Parents<'_>, AutomergeError> {
         let obj_id = self.exid_to_obj(obj.as_ref())?;
-        Ok(Parents {
-            obj: obj_id,
-            doc: self,
-        })
+        Ok(self.ops.parents(obj_id))
     }
 
     pub fn path_to_object<O: AsRef<ExId>>(
@@ -259,21 +264,6 @@ impl Automerge {
         Ok(path)
     }
 
-    /// Export a key to a prop.
-    pub(crate) fn export_key(&self, obj: ObjId, key: Key) -> Prop {
-        match key {
-            Key::Map(m) => Prop::Map(self.ops.m.props.get(m).into()),
-            Key::Seq(opid) => {
-                let i = self
-                    .ops
-                    .search(&obj, query::ElemIdPos::new(opid))
-                    .index()
-                    .unwrap();
-                Prop::Seq(i)
-            }
-        }
-    }
-
     /// Get the keys of the object `obj`.
     ///
     /// For a map this returns the keys of the map.
@@ -587,14 +577,14 @@ impl Automerge {
 
     /// Load a document.
     pub fn load(data: &[u8]) -> Result<Self, AutomergeError> {
-        Self::load_with::<()>(data, ApplyOptions::default())
+        Self::load_with::<()>(data, None)
     }
 
     /// Load a document.
-    #[tracing::instrument(skip(data, options), err)]
+    #[tracing::instrument(skip(data, observer), err)]
     pub fn load_with<Obs: OpObserver>(
         data: &[u8],
-        mut options: ApplyOptions<'_, Obs>,
+        mut observer: Option<&mut Obs>,
     ) -> Result<Self, AutomergeError> {
         if data.is_empty() {
             tracing::trace!("no data, initializing empty document");
@@ -606,7 +596,6 @@ impl Automerge {
         if !first_chunk.checksum_valid() {
             return Err(load::Error::BadChecksum.into());
         }
-        let observer = &mut options.op_observer;
 
         let mut am = match first_chunk {
             storage::Chunk::Document(d) => {
@@ -616,7 +605,7 @@ impl Automerge {
                     result: op_set,
                     changes,
                     heads,
-                } = match observer {
+                } = match &mut observer {
                     Some(o) => storage::load::reconstruct_document(&d, OpSet::observed_builder(*o)),
                     None => storage::load::reconstruct_document(&d, OpSet::builder()),
                 }
@@ -651,7 +640,7 @@ impl Automerge {
                 let change = Change::new_from_unverified(stored_change.into_owned(), None)
                     .map_err(|e| load::Error::InvalidChangeColumns(Box::new(e)))?;
                 let mut am = Self::new();
-                am.apply_change(change, observer);
+                am.apply_change(change, &mut observer);
                 am
             }
             storage::Chunk::CompressedChange(stored_change, compressed) => {
@@ -662,7 +651,7 @@ impl Automerge {
                 )
                 .map_err(|e| load::Error::InvalidChangeColumns(Box::new(e)))?;
                 let mut am = Self::new();
-                am.apply_change(change, observer);
+                am.apply_change(change, &mut observer);
                 am
             }
         };
@@ -670,7 +659,7 @@ impl Automerge {
         match load::load_changes(remaining.reset()) {
             load::LoadedChanges::Complete(c) => {
                 for change in c {
-                    am.apply_change(change, observer);
+                    am.apply_change(change, &mut observer);
                 }
             }
             load::LoadedChanges::Partial { error, .. } => return Err(error.into()),
@@ -680,14 +669,14 @@ impl Automerge {
 
     /// Load an incremental save of a document.
     pub fn load_incremental(&mut self, data: &[u8]) -> Result<usize, AutomergeError> {
-        self.load_incremental_with::<()>(data, ApplyOptions::default())
+        self.load_incremental_with::<()>(data, None)
     }
 
     /// Load an incremental save of a document.
     pub fn load_incremental_with<Obs: OpObserver>(
         &mut self,
         data: &[u8],
-        options: ApplyOptions<'_, Obs>,
+        op_observer: Option<&mut Obs>,
     ) -> Result<usize, AutomergeError> {
         let changes = match load::load_changes(storage::parse::Input::new(data)) {
             load::LoadedChanges::Complete(c) => c,
@@ -697,7 +686,7 @@ impl Automerge {
             }
         };
         let start = self.ops.len();
-        self.apply_changes_with(changes, options)?;
+        self.apply_changes_with(changes, op_observer)?;
         let delta = self.ops.len() - start;
         Ok(delta)
     }
@@ -717,14 +706,14 @@ impl Automerge {
         &mut self,
         changes: impl IntoIterator<Item = Change>,
     ) -> Result<(), AutomergeError> {
-        self.apply_changes_with::<_, ()>(changes, ApplyOptions::default())
+        self.apply_changes_with::<_, ()>(changes, None)
     }
 
     /// Apply changes to this document.
     pub fn apply_changes_with<I: IntoIterator<Item = Change>, Obs: OpObserver>(
         &mut self,
         changes: I,
-        mut options: ApplyOptions<'_, Obs>,
+        mut op_observer: Option<&mut Obs>,
     ) -> Result<(), AutomergeError> {
         for c in changes {
             if !self.history_index.contains_key(&c.hash()) {
@@ -735,7 +724,7 @@ impl Automerge {
                     ));
                 }
                 if self.is_causally_ready(&c) {
-                    self.apply_change(c, &mut options.op_observer);
+                    self.apply_change(c, &mut op_observer);
                 } else {
                     self.queue.push(c);
                 }
@@ -743,7 +732,7 @@ impl Automerge {
         }
         while let Some(c) = self.pop_next_causally_ready_change() {
             if !self.history_index.contains_key(&c.hash()) {
-                self.apply_change(c, &mut options.op_observer);
+                self.apply_change(c, &mut op_observer);
             }
         }
         Ok(())
@@ -831,14 +820,14 @@ impl Automerge {
 
     /// Takes all the changes in `other` which are not in `self` and applies them
     pub fn merge(&mut self, other: &mut Self) -> Result<Vec<ChangeHash>, AutomergeError> {
-        self.merge_with::<()>(other, ApplyOptions::default())
+        self.merge_with::<()>(other, None)
     }
 
     /// Takes all the changes in `other` which are not in `self` and applies them
-    pub fn merge_with<'a, Obs: OpObserver>(
+    pub fn merge_with<Obs: OpObserver>(
         &mut self,
         other: &mut Self,
-        options: ApplyOptions<'a, Obs>,
+        op_observer: Option<&mut Obs>,
     ) -> Result<Vec<ChangeHash>, AutomergeError> {
         // TODO: Make this fallible and figure out how to do this transactionally
         let changes = self
@@ -847,7 +836,7 @@ impl Automerge {
             .cloned()
             .collect::<Vec<_>>();
         tracing::trace!(changes=?changes.iter().map(|c| c.hash()).collect::<Vec<_>>(), "merging new changes");
-        self.apply_changes_with(changes, options)?;
+        self.apply_changes_with(changes, op_observer)?;
         Ok(self.get_heads())
     }
 
@@ -1178,9 +1167,17 @@ impl Automerge {
         }
     }
 
+    /// Return a graphviz representation of the opset.
+    ///
+    /// # Arguments
+    ///
+    /// * objects: An optional list of object IDs to display, if not specified all objects are
+    ///            visualised
     #[cfg(feature = "optree-visualisation")]
-    pub fn visualise_optree(&self) -> String {
-        self.ops.visualise()
+    pub fn visualise_optree(&self, objects: Option<Vec<ExId>>) -> String {
+        let objects =
+            objects.map(|os| os.iter().filter_map(|o| self.exid_to_obj(o).ok()).collect());
+        self.ops.visualise(objects)
     }
 }
 

automerge/src/automerge/tests.rs

@@ -1437,19 +1437,15 @@ fn observe_counter_change_application_overwrite() {
     doc1.increment(ROOT, "counter", 5).unwrap();
     doc1.commit();
 
-    let mut observer = VecOpObserver::default();
-    let mut doc3 = doc1.clone();
-    doc3.merge_with(
-        &mut doc2,
-        ApplyOptions::default().with_op_observer(&mut observer),
-    )
-    .unwrap();
+    let mut doc3 = doc1.fork().with_observer(VecOpObserver::default());
+    doc3.merge(&mut doc2).unwrap();
 
     assert_eq!(
-        observer.take_patches(),
+        doc3.observer().take_patches(),
         vec![Patch::Put {
             obj: ExId::Root,
-            key: Prop::Map("counter".into()),
+            path: vec![],
+            prop: Prop::Map("counter".into()),
             value: (
                 ScalarValue::Str("mystring".into()).into(),
                 ExId::Id(2, doc2.get_actor().clone(), 1)
@@ -1458,16 +1454,11 @@ fn observe_counter_change_application_overwrite() {
         }]
     );
 
-    let mut observer = VecOpObserver::default();
-    let mut doc4 = doc2.clone();
-    doc4.merge_with(
-        &mut doc1,
-        ApplyOptions::default().with_op_observer(&mut observer),
-    )
-    .unwrap();
+    let mut doc4 = doc2.clone().with_observer(VecOpObserver::default());
+    doc4.merge(&mut doc1).unwrap();
 
     // no patches as the increments operate on an invisible counter
-    assert_eq!(observer.take_patches(), vec![]);
+    assert_eq!(doc4.observer().take_patches(), vec![]);
 }
 
 #[test]
@@ -1478,20 +1469,15 @@ fn observe_counter_change_application() {
     doc.increment(ROOT, "counter", 5).unwrap();
     let changes = doc.get_changes(&[]).unwrap().into_iter().cloned();
 
-    let mut new_doc = AutoCommit::new();
-    let mut observer = VecOpObserver::default();
-    new_doc
-        .apply_changes_with(
-            changes,
-            ApplyOptions::default().with_op_observer(&mut observer),
-        )
-        .unwrap();
+    let mut new_doc = AutoCommit::new().with_observer(VecOpObserver::default());
+    new_doc.apply_changes(changes).unwrap();
     assert_eq!(
-        observer.take_patches(),
+        new_doc.observer().take_patches(),
         vec![
             Patch::Put {
                 obj: ExId::Root,
-                key: Prop::Map("counter".into()),
+                path: vec![],
+                prop: Prop::Map("counter".into()),
                 value: (
                     ScalarValue::counter(1).into(),
                     ExId::Id(1, doc.get_actor().clone(), 0)
@@ -1500,12 +1486,14 @@ fn observe_counter_change_application() {
             },
             Patch::Increment {
                 obj: ExId::Root,
-                key: Prop::Map("counter".into()),
+                path: vec![],
+                prop: Prop::Map("counter".into()),
                 value: (2, ExId::Id(2, doc.get_actor().clone(), 0)),
             },
             Patch::Increment {
                 obj: ExId::Root,
-                key: Prop::Map("counter".into()),
+                path: vec![],
+                prop: Prop::Map("counter".into()),
                 value: (5, ExId::Id(3, doc.get_actor().clone(), 0)),
             }
         ]
@@ -1514,7 +1502,7 @@ fn observe_counter_change_application() {
 
 #[test]
 fn get_changes_heads_empty() {
-    let mut doc = AutoCommit::new();
+    let mut doc = AutoCommit::default();
     doc.put(ROOT, "key1", 1).unwrap();
     doc.commit();
     doc.put(ROOT, "key2", 1).unwrap();

automerge/src/lib.rs

@@ -75,7 +75,6 @@ mod map_range_at;
 mod op_observer;
 mod op_set;
 mod op_tree;
-mod options;
 mod parents;
 mod query;
 mod storage;
@@ -88,7 +87,7 @@ mod values;
 mod visualisation;
 
 pub use crate::automerge::Automerge;
-pub use autocommit::AutoCommit;
+pub use autocommit::{AutoCommit, AutoCommitWithObs};
 pub use autoserde::AutoSerde;
 pub use change::{Change, LoadError as LoadChangeError};
 pub use error::AutomergeError;
@@ -105,7 +104,6 @@ pub use map_range_at::MapRangeAt;
 pub use op_observer::OpObserver;
 pub use op_observer::Patch;
 pub use op_observer::VecOpObserver;
-pub use options::ApplyOptions;
 pub use parents::Parents;
 pub use types::{ActorId, ChangeHash, ObjType, OpType, Prop};
 pub use value::{ScalarValue, Value};

automerge/src/op_observer.rs

@@ -1,50 +1,105 @@
 use crate::exid::ExId;
+use crate::Parents;
 use crate::Prop;
 use crate::Value;
 
 /// An observer of operations applied to the document.
-pub trait OpObserver {
+pub trait OpObserver: Default + Clone {
     /// A new value has been inserted into the given object.
     ///
     /// - `objid`: the object that has been inserted into.
     /// - `index`: the index the new value has been inserted at.
     /// - `tagged_value`: the value that has been inserted and the id of the operation that did the
     /// insert.
-    fn insert(&mut self, objid: ExId, index: usize, tagged_value: (Value<'_>, ExId));
+    fn insert(
+        &mut self,
+        parents: Parents<'_>,
+        objid: ExId,
+        index: usize,
+        tagged_value: (Value<'_>, ExId),
+    );
 
     /// A new value has been put into the given object.
     ///
     /// - `objid`: the object that has been put into.
-    /// - `key`: the key that the value as been put at.
+    /// - `prop`: the prop that the value as been put at.
     /// - `tagged_value`: the value that has been put into the object and the id of the operation
     /// that did the put.
     /// - `conflict`: whether this put conflicts with other operations.
-    fn put(&mut self, objid: ExId, key: Prop, tagged_value: (Value<'_>, ExId), conflict: bool);
+    fn put(
+        &mut self,
+        parents: Parents<'_>,
+        objid: ExId,
+        prop: Prop,
+        tagged_value: (Value<'_>, ExId),
+        conflict: bool,
+    );
 
     /// A counter has been incremented.
     ///
     /// - `objid`: the object that contains the counter.
-    /// - `key`: they key that the chounter is at.
+    /// - `prop`: they prop that the chounter is at.
     /// - `tagged_value`: the amount the counter has been incremented by, and the the id of the
     /// increment operation.
-    fn increment(&mut self, objid: ExId, key: Prop, tagged_value: (i64, ExId));
+    fn increment(
+        &mut self,
+        parents: Parents<'_>,
+        objid: ExId,
+        prop: Prop,
+        tagged_value: (i64, ExId),
+    );
 
     /// A value has beeen deleted.
     ///
     /// - `objid`: the object that has been deleted in.
-    /// - `key`: the key of the value that has been deleted.
-    fn delete(&mut self, objid: ExId, key: Prop);
+    /// - `prop`: the prop of the value that has been deleted.
+    fn delete(&mut self, parents: Parents<'_>, objid: ExId, prop: Prop);
+
+    /// Merge data with an other observer
+    ///
+    /// - `other`: Another Op Observer of the same type
+    fn merge(&mut self, other: &Self);
+
+    /// Branch off to begin a transaction - allows state information to be coppied if needed
+    ///
+    /// - `other`: Another Op Observer of the same type
+    fn branch(&self) -> Self {
+        Self::default()
+    }
 }
 
 impl OpObserver for () {
-    fn insert(&mut self, _objid: ExId, _index: usize, _tagged_value: (Value<'_>, ExId)) {}
-
-    fn put(&mut self, _objid: ExId, _key: Prop, _tagged_value: (Value<'_>, ExId), _conflict: bool) {
+    fn insert(
+        &mut self,
+        _parents: Parents<'_>,
+        _objid: ExId,
+        _index: usize,
+        _tagged_value: (Value<'_>, ExId),
+    ) {
     }
 
-    fn increment(&mut self, _objid: ExId, _key: Prop, _tagged_value: (i64, ExId)) {}
+    fn put(
+        &mut self,
+        _parents: Parents<'_>,
+        _objid: ExId,
+        _prop: Prop,
+        _tagged_value: (Value<'_>, ExId),
+        _conflict: bool,
+    ) {
+    }
 
-    fn delete(&mut self, _objid: ExId, _key: Prop) {}
+    fn increment(
+        &mut self,
+        _parents: Parents<'_>,
+        _objid: ExId,
+        _prop: Prop,
+        _tagged_value: (i64, ExId),
+    ) {
+    }
+
+    fn delete(&mut self, _parents: Parents<'_>, _objid: ExId, _prop: Prop) {}
+
+    fn merge(&mut self, _other: &Self) {}
 }
 
 /// Capture operations into a [`Vec`] and store them as patches.
@@ -62,45 +117,77 @@ impl VecOpObserver {
 }
 
 impl OpObserver for VecOpObserver {
-    fn insert(&mut self, obj_id: ExId, index: usize, (value, id): (Value<'_>, ExId)) {
+    fn insert(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ExId,
+        index: usize,
+        (value, id): (Value<'_>, ExId),
+    ) {
+        let path = parents.path();
         self.patches.push(Patch::Insert {
-            obj: obj_id,
+            obj,
+            path,
             index,
             value: (value.into_owned(), id),
         });
     }
 
-    fn put(&mut self, objid: ExId, key: Prop, (value, id): (Value<'_>, ExId), conflict: bool) {
+    fn put(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ExId,
+        prop: Prop,
+        (value, id): (Value<'_>, ExId),
+        conflict: bool,
+    ) {
+        let path = parents.path();
         self.patches.push(Patch::Put {
-            obj: objid,
-            key,
+            obj,
+            path,
+            prop,
             value: (value.into_owned(), id),
             conflict,
         });
     }
 
-    fn increment(&mut self, objid: ExId, key: Prop, tagged_value: (i64, ExId)) {
+    fn increment(
+        &mut self,
+        mut parents: Parents<'_>,
+        obj: ExId,
+        prop: Prop,
+        tagged_value: (i64, ExId),
+    ) {
+        let path = parents.path();
         self.patches.push(Patch::Increment {
-            obj: objid,
-            key,
+            obj,
+            path,
+            prop,
             value: tagged_value,
         });
     }
 
-    fn delete(&mut self, objid: ExId, key: Prop) {
-        self.patches.push(Patch::Delete { obj: objid, key })
+    fn delete(&mut self, mut parents: Parents<'_>, obj: ExId, prop: Prop) {
+        let path = parents.path();
+        self.patches.push(Patch::Delete { obj, path, prop })
+    }
+
+    fn merge(&mut self, other: &Self) {
+        self.patches.extend_from_slice(other.patches.as_slice())
     }
 }
 
 /// A notification to the application that something has changed in a document.
 #[derive(Debug, Clone, PartialEq)]
 pub enum Patch {
-    /// Associating a new value with a key in a map, or an existing list element
+    /// Associating a new value with a prop in a map, or an existing list element
     Put {
+        /// path to the object
+        path: Vec<(ExId, Prop)>,
         /// The object that was put into.
         obj: ExId,
-        /// The key that the new value was put at.
-        key: Prop,
+        /// The prop that the new value was put at.
+        prop: Prop,
         /// The value that was put, and the id of the operation that put it there.
         value: (Value<'static>, ExId),
         /// Whether this put conflicts with another.
@@ -108,6 +195,8 @@ pub enum Patch {
     },
     /// Inserting a new element into a list/text
     Insert {
+        /// path to the object
+        path: Vec<(ExId, Prop)>,
         /// The object that was inserted into.
         obj: ExId,
         /// The index that the new value was inserted at.
@@ -117,19 +206,23 @@ pub enum Patch {
     },
     /// Incrementing a counter.
     Increment {
+        /// path to the object
+        path: Vec<(ExId, Prop)>,
         /// The object that was incremented in.
         obj: ExId,
-        /// The key that was incremented.
-        key: Prop,
+        /// The prop that was incremented.
+        prop: Prop,
         /// The amount that the counter was incremented by, and the id of the operation that
         /// did the increment.
         value: (i64, ExId),
     },
     /// Deleting an element from a list/text
     Delete {
+        /// path to the object
+        path: Vec<(ExId, Prop)>,
         /// The object that was deleted from.
         obj: ExId,
-        /// The key that was deleted.
-        key: Prop,
+        /// The prop that was deleted.
+        prop: Prop,
     },
 }

automerge/src/op_set.rs

@@ -2,8 +2,9 @@ use crate::clock::Clock;
 use crate::exid::ExId;
 use crate::indexed_cache::IndexedCache;
 use crate::op_tree::{self, OpTree};
+use crate::parents::Parents;
 use crate::query::{self, OpIdSearch, TreeQuery};
-use crate::types::{self, ActorId, Key, ObjId, Op, OpId, OpIds, OpType};
+use crate::types::{self, ActorId, Key, ObjId, Op, OpId, OpIds, OpType, Prop};
 use crate::{ObjType, OpObserver};
 use fxhash::FxBuildHasher;
 use std::borrow::Borrow;
@@ -68,12 +69,29 @@ impl OpSetInternal {
         }
     }
 
+    pub(crate) fn parents(&self, obj: ObjId) -> Parents<'_> {
+        Parents { obj, ops: self }
+    }
+
     pub(crate) fn parent_object(&self, obj: &ObjId) -> Option<(ObjId, Key)> {
         let parent = self.trees.get(obj)?.parent?;
         let key = self.search(&parent, OpIdSearch::new(obj.0)).key().unwrap();
         Some((parent, key))
     }
 
+    pub(crate) fn export_key(&self, obj: ObjId, key: Key) -> Prop {
+        match key {
+            Key::Map(m) => Prop::Map(self.m.props.get(m).into()),
+            Key::Seq(opid) => {
+                let i = self
+                    .search(&obj, query::ElemIdPos::new(opid))
+                    .index()
+                    .unwrap();
+                Prop::Seq(i)
+            }
+        }
+    }
+
     pub(crate) fn keys(&self, obj: ObjId) -> Option<query::Keys<'_>> {
         if let Some(tree) = self.trees.get(&obj) {
             tree.internal.keys()
@@ -245,6 +263,8 @@ impl OpSetInternal {
         } = q;
 
         let ex_obj = self.id_to_exid(obj.0);
+        let parents = self.parents(*obj);
+
         let key = match op.key {
             Key::Map(index) => self.m.props[index].clone().into(),
             Key::Seq(_) => seen.into(),
@@ -252,21 +272,26 @@ impl OpSetInternal {
 
         if op.insert {
             let value = (op.value(), self.id_to_exid(op.id));
-            observer.insert(ex_obj, seen, value);
+            observer.insert(parents, ex_obj, seen, value);
         } else if op.is_delete() {
             if let Some(winner) = &values.last() {
                 let value = (winner.value(), self.id_to_exid(winner.id));
                 let conflict = values.len() > 1;
-                observer.put(ex_obj, key, value, conflict);
-            } else {
-                observer.delete(ex_obj, key);
+                observer.put(parents, ex_obj, key, value, conflict);
+            } else if had_value_before {
+                observer.delete(parents, ex_obj, key);
             }
         } else if let Some(value) = op.get_increment_value() {
             // only observe this increment if the counter is visible, i.e. the counter's
             // create op is in the values
-            if values.iter().any(|value| op.pred.contains(&value.id)) {
+            //if values.iter().any(|value| op.pred.contains(&value.id)) {
+            if values
+                .last()
+                .map(|value| op.pred.contains(&value.id))
+                .unwrap_or_default()
+            {
                 // we have observed the value
-                observer.increment(ex_obj, key, (value, self.id_to_exid(op.id)));
+                observer.increment(parents, ex_obj, key, (value, self.id_to_exid(op.id)));
             }
         } else {
             let winner = if let Some(last_value) = values.last() {
@@ -280,10 +305,10 @@ impl OpSetInternal {
             };
             let value = (winner.value(), self.id_to_exid(winner.id));
             if op.is_list_op() && !had_value_before {
-                observer.insert(ex_obj, seen, value);
+                observer.insert(parents, ex_obj, seen, value);
             } else {
                 let conflict = !values.is_empty();
-                observer.put(ex_obj, key, value, conflict);
+                observer.put(parents, ex_obj, key, value, conflict);
             }
         }
 
@@ -300,10 +325,24 @@ impl OpSetInternal {
         self.trees.get(id).map(|tree| tree.objtype)
     }
 
+    /// Return a graphviz representation of the opset.
+    ///
+    /// # Arguments
+    ///
+    /// * objects: An optional list of object IDs to display, if not specified all objects are
+    ///            visualised
     #[cfg(feature = "optree-visualisation")]
-    pub(crate) fn visualise(&self) -> String {
+    pub(crate) fn visualise(&self, objects: Option<Vec<ObjId>>) -> String {
+        use std::borrow::Cow;
         let mut out = Vec::new();
-        let graph = super::visualisation::GraphVisualisation::construct(&self.trees, &self.m);
+        let trees = if let Some(objects) = objects {
+            let mut filtered = self.trees.clone();
+            filtered.retain(|k, _| objects.contains(k));
+            Cow::Owned(filtered)
+        } else {
+            Cow::Borrowed(&self.trees)
+        };
+        let graph = super::visualisation::GraphVisualisation::construct(&trees, &self.m);
         dot::render(&graph, &mut out).unwrap();
         String::from_utf8_lossy(&out[..]).to_string()
     }

automerge/src/options.rs

@@ -1,16 +0,0 @@
-#[derive(Debug, Default)]
-pub struct ApplyOptions<'a, Obs> {
-    pub op_observer: Option<&'a mut Obs>,
-}
-
-impl<'a, Obs> ApplyOptions<'a, Obs> {
-    pub fn with_op_observer(mut self, op_observer: &'a mut Obs) -> Self {
-        self.op_observer = Some(op_observer);
-        self
-    }
-
-    pub fn set_op_observer(&mut self, op_observer: &'a mut Obs) -> &mut Self {
-        self.op_observer = Some(op_observer);
-        self
-    }
-}

automerge/src/parents.rs

@@ -1,18 +1,33 @@
-use crate::{exid::ExId, types::ObjId, Automerge, Prop};
+use crate::op_set::OpSet;
+use crate::types::ObjId;
+use crate::{exid::ExId, Prop};
 
 #[derive(Debug)]
 pub struct Parents<'a> {
     pub(crate) obj: ObjId,
-    pub(crate) doc: &'a Automerge,
+    pub(crate) ops: &'a OpSet,
+}
+
+impl<'a> Parents<'a> {
+    pub fn path(&mut self) -> Vec<(ExId, Prop)> {
+        let mut path = self.collect::<Vec<_>>();
+        path.reverse();
+        path
+    }
 }
 
 impl<'a> Iterator for Parents<'a> {
     type Item = (ExId, Prop);
 
     fn next(&mut self) -> Option<Self::Item> {
-        if let Some((obj, key)) = self.doc.parent_object(self.obj) {
+        if self.obj.is_root() {
+            None
+        } else if let Some((obj, key)) = self.ops.parent_object(&self.obj) {
             self.obj = obj;
-            Some((self.doc.id_to_exid(obj.0), self.doc.export_key(obj, key)))
+            Some((
+                self.ops.id_to_exid(self.obj.0),
+                self.ops.export_key(self.obj, key),
+            ))
         } else {
             None
         }

automerge/src/query/seek_op_with_patch.rs

@@ -8,8 +8,6 @@ use std::fmt::Debug;
 pub(crate) struct SeekOpWithPatch<'a> {
     op: Op,
     pub(crate) pos: usize,
-    /// A position counter for after we find the insert position to record conflicts.
-    later_pos: usize,
     pub(crate) succ: Vec<usize>,
     found: bool,
     pub(crate) seen: usize,
@@ -26,7 +24,6 @@ impl<'a> SeekOpWithPatch<'a> {
             op: op.clone(),
             succ: vec![],
             pos: 0,
-            later_pos: 0,
             found: false,
             seen: 0,
             last_seen: None,
@@ -176,6 +173,10 @@ impl<'a> TreeQuery<'a> for SeekOpWithPatch<'a> {
                             self.values.push(e);
                         }
                         self.succ.push(self.pos);
+
+                        if e.visible() {
+                            self.had_value_before = true;
+                        }
                     } else if e.visible() {
                         self.values.push(e);
                     }
@@ -184,7 +185,6 @@ impl<'a> TreeQuery<'a> for SeekOpWithPatch<'a> {
                     // we reach an op with an opId greater than that of the new operation
                     if m.lamport_cmp(e.id, self.op.id) == Ordering::Greater {
                         self.found = true;
-                        self.later_pos = self.pos + 1;
                         return QueryResult::Next;
                     }
 
@@ -202,7 +202,6 @@ impl<'a> TreeQuery<'a> for SeekOpWithPatch<'a> {
                     if e.visible() {
                         self.values.push(e);
                     }
-                    self.later_pos += 1;
                 }
                 QueryResult::Next
             }

automerge/src/storage/load/reconstruct_document.rs

@@ -236,9 +236,9 @@ impl LoadingObject {
     }
 
     fn append_op(&mut self, op: Op) -> Result<(), Error> {
-        // Collect set operations so we can find the keys which delete operations refer to in
-        // `finish`
-        if matches!(op.action, OpType::Put(_)) {
+        // Collect set and make operations so we can find the keys which delete operations refer to
+        // in `finish`
+        if matches!(op.action, OpType::Put(_) | OpType::Make(_)) {
             match op.key {
                 Key::Map(_) => {
                     self.set_ops.insert(op.id, op.key);

automerge/src/sync.rs

@@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet};
 
 use crate::{
     storage::{parse, Change as StoredChange, ReadChangeOpError},
-    ApplyOptions, Automerge, AutomergeError, Change, ChangeHash, OpObserver,
+    Automerge, AutomergeError, Change, ChangeHash, OpObserver,
 };
 
 mod bloom;
@@ -104,14 +104,14 @@ impl Automerge {
         sync_state: &mut State,
         message: Message,
     ) -> Result<(), AutomergeError> {
-        self.receive_sync_message_with::<()>(sync_state, message, ApplyOptions::default())
+        self.receive_sync_message_with::<()>(sync_state, message, None)
     }
 
-    pub fn receive_sync_message_with<'a, Obs: OpObserver>(
+    pub fn receive_sync_message_with<Obs: OpObserver>(
         &mut self,
         sync_state: &mut State,
         message: Message,
-        options: ApplyOptions<'a, Obs>,
+        op_observer: Option<&mut Obs>,
     ) -> Result<(), AutomergeError> {
         let before_heads = self.get_heads();
 
@@ -124,7 +124,7 @@ impl Automerge {
 
         let changes_is_empty = message_changes.is_empty();
         if !changes_is_empty {
-            self.apply_changes_with(message_changes, options)?;
+            self.apply_changes_with(message_changes, op_observer)?;
             sync_state.shared_heads = advance_heads(
                 &before_heads.iter().collect(),
                 &self.get_heads().into_iter().collect(),

automerge/src/transaction.rs

@@ -11,4 +11,4 @@ pub use manual_transaction::Transaction;
 pub use result::Failure;
 pub use result::Success;
 
-pub type Result<O, E> = std::result::Result<Success<O>, Failure<E>>;
+pub type Result<O, Obs, E> = std::result::Result<Success<O, Obs>, Failure<E>>;

automerge/src/transaction/commit.rs

@@ -1,12 +1,11 @@
 /// Optional metadata for a commit.
 #[derive(Debug, Default)]
-pub struct CommitOptions<'a, Obs> {
+pub struct CommitOptions {
     pub message: Option<String>,
     pub time: Option<i64>,
-    pub op_observer: Option<&'a mut Obs>,
 }
 
-impl<'a, Obs> CommitOptions<'a, Obs> {
+impl CommitOptions {
     /// Add a message to the commit.
     pub fn with_message<S: Into<String>>(mut self, message: S) -> Self {
         self.message = Some(message.into());
@@ -30,14 +29,4 @@ impl<'a, Obs> CommitOptions<'a, Obs> {
         self.time = Some(time);
         self
     }
-
-    pub fn with_op_observer(mut self, op_observer: &'a mut Obs) -> Self {
-        self.op_observer = Some(op_observer);
-        self
-    }
-
-    pub fn set_op_observer(&mut self, op_observer: &'a mut Obs) -> &mut Self {
-        self.op_observer = Some(op_observer);
-        self
-    }
 }

automerge/src/transaction/inner.rs

@@ -26,13 +26,12 @@ impl TransactionInner {
 
     /// Commit the operations performed in this transaction, returning the hashes corresponding to
     /// the new heads.
-    #[tracing::instrument(skip(self, doc, op_observer))]
-    pub(crate) fn commit<Obs: OpObserver>(
+    #[tracing::instrument(skip(self, doc))]
+    pub(crate) fn commit(
         mut self,
         doc: &mut Automerge,
         message: Option<String>,
         time: Option<i64>,
-        op_observer: Option<&mut Obs>,
     ) -> ChangeHash {
         if message.is_some() {
             self.message = message;
@@ -42,25 +41,27 @@ impl TransactionInner {
             self.time = t;
         }
 
-        if let Some(observer) = op_observer {
-            for (obj, prop, op) in &self.operations {
-                let ex_obj = doc.ops.id_to_exid(obj.0);
-                if op.insert {
-                    let value = (op.value(), doc.id_to_exid(op.id));
-                    match prop {
-                        Prop::Map(_) => panic!("insert into a map"),
-                        Prop::Seq(index) => observer.insert(ex_obj, *index, value),
+        /*
+                if let Some(observer) = op_observer {
+                    for (obj, prop, op) in &self.operations {
+                        let ex_obj = doc.ops.id_to_exid(obj.0);
+                        if op.insert {
+                            let value = (op.value(), doc.id_to_exid(op.id));
+                            match prop {
+                                Prop::Map(_) => panic!("insert into a map"),
+                                Prop::Seq(index) => observer.insert(ex_obj, *index, value),
+                            }
+                        } else if op.is_delete() {
+                            observer.delete(ex_obj, prop.clone());
+                        } else if let Some(value) = op.get_increment_value() {
+                            observer.increment(ex_obj, prop.clone(), (value, doc.id_to_exid(op.id)));
+                        } else {
+                            let value = (op.value(), doc.ops.id_to_exid(op.id));
+                            observer.put(ex_obj, prop.clone(), value, false);
+                        }
                     }
-                } else if op.is_delete() {
-                    observer.delete(ex_obj, prop.clone());
-                } else if let Some(value) = op.get_increment_value() {
-                    observer.increment(ex_obj, prop.clone(), (value, doc.id_to_exid(op.id)));
-                } else {
-                    let value = (op.value(), doc.ops.id_to_exid(op.id));
-                    observer.put(ex_obj, prop.clone(), value, false);
                 }
-            }
-        }
+        */
 
         let num_ops = self.pending_ops();
         let change = self.export(&doc.ops.m);
@@ -150,9 +151,10 @@ impl TransactionInner {
     /// - The object does not exist
     /// - The key is the wrong type for the object
     /// - The key does not exist in the object
-    pub(crate) fn put<P: Into<Prop>, V: Into<ScalarValue>>(
+    pub(crate) fn put<P: Into<Prop>, V: Into<ScalarValue>, Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         prop: P,
         value: V,
@@ -160,7 +162,7 @@ impl TransactionInner {
         let obj = doc.exid_to_obj(ex_obj)?;
         let value = value.into();
         let prop = prop.into();
-        self.local_op(doc, obj, prop, value.into())?;
+        self.local_op(doc, op_observer, obj, prop, value.into())?;
         Ok(())
     }
 
@@ -177,16 +179,19 @@ impl TransactionInner {
     /// - The object does not exist
     /// - The key is the wrong type for the object
     /// - The key does not exist in the object
-    pub(crate) fn put_object<P: Into<Prop>>(
+    pub(crate) fn put_object<P: Into<Prop>, Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         prop: P,
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
         let obj = doc.exid_to_obj(ex_obj)?;
         let prop = prop.into();
-        let id = self.local_op(doc, obj, prop, value.into())?.unwrap();
+        let id = self
+            .local_op(doc, op_observer, obj, prop, value.into())?
+            .unwrap();
         let id = doc.id_to_exid(id);
         Ok(id)
     }
@@ -195,9 +200,11 @@ impl TransactionInner {
         OpId(self.start_op.get() + self.pending_ops() as u64, self.actor)
     }
 
-    fn insert_local_op(
+    #[allow(clippy::too_many_arguments)]
+    fn insert_local_op<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         prop: Prop,
         op: Op,
         pos: usize,
@@ -210,12 +217,13 @@ impl TransactionInner {
             doc.ops.insert(pos, &obj, op.clone());
         }
 
-        self.operations.push((obj, prop, op));
+        self.finalize_op(doc, op_observer, obj, prop, op);
     }
 
-    pub(crate) fn insert<V: Into<ScalarValue>>(
+    pub(crate) fn insert<V: Into<ScalarValue>, Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         index: usize,
         value: V,
@@ -223,26 +231,28 @@ impl TransactionInner {
         let obj = doc.exid_to_obj(ex_obj)?;
         let value = value.into();
         tracing::trace!(obj=?obj, value=?value, "inserting value");
-        self.do_insert(doc, obj, index, value.into())?;
+        self.do_insert(doc, op_observer, obj, index, value.into())?;
         Ok(())
     }
 
-    pub(crate) fn insert_object(
+    pub(crate) fn insert_object<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         index: usize,
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
         let obj = doc.exid_to_obj(ex_obj)?;
-        let id = self.do_insert(doc, obj, index, value.into())?;
+        let id = self.do_insert(doc, op_observer, obj, index, value.into())?;
         let id = doc.id_to_exid(id);
         Ok(id)
     }
 
-    fn do_insert(
+    fn do_insert<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         obj: ObjId,
         index: usize,
         action: OpType,
@@ -263,27 +273,30 @@ impl TransactionInner {
         };
 
         doc.ops.insert(query.pos(), &obj, op.clone());
-        self.operations.push((obj, Prop::Seq(index), op));
+
+        self.finalize_op(doc, op_observer, obj, Prop::Seq(index), op);
 
         Ok(id)
     }
 
-    pub(crate) fn local_op(
+    pub(crate) fn local_op<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         obj: ObjId,
         prop: Prop,
         action: OpType,
     ) -> Result<Option<OpId>, AutomergeError> {
         match prop {
-            Prop::Map(s) => self.local_map_op(doc, obj, s, action),
-            Prop::Seq(n) => self.local_list_op(doc, obj, n, action),
+            Prop::Map(s) => self.local_map_op(doc, op_observer, obj, s, action),
+            Prop::Seq(n) => self.local_list_op(doc, op_observer, obj, n, action),
         }
     }
 
-    fn local_map_op(
+    fn local_map_op<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         obj: ObjId,
         prop: String,
         action: OpType,
@@ -324,14 +337,15 @@ impl TransactionInner {
 
         let pos = query.pos;
         let ops_pos = query.ops_pos;
-        self.insert_local_op(doc, Prop::Map(prop), op, pos, obj, &ops_pos);
+        self.insert_local_op(doc, op_observer, Prop::Map(prop), op, pos, obj, &ops_pos);
 
         Ok(Some(id))
     }
 
-    fn local_list_op(
+    fn local_list_op<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         obj: ObjId,
         index: usize,
         action: OpType,
@@ -363,40 +377,43 @@ impl TransactionInner {
 
         let pos = query.pos;
         let ops_pos = query.ops_pos;
-        self.insert_local_op(doc, Prop::Seq(index), op, pos, obj, &ops_pos);
+        self.insert_local_op(doc, op_observer, Prop::Seq(index), op, pos, obj, &ops_pos);
 
         Ok(Some(id))
     }
 
-    pub(crate) fn increment<P: Into<Prop>>(
+    pub(crate) fn increment<P: Into<Prop>, Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         obj: &ExId,
         prop: P,
         value: i64,
     ) -> Result<(), AutomergeError> {
         let obj = doc.exid_to_obj(obj)?;
-        self.local_op(doc, obj, prop.into(), OpType::Increment(value))?;
+        self.local_op(doc, op_observer, obj, prop.into(), OpType::Increment(value))?;
         Ok(())
     }
 
-    pub(crate) fn delete<P: Into<Prop>>(
+    pub(crate) fn delete<P: Into<Prop>, Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         prop: P,
     ) -> Result<(), AutomergeError> {
         let obj = doc.exid_to_obj(ex_obj)?;
         let prop = prop.into();
-        self.local_op(doc, obj, prop, OpType::Delete)?;
+        self.local_op(doc, op_observer, obj, prop, OpType::Delete)?;
         Ok(())
     }
 
     /// Splice new elements into the given sequence. Returns a vector of the OpIds used to insert
     /// the new elements
-    pub(crate) fn splice(
+    pub(crate) fn splice<Obs: OpObserver>(
         &mut self,
         doc: &mut Automerge,
+        op_observer: &mut Obs,
         ex_obj: &ExId,
         mut pos: usize,
         del: usize,
@@ -405,15 +422,48 @@ impl TransactionInner {
         let obj = doc.exid_to_obj(ex_obj)?;
         for _ in 0..del {
             // del()
-            self.local_op(doc, obj, pos.into(), OpType::Delete)?;
+            self.local_op(doc, op_observer, obj, pos.into(), OpType::Delete)?;
         }
         for v in vals {
             // insert()
-            self.do_insert(doc, obj, pos, v.clone().into())?;
+            self.do_insert(doc, op_observer, obj, pos, v.clone().into())?;
             pos += 1;
         }
         Ok(())
     }
+
+    fn finalize_op<Obs: OpObserver>(
+        &mut self,
+        doc: &mut Automerge,
+        op_observer: &mut Obs,
+        obj: ObjId,
+        prop: Prop,
+        op: Op,
+    ) {
+        // TODO - id_to_exid should be a noop if not used - change type to Into<ExId>?
+        let ex_obj = doc.ops.id_to_exid(obj.0);
+        let parents = doc.ops.parents(obj);
+        if op.insert {
+            let value = (op.value(), doc.ops.id_to_exid(op.id));
+            match prop {
+                Prop::Map(_) => panic!("insert into a map"),
+                Prop::Seq(index) => op_observer.insert(parents, ex_obj, index, value),
+            }
+        } else if op.is_delete() {
+            op_observer.delete(parents, ex_obj, prop.clone());
+        } else if let Some(value) = op.get_increment_value() {
+            op_observer.increment(
+                parents,
+                ex_obj,
+                prop.clone(),
+                (value, doc.ops.id_to_exid(op.id)),
+            );
+        } else {
+            let value = (op.value(), doc.ops.id_to_exid(op.id));
+            op_observer.put(parents, ex_obj, prop.clone(), value, false);
+        }
+        self.operations.push((obj, prop, op));
+    }
 }
 
 #[cfg(test)]

automerge/src/transaction/manual_transaction.rs

@@ -20,14 +20,15 @@ use super::{CommitOptions, Transactable, TransactionInner};
 /// intermediate state.
 /// This is consistent with `?` error handling.
 #[derive(Debug)]
-pub struct Transaction<'a> {
+pub struct Transaction<'a, Obs: OpObserver> {
     // this is an option so that we can take it during commit and rollback to prevent it being
     // rolled back during drop.
     pub(crate) inner: Option<TransactionInner>,
     pub(crate) doc: &'a mut Automerge,
+    pub op_observer: Obs,
 }
 
-impl<'a> Transaction<'a> {
+impl<'a, Obs: OpObserver> Transaction<'a, Obs> {
     /// Get the heads of the document before this transaction was started.
     pub fn get_heads(&self) -> Vec<ChangeHash> {
         self.doc.get_heads()
@@ -36,10 +37,7 @@ impl<'a> Transaction<'a> {
     /// Commit the operations performed in this transaction, returning the hashes corresponding to
     /// the new heads.
     pub fn commit(mut self) -> ChangeHash {
-        self.inner
-            .take()
-            .unwrap()
-            .commit::<()>(self.doc, None, None, None)
+        self.inner.take().unwrap().commit(self.doc, None, None)
     }
 
     /// Commit the operations in this transaction with some options.
@@ -56,15 +54,13 @@ impl<'a> Transaction<'a> {
     /// tx.put_object(ROOT, "todos", ObjType::List).unwrap();
     /// let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as
     /// i64;
-    /// tx.commit_with::<()>(CommitOptions::default().with_message("Create todos list").with_time(now));
+    /// tx.commit_with(CommitOptions::default().with_message("Create todos list").with_time(now));
     /// ```
-    pub fn commit_with<Obs: OpObserver>(mut self, options: CommitOptions<'_, Obs>) -> ChangeHash {
-        self.inner.take().unwrap().commit(
-            self.doc,
-            options.message,
-            options.time,
-            options.op_observer,
-        )
+    pub fn commit_with(mut self, options: CommitOptions) -> ChangeHash {
+        self.inner
+            .take()
+            .unwrap()
+            .commit(self.doc, options.message, options.time)
     }
 
     /// Undo the operations added in this transaction, returning the number of cancelled
@@ -74,7 +70,7 @@ impl<'a> Transaction<'a> {
     }
 }
 
-impl<'a> Transactable for Transaction<'a> {
+impl<'a, Obs: OpObserver> Transactable for Transaction<'a, Obs> {
     /// Get the number of pending operations in this transaction.
     fn pending_ops(&self) -> usize {
         self.inner.as_ref().unwrap().pending_ops()
@@ -97,7 +93,7 @@ impl<'a> Transactable for Transaction<'a> {
         self.inner
             .as_mut()
             .unwrap()
-            .put(self.doc, obj.as_ref(), prop, value)
+            .put(self.doc, &mut self.op_observer, obj.as_ref(), prop, value)
     }
 
     fn put_object<O: AsRef<ExId>, P: Into<Prop>>(
@@ -106,10 +102,13 @@ impl<'a> Transactable for Transaction<'a> {
         prop: P,
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
-        self.inner
-            .as_mut()
-            .unwrap()
-            .put_object(self.doc, obj.as_ref(), prop, value)
+        self.inner.as_mut().unwrap().put_object(
+            self.doc,
+            &mut self.op_observer,
+            obj.as_ref(),
+            prop,
+            value,
+        )
     }
 
     fn insert<O: AsRef<ExId>, V: Into<ScalarValue>>(
@@ -118,10 +117,13 @@ impl<'a> Transactable for Transaction<'a> {
         index: usize,
         value: V,
     ) -> Result<(), AutomergeError> {
-        self.inner
-            .as_mut()
-            .unwrap()
-            .insert(self.doc, obj.as_ref(), index, value)
+        self.inner.as_mut().unwrap().insert(
+            self.doc,
+            &mut self.op_observer,
+            obj.as_ref(),
+            index,
+            value,
+        )
     }
 
     fn insert_object<O: AsRef<ExId>>(
@@ -130,10 +132,13 @@ impl<'a> Transactable for Transaction<'a> {
         index: usize,
         value: ObjType,
     ) -> Result<ExId, AutomergeError> {
-        self.inner
-            .as_mut()
-            .unwrap()
-            .insert_object(self.doc, obj.as_ref(), index, value)
+        self.inner.as_mut().unwrap().insert_object(
+            self.doc,
+            &mut self.op_observer,
+            obj.as_ref(),
+            index,
+            value,
+        )
     }
 
     fn increment<O: AsRef<ExId>, P: Into<Prop>>(
@@ -142,10 +147,13 @@ impl<'a> Transactable for Transaction<'a> {
         prop: P,
         value: i64,
     ) -> Result<(), AutomergeError> {
-        self.inner
-            .as_mut()
-            .unwrap()
-            .increment(self.doc, obj.as_ref(), prop, value)
+        self.inner.as_mut().unwrap().increment(
+            self.doc,
+            &mut self.op_observer,
+            obj.as_ref(),
+            prop,
+            value,
+        )
     }
 
     fn delete<O: AsRef<ExId>, P: Into<Prop>>(
@@ -156,7 +164,7 @@ impl<'a> Transactable for Transaction<'a> {
         self.inner
             .as_mut()
             .unwrap()
-            .delete(self.doc, obj.as_ref(), prop)
+            .delete(self.doc, &mut self.op_observer, obj.as_ref(), prop)
     }
 
     /// Splice new elements into the given sequence. Returns a vector of the OpIds used to insert
@@ -168,10 +176,14 @@ impl<'a> Transactable for Transaction<'a> {
         del: usize,
         vals: V,
     ) -> Result<(), AutomergeError> {
-        self.inner
-            .as_mut()
-            .unwrap()
-            .splice(self.doc, obj.as_ref(), pos, del, vals)
+        self.inner.as_mut().unwrap().splice(
+            self.doc,
+            &mut self.op_observer,
+            obj.as_ref(),
+            pos,
+            del,
+            vals,
+        )
     }
 
     fn keys<O: AsRef<ExId>>(&self, obj: O) -> Keys<'_, '_> {
@@ -291,7 +303,7 @@ impl<'a> Transactable for Transaction<'a> {
 // intermediate state.
 // This defaults to rolling back the transaction to be compatible with `?` error returning before
 // reaching a call to `commit`.
-impl<'a> Drop for Transaction<'a> {
+impl<'a, Obs: OpObserver> Drop for Transaction<'a, Obs> {
     fn drop(&mut self) {
         if let Some(txn) = self.inner.take() {
             txn.rollback(self.doc);

automerge/src/transaction/result.rs

@@ -2,11 +2,12 @@ use crate::ChangeHash;
 
 /// The result of a successful, and committed, transaction.
 #[derive(Debug)]
-pub struct Success<O> {
+pub struct Success<O, Obs> {
     /// The result of the transaction.
     pub result: O,
     /// The hash of the change, also the head of the document.
     pub hash: ChangeHash,
+    pub op_observer: Obs,
 }
 
 /// The result of a failed, and rolled back, transaction.

automerge/tests/test.rs

@@ -1,7 +1,7 @@
 use automerge::transaction::Transactable;
 use automerge::{
-    ActorId, ApplyOptions, AutoCommit, Automerge, AutomergeError, Change, ExpandedChange, ObjType,
-    ScalarValue, VecOpObserver, ROOT,
+    ActorId, AutoCommit, Automerge, AutomergeError, Change, ExpandedChange, ObjType, ScalarValue,
+    VecOpObserver, ROOT,
 };
 
 // set up logging for all the tests
@@ -1005,13 +1005,8 @@ fn observe_counter_change_application() {
     doc.increment(ROOT, "counter", 5).unwrap();
     let changes = doc.get_changes(&[]).unwrap().into_iter().cloned();
 
-    let mut doc = AutoCommit::new();
-    let mut observer = VecOpObserver::default();
-    doc.apply_changes_with(
-        changes,
-        ApplyOptions::default().with_op_observer(&mut observer),
-    )
-    .unwrap();
+    let mut doc = AutoCommit::new().with_observer(VecOpObserver::default());
+    doc.apply_changes(changes).unwrap();
 }
 
 #[test]
@@ -1332,3 +1327,19 @@ fn load_incremental_with_corrupted_tail() {
         }
     );
 }
+
+#[test]
+fn load_doc_with_deleted_objects() {
+    // Reproduces an issue where a document with deleted objects failed to load
+    let mut doc = AutoCommit::new();
+    doc.put_object(ROOT, "list", ObjType::List).unwrap();
+    doc.put_object(ROOT, "text", ObjType::Text).unwrap();
+    doc.put_object(ROOT, "map", ObjType::Map).unwrap();
+    doc.put_object(ROOT, "table", ObjType::Table).unwrap();
+    doc.delete(&ROOT, "list").unwrap();
+    doc.delete(&ROOT, "text").unwrap();
+    doc.delete(&ROOT, "map").unwrap();
+    doc.delete(&ROOT, "table").unwrap();
+    let saved = doc.save();
+    Automerge::load(&saved).unwrap();
+}

scripts/ci/js_tests

@@ -3,18 +3,11 @@ set -e
 THIS_SCRIPT=$(dirname "$0");
 WASM_PROJECT=$THIS_SCRIPT/../../automerge-wasm;
 JS_PROJECT=$THIS_SCRIPT/../../automerge-js;
+E2E_PROJECT=$THIS_SCRIPT/../../automerge-js/e2e;
 
-yarn --cwd $WASM_PROJECT install;
-# This will take care of running wasm-pack
-yarn --cwd $WASM_PROJECT build;
-# If the dependencies are already installed we delete automerge-wasm. This makes
-# this script usable for iterative development.
-if [ -d $JS_PROJECT/node_modules/automerge-wasm ]; then
-    rm -rf $JS_PROJECT/node_modules/automerge-wasm
-fi
-# --check-files forces yarn to check if the local dep has changed
-yarn --cwd $JS_PROJECT install --check-files;
-yarn --cwd $JS_PROJECT test;
-
-
-
+yarn --cwd $E2E_PROJECT install;
+# This will build the automerge-wasm project, publish it to a local NPM
+# repository, then run `yarn build` in the `automerge-js` directory with
+# the local registry
+yarn --cwd $E2E_PROJECT e2e buildjs;
+yarn --cwd $JS_PROJECT test

scripts/ci/wasm_tests

@@ -1,6 +1,9 @@
 THIS_SCRIPT=$(dirname "$0");
+E2E_PROJECT=$THIS_SCRIPT/../../automerge-js/e2e;
 WASM_PROJECT=$THIS_SCRIPT/../../automerge-wasm;
 
-yarn --cwd $WASM_PROJECT install;
-yarn --cwd $WASM_PROJECT build;
+# This takes care of publishing the correct version of automerge-types in
+# a local NPM registry and calling yarn install with that registry available
+yarn --cwd $E2E_PROJECT install
+yarn --cwd $E2E_PROJECT e2e buildwasm
 yarn --cwd $WASM_PROJECT test;