automerge/javascript/e2e/index.ts

import {once} from "events"
import {setTimeout} from "timers/promises"
import {spawn, ChildProcess} from "child_process"
import * as child_process from "child_process"
import {command, subcommands, run, array, multioption, option, Type} from "cmd-ts"
import * as path from "path"
import * as fsPromises from "fs/promises"
import fetch from "node-fetch"

const VERDACCIO_DB_PATH = path.normalize(`${__dirname}/verdacciodb`)
const VERDACCIO_CONFIG_PATH = path.normalize(`${__dirname}/verdaccio.yaml`)
const AUTOMERGE_WASM_PATH = path.normalize(`${__dirname}/../../rust/automerge-wasm`)
const AUTOMERGE_JS_PATH = path.normalize(`${__dirname}/..`)
const EXAMPLES_DIR = path.normalize(path.join(__dirname, "../", "examples"))

// The different example projects in "../examples"
type Example = "webpack" | "vite" | "create-react-app"

// Type to parse strings to `Example` so the types line up for the `buildExamples` commmand
const ReadExample: Type<string, Example> = {
    async from(str) {
        if (str === "webpack") {
            return "webpack"
        } else if (str === "vite") {
            return "vite"
        } else if (str === "create-react-app") {
            return "create-react-app"
        } else {
            throw new Error(`Unknown example type ${str}`)
        }
    }
}

type Profile = "dev" | "release"

const ReadProfile: Type<string, Profile> = {
    async from(str) {
        if (str === "dev") {
            return "dev"
        } else if (str === "release") {
            return "release"
        } else {
            throw new Error(`Unknown profile ${str}`)
        }
    }
}

const buildjs = command({
    name: "buildjs",
    args: {
        profile: option({
            type: ReadProfile,
            long: "profile",
            short: "p",
            defaultValue: () => "dev" as Profile
        })
    },
    handler: ({profile}) => {
        console.log("building js")
        withPublishedWasm(profile, async (registryUrl: string) => {
            await buildAndPublishAutomergeJs(registryUrl)
        })
    }
})

const buildWasm = command({
    name: "buildwasm",
    args: {
        profile: option({
            type: ReadProfile,
            long: "profile",
            short: "p",
            defaultValue: () => "dev" as Profile
        })
    },
    handler: ({profile}) => {
        console.log("building automerge-wasm")
        withRegistry(
            buildAutomergeWasm(profile),
        )
    }
})

const buildexamples = command({
    name: "buildexamples",
    args: {
        examples: multioption({
            long: "example",
            short: "e",
            type: array(ReadExample),
        }),
        profile: option({
            type: ReadProfile,
            long: "profile",
            short: "p",
            defaultValue: () => "dev" as Profile
        })
    },
    handler: ({examples, profile}) => {
        if (examples.length === 0) {
            examples = ["webpack", "vite", "create-react-app"]
        }
        buildExamples(examples, profile)
    }
})


const runRegistry = command({
    name: "run-registry",
    args: {
        profile: option({
            type: ReadProfile,
            long: "profile",
            short: "p",
            defaultValue: () => "dev" as Profile
        })
    },
    handler: ({profile}) => {
        withPublishedWasm(profile, async (registryUrl: string) => {
            await buildAndPublishAutomergeJs(registryUrl)
            console.log("\n************************")
            console.log(`      Verdaccio NPM registry is running at ${registryUrl}`)
            console.log("      press CTRL-C to exit ")
            console.log("************************")
            await once(process, "SIGINT")
        }).catch(e => {
            console.error(`Failed: ${e}`)
        })
    }
})


const app = subcommands({
    name: "e2e",
    cmds: {buildjs, buildexamples, buildwasm: buildWasm, "run-registry": runRegistry}
})

run(app, process.argv.slice(2))

async function buildExamples(examples: Array<Example>, profile: Profile) {
    await withPublishedWasm(profile, async (registryUrl) => {
        printHeader("building and publishing automerge")
        await buildAndPublishAutomergeJs(registryUrl) 
        for (const example of examples) {
            printHeader(`building ${example} example`)
            if (example === "webpack") {
                const projectPath = path.join(EXAMPLES_DIR, example)
                await removeExistingAutomerge(projectPath)
                await fsPromises.rm(path.join(projectPath, "yarn.lock"), {force: true})
                await spawnAndWait("yarn", ["--cwd", projectPath, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
                await spawnAndWait("yarn", ["--cwd", projectPath, "build"], {stdio: "inherit"})
            } else if (example === "vite") {
                const projectPath = path.join(EXAMPLES_DIR, example)
                await removeExistingAutomerge(projectPath)
                await fsPromises.rm(path.join(projectPath, "yarn.lock"), {force: true})
                await spawnAndWait("yarn", ["--cwd", projectPath, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
                await spawnAndWait("yarn", ["--cwd", projectPath, "build"], {stdio: "inherit"})
            } else if (example === "create-react-app") {
                const projectPath = path.join(EXAMPLES_DIR, example)
                await removeExistingAutomerge(projectPath)
                await fsPromises.rm(path.join(projectPath, "yarn.lock"), {force: true})
                await spawnAndWait("yarn", ["--cwd", projectPath, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
                await spawnAndWait("yarn", ["--cwd", projectPath, "build"], {stdio: "inherit"})
            }
        }
    })
}

type WithRegistryAction = (registryUrl: string) => Promise<void>

async function withRegistry(action: WithRegistryAction, ...actions: Array<WithRegistryAction>) {
    // First, start verdaccio
    printHeader("Starting verdaccio NPM server")
    const verd = await VerdaccioProcess.start()
    actions.unshift(action)

    for (const action of actions) {
        try {
            type Step = "verd-died" | "action-completed"
            const verdDied: () => Promise<Step> = async () => {
                await verd.died()
                return "verd-died"
            }
            const actionComplete: () => Promise<Step> = async () => {
                await action("http://localhost:4873")
                return "action-completed"
            }
            const result = await Promise.race([verdDied(), actionComplete()])
            if (result === "verd-died") {
                throw new Error("verdaccio unexpectedly exited")
            }
        } catch(e) {
            await verd.kill()
            throw e
        }
    }
    await verd.kill()
}

async function withPublishedWasm(profile: Profile, action: WithRegistryAction) {
    await withRegistry(
        buildAutomergeWasm(profile),
        publishAutomergeWasm,
        action
    )
}

function buildAutomergeWasm(profile: Profile): WithRegistryAction {
    return async (registryUrl: string) => {
        printHeader("building automerge-wasm")
        await spawnAndWait("yarn", ["--cwd", AUTOMERGE_WASM_PATH, "--registry", registryUrl, "install"], {stdio: "inherit"})
        const cmd = profile === "release" ? "release" : "debug"
        await spawnAndWait("yarn", ["--cwd", AUTOMERGE_WASM_PATH, cmd], {stdio: "inherit"})
    }
}

async function publishAutomergeWasm(registryUrl: string) {
    printHeader("Publishing automerge-wasm to verdaccio")
    await fsPromises.rm(path.join(VERDACCIO_DB_PATH, "@automerge/automerge-wasm"), { recursive: true, force: true} )
    await yarnPublish(registryUrl, AUTOMERGE_WASM_PATH)
}

async function buildAndPublishAutomergeJs(registryUrl: string) {
    // Build the js package
    printHeader("Building automerge")
    await removeExistingAutomerge(AUTOMERGE_JS_PATH)
    await removeFromVerdaccio("@automerge/automerge")
    await fsPromises.rm(path.join(AUTOMERGE_JS_PATH, "yarn.lock"), {force: true})
    await spawnAndWait("yarn", ["--cwd", AUTOMERGE_JS_PATH, "install", "--registry", registryUrl, "--check-files"], {stdio: "inherit"})
    await spawnAndWait("yarn", ["--cwd", AUTOMERGE_JS_PATH, "build"], {stdio: "inherit"})
    await yarnPublish(registryUrl, AUTOMERGE_JS_PATH)
}

/**
 * A running verdaccio process
 *
 */
class VerdaccioProcess {
    child: ChildProcess
    stdout: Array<Buffer>
    stderr: Array<Buffer>

    constructor(child: ChildProcess) {
        this.child = child

        // Collect stdout/stderr otherwise the subprocess gets blocked writing
        this.stdout = []
        this.stderr = []
        this.child.stdout && this.child.stdout.on("data", (data) => this.stdout.push(data))
        this.child.stderr && this.child.stderr.on("data", (data) => this.stderr.push(data))

        const errCallback = (e: any) => {
            console.error("!!!!!!!!!ERROR IN VERDACCIO PROCESS!!!!!!!!!")
            console.error("    ", e)
            if (this.stdout.length > 0) {
                console.log("\n**Verdaccio stdout**")
                const stdout = Buffer.concat(this.stdout)
                process.stdout.write(stdout)
            }

            if (this.stderr.length > 0) {
                console.log("\n**Verdaccio stderr**")
                const stdout = Buffer.concat(this.stderr)
                process.stdout.write(stdout)
            }
            process.exit(-1)
        }
        this.child.on("error", errCallback)
    }

    /**
     * Spawn a verdaccio process and wait for it to respond succesfully to http requests
     *
     * The returned `VerdaccioProcess` can be used to control the subprocess
     */
    static async start() {
        const child = spawn("yarn", ["verdaccio", "--config", VERDACCIO_CONFIG_PATH], {env: { ...process.env, FORCE_COLOR: "true"}})

        // Forward stdout and stderr whilst waiting for startup to complete
        const stdoutCallback = (data: Buffer) => process.stdout.write(data)
        const stderrCallback = (data: Buffer) => process.stderr.write(data)
        child.stdout && child.stdout.on("data", stdoutCallback)
        child.stderr && child.stderr.on("data", stderrCallback)

        const healthCheck = async () => {
            while (true) {
                try {
                    const resp = await fetch("http://localhost:4873")
                    if (resp.status === 200) {
                        return
                    } else {
                        console.log(`Healthcheck failed: bad status ${resp.status}`)
                    }
                } catch (e) {
                    console.error(`Healthcheck failed: ${e}`)
                }
                await setTimeout(500)
            }
        }
        await withTimeout(healthCheck(), 10000)

        // Stop forwarding stdout/stderr
        child.stdout && child.stdout.off("data", stdoutCallback)
        child.stderr && child.stderr.off("data", stderrCallback)
        return new VerdaccioProcess(child)
    }

    /**
     * Send a SIGKILL to the process and wait for it to stop
     */
    async kill() {
        this.child.stdout && this.child.stdout.destroy()
        this.child.stderr && this.child.stderr.destroy()
        this.child.kill();
        try {
            await withTimeout(once(this.child, "close"), 500)
        } catch (e) {
            console.error("unable to kill verdaccio subprocess, trying -9")
            this.child.kill(9)
            await withTimeout(once(this.child, "close"), 500)
        }
    }

    /**
     * A promise which resolves if the subprocess exits for some reason
     */
    async died(): Promise<number | null> {
        const [exit, _signal] = await once(this.child, "exit")
        return exit
    }
}

function printHeader(header: string) {
    console.log("\n===============================")
    console.log(`           ${header}`)
    console.log("===============================")
}

/**
 * Removes the automerge, @automerge/automerge-wasm, and @automerge/automerge packages from
 * `$packageDir/node_modules`
 *
 * This is useful to force refreshing a package by use in combination with
 * `yarn install --check-files`, which checks if a package is present in
 * `node_modules` and if it is not forces a reinstall.
 *
 * @param packageDir - The directory containing the package.json of the target project
 */
async function removeExistingAutomerge(packageDir: string) {
    await fsPromises.rm(path.join(packageDir, "node_modules", "@automerge"), {recursive: true, force: true})
    await fsPromises.rm(path.join(packageDir, "node_modules", "automerge"), {recursive: true, force: true})
}

type SpawnResult = {
    stdout?: Buffer,
    stderr?: Buffer,
}

async function spawnAndWait(cmd: string, args: Array<string>, options: child_process.SpawnOptions): Promise<SpawnResult> {
    const child = spawn(cmd, args, options)
    let stdout = null
    let stderr = null
    if (child.stdout) {
        stdout = []
        child.stdout.on("data", data => stdout.push(data))
    }
    if (child.stderr) {
        stderr = []
        child.stderr.on("data", data => stderr.push(data))
    }

    const [exit, _signal] = await once(child, "exit")
    if (exit && exit !== 0) {
        throw new Error("nonzero exit code")
    }
    return {
        stderr: stderr? Buffer.concat(stderr) : null,
        stdout: stdout ? Buffer.concat(stdout) : null
    }
}

/**
 * Remove a package from the verdaccio registry. This is necessary because we
 * often want to _replace_ a version rather than update the version number.
 * Obviously this is very bad and verboten in normal circumastances, but the
 * whole point here is to be able to test the entire packaging story so it's
 * okay I Promise.
 */
async function removeFromVerdaccio(packageName: string) {
    await fsPromises.rm(path.join(VERDACCIO_DB_PATH, packageName), {force: true, recursive: true})
}

async function yarnPublish(registryUrl: string, cwd: string) {
    await spawnAndWait(
        "yarn",
        [
            "--registry",
            registryUrl,
            "--cwd",
            cwd,
            "publish",
            "--non-interactive",
        ],
        {
            stdio: "inherit",
            env: {
                ...process.env,
                FORCE_COLOR: "true",
                // This is a fake token, it just has to be the right format
                npm_config__auth: "//localhost:4873/:_authToken=Gp2Mgxm4faa/7wp0dMSuRA=="
            }
        })
}

/**
 * Wait for a given delay to resolve a promise, throwing an error if the
 * promise doesn't resolve with the timeout
 *
 * @param promise - the promise to wait for @param timeout - the delay in
 * milliseconds to wait before throwing
 */
async function withTimeout<T>(promise: Promise<T>, timeout: number): Promise<T> {
    type Step = "timed-out" | {result: T}
    const timedOut: () => Promise<Step> = async () => {
        await setTimeout(timeout)
        return "timed-out"
    }
    const succeeded: () => Promise<Step> = async () => {
        const result = await promise
        return {result}
    }
    const result = await Promise.race([timedOut(), succeeded()])
    if (result === "timed-out") {
        throw new Error("timed out")
    } else {
        return result.result
    }
}