ThetaDev/automerge
1
0
Fork 0
Code Issues Pull requests Projects Releases Activity

Merge remote-tracking branch 'origin/experiment' into getnerate-patches

Browse source
This commit is contained in:
Orion Henry 2022-03-30 13:04:51 -06:00
commit ab580df947
26 changed files with 433 additions and 218 deletions

2
automerge-cli/src/examine.rs
View file

@ -35,7 +35,7 @@ pub fn examine(
if is_tty {
let json_changes = serde_json::to_value(uncompressed_changes).unwrap();
colored_json::write_colored_json(&json_changes, &mut output).unwrap();
writeln!(&mut output).unwrap();
writeln!(output).unwrap();
} else {
let json_changes = serde_json::to_string_pretty(&uncompressed_changes).unwrap();
output

12
automerge-cli/src/import.rs
View file

@ -3,14 +3,14 @@ use automerge::transaction::Transactable;
pub(crate) fn initialize_from_json(
json_value: &serde_json::Value,
) -> Result<am::AutoCommit, am::AutomergeError> {
) -> anyhow::Result<am::AutoCommit> {
let mut doc = am::AutoCommit::new();
match json_value {
serde_json::Value::Object(m) => {
import_map(&mut doc, &am::ObjId::Root, m)?;
Ok(doc)
}
_ => Err(am::AutomergeError::Decoding),
_ => anyhow::bail!("expected an object"),
}
}
@ -18,7 +18,7 @@ fn import_map(
doc: &mut am::AutoCommit,
obj: &am::ObjId,
map: &serde_json::Map<String, serde_json::Value>,
) -> Result<(), am::AutomergeError> {
) -> anyhow::Result<()> {
for (key, value) in map {
match value {
serde_json::Value::Null => {
@ -42,7 +42,7 @@ fn import_map(
} else if let Some(m) = n.as_f64() {
doc.set(obj, key, m)?;
} else {
return Err(am::AutomergeError::Decoding);
anyhow::bail!("not a number");
}
}
serde_json::Value::Object(map) => {
@ -58,7 +58,7 @@ fn import_list(
doc: &mut am::AutoCommit,
obj: &am::ObjId,
list: &[serde_json::Value],
) -> Result<(), am::AutomergeError> {
) -> anyhow::Result<()> {
for (i, value) in list.iter().enumerate() {
match value {
serde_json::Value::Null => {
@ -82,7 +82,7 @@ fn import_list(
} else if let Some(m) = n.as_f64() {
doc.insert(obj, i, m)?;
} else {
return Err(am::AutomergeError::Decoding);
anyhow::bail!("not a number");
}
}
serde_json::Value::Object(map) => {

1
automerge-wasm/index.d.ts vendored
View file

@ -134,6 +134,7 @@ export class Automerge {
// low level change functions
applyChanges(changes: Change[]): void;
getChanges(have_deps: Heads): Change[];
getChangeByHash(hash: Hash): Change | null;
getChangesAdded(other: Automerge): Change[];
getHeads(): Heads;
getLastLocalChange(): Change;

11
automerge-wasm/src/lib.rs
View file

@ -454,6 +454,17 @@ impl Automerge {
Ok(changes)
}
#[wasm_bindgen(js_name = getChangeByHash)]
pub fn get_change_by_hash(&mut self, hash: JsValue) -> Result<JsValue, JsValue> {
let hash = hash.into_serde().map_err(to_js_err)?;
let change = self.0.get_change_by_hash(&hash);
if let Some(c) = change {
Ok(Uint8Array::from(c.raw_bytes()).into())
} else {
Ok(JsValue::null())
}
}
#[wasm_bindgen(js_name = getChangesAdded)]
pub fn get_changes_added(&mut self, other: &mut Automerge) -> Result<Array, JsValue> {
let changes = self.0.get_changes_added(&mut other.0);

14
automerge-wasm/test/test.ts
View file

@ -376,6 +376,20 @@ describe('Automerge', () => {
assert.deepEqual(doc.materialize("/list/0"), { foo: "bar"})
})
it('should be able to fetch changes by hash', () => {
let doc1 = create("aaaa")
let doc2 = create("bbbb")
doc1.set("/","a","b")
doc2.set("/","b","c")
let head1 = doc1.getHeads()
let head2 = doc2.getHeads()
let change1 = doc1.getChangeByHash(head1[0])
let change2 = doc1.getChangeByHash(head2[0])
assert.deepEqual(change2, null)
if (change1 === null) { throw new RangeError("change1 should not be null") }
assert.deepEqual(decodeChange(change1).hash, head1[0])
})
it('recursive sets are possible', () => {
let doc = create("aaaa")
let l1 = doc.set_object("_root","list",[{ foo: "bar"}, [1,2,3]])

79
automerge/src/autocommit.rs
View file

@ -1,11 +1,11 @@
use crate::exid::ExId;
use crate::transaction::{CommitOptions, Transactable};
use crate::types::Patch;
use crate::{
change::export_change, transaction::TransactionInner, ActorId, Automerge, AutomergeError,
Change, ChangeHash, Prop, Value,
};
use crate::{sync, Keys, KeysAt, ObjType, ScalarValue};
use crate::{
transaction::TransactionInner, ActorId, Automerge, AutomergeError, Change, ChangeHash, Prop,
Value,
};
/// An automerge document that automatically manages transactions.
#[derive(Debug, Clone)]
@ -61,67 +61,10 @@ impl AutoCommit {
fn ensure_transaction_open(&mut self) {
if self.transaction.is_none() {
let actor = self.doc.get_actor_index();
let seq = self.doc.states.entry(actor).or_default().len() as u64 + 1;
let mut deps = self.doc.get_heads();
if seq > 1 {
let last_hash = self.get_hash(actor, seq - 1).unwrap();
if !deps.contains(&last_hash) {
deps.push(last_hash);
}
}
self.transaction = Some(TransactionInner {
actor,
seq,
start_op: self.doc.max_op + 1,
time: 0,
message: None,
extra_bytes: Default::default(),
hash: None,
operations: vec![],
deps,
});
self.transaction = Some(self.doc.transaction_inner());
}
}
fn get_hash(&mut self, actor: usize, seq: u64) -> Result<ChangeHash, AutomergeError> {
self.doc
.states
.get(&actor)
.and_then(|v| v.get(seq as usize - 1))
.and_then(|&i| self.doc.history.get(i))
.map(|c| c.hash)
.ok_or(AutomergeError::InvalidSeq(seq))
}
fn update_history(&mut self, change: Change) -> usize {
self.doc.max_op = std::cmp::max(self.doc.max_op, change.start_op + change.len() as u64 - 1);
self.update_deps(&change);
let history_index = self.doc.history.len();
self.doc
.states
.entry(self.doc.ops.m.actors.cache(change.actor_id().clone()))
.or_default()
.push(history_index);
self.doc.history_index.insert(change.hash, history_index);
self.doc.history.push(change);
history_index
}
fn update_deps(&mut self, change: &Change) {
for d in &change.deps {
self.doc.deps.remove(d);
}
self.doc.deps.insert(change.hash);
}
pub fn fork(&mut self) -> Self {
self.ensure_transaction_closed();
Self {
@ -132,11 +75,7 @@ impl AutoCommit {
fn ensure_transaction_closed(&mut self) {
if let Some(tx) = self.transaction.take() {
self.update_history(export_change(
tx,
&self.doc.ops.m.actors,
&self.doc.ops.m.props,
));
tx.commit(&mut self.doc, None, None);
}
}
@ -238,10 +177,7 @@ impl AutoCommit {
}
pub fn commit(&mut self) -> ChangeHash {
// ensure that even no changes triggers a change
self.ensure_transaction_open();
let tx = self.transaction.take().unwrap();
tx.commit(&mut self.doc, None, None)
self.commit_with(CommitOptions::default())
}
/// Commit the current operations with some options.
@ -260,6 +196,7 @@ impl AutoCommit {
/// doc.commit_with(CommitOptions::default().with_message("Create todos list").with_time(now));
/// ```
pub fn commit_with(&mut self, options: CommitOptions) -> ChangeHash {
// ensure that even no changes triggers a change
self.ensure_transaction_open();
let tx = self.transaction.take().unwrap();
tx.commit(&mut self.doc, options.message, options.time)

232
automerge/src/automerge.rs
View file

@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet, VecDeque};
use std::num::NonZeroU64;
use crate::change::encode_document;
use crate::exid::ExId;
@ -15,7 +16,7 @@ use crate::{AutomergeError, Change, Prop};
use serde::Serialize;
use std::cmp::Ordering;
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Actor {
Unused(ActorId),
Cached(usize),
@ -24,14 +25,23 @@ pub(crate) enum Actor {
/// An automerge document.
#[derive(Debug, Clone)]
pub struct Automerge {
/// The list of unapplied changes that are not causally ready.
pub(crate) queue: Vec<Change>,
/// The history of changes that form this document, topologically sorted too.
pub(crate) history: Vec<Change>,
/// Mapping from change hash to index into the history list.
pub(crate) history_index: HashMap<ChangeHash, usize>,
/// Mapping from actor index to list of seqs seen for them.
pub(crate) states: HashMap<usize, Vec<usize>>,
/// Current dependencies of this document (heads hashes).
pub(crate) deps: HashSet<ChangeHash>,
/// Heads at the last save.
pub(crate) saved: Vec<ChangeHash>,
/// The set of operations that form this document.
pub(crate) ops: OpSet,
/// The current actor.
pub(crate) actor: Actor,
/// The maximum operation counter this document has seen.
pub(crate) max_op: u64,
pub(crate) patches: Option<Vec<Patch>>,
}
@ -107,6 +117,13 @@ impl Automerge {
/// Start a transaction.
pub fn transaction(&mut self) -> Transaction {
Transaction {
inner: Some(self.transaction_inner()),
doc: self,
}
}
pub(crate) fn transaction_inner(&mut self) -> TransactionInner {
let actor = self.get_actor_index();
let seq = self.states.get(&actor).map_or(0, |v| v.len()) as u64 + 1;
let mut deps = self.get_heads();
@ -117,20 +134,17 @@ impl Automerge {
}
}
let tx_inner = TransactionInner {
TransactionInner {
actor,
seq,
start_op: self.max_op + 1,
// SAFETY: this unwrap is safe as we always add 1
start_op: NonZeroU64::new(self.max_op + 1).unwrap(),
time: 0,
message: None,
extra_bytes: Default::default(),
hash: None,
operations: vec![],
deps,
};
Transaction {
inner: Some(tx_inner),
doc: self,
}
}
@ -319,6 +333,7 @@ impl Automerge {
}
}
/// Get the type of this object, if it is an object.
pub fn object_type<O: AsRef<ExId>>(&self, obj: O) -> Option<ObjType> {
let obj = self.exid_to_obj(obj.as_ref()).ok()?;
self.ops.object_type(&obj)
@ -570,7 +585,7 @@ impl Automerge {
.enumerate()
.map(|(i, c)| {
let actor = self.ops.m.actors.cache(change.actor_id().clone());
let id = OpId(change.start_op + i as u64, actor);
let id = OpId(change.start_op.get() + i as u64, actor);
let obj = match c.obj {
legacy::ObjectId::Root => ObjId::root(),
legacy::ObjectId::Id(id) => ObjId(OpId(id.0, self.ops.m.actors.cache(id.1))),
@ -866,7 +881,7 @@ impl Automerge {
}
pub(crate) fn update_history(&mut self, change: Change, num_ops: usize) -> usize {
self.max_op = std::cmp::max(self.max_op, change.start_op + num_ops as u64 - 1);
self.max_op = std::cmp::max(self.max_op, change.start_op.get() + num_ops as u64 - 1);
self.update_deps(&change);
@ -989,6 +1004,7 @@ mod tests {
use pretty_assertions::assert_eq;
use super::*;
use crate::op_set::B;
use crate::transaction::Transactable;
use crate::*;
use std::convert::TryInto;
@ -1426,4 +1442,202 @@ mod tests {
let doc = Automerge::load(&bytes).unwrap();
assert_eq!(doc.get_change_by_hash(&hash).unwrap().hash, hash);
}
#[test]
fn load_change_with_zero_start_op() {
let bytes = &[
133, 111, 74, 131, 202, 50, 52, 158, 2, 96, 163, 163, 83, 255, 255, 255, 50, 50, 50,
50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 255, 255, 245, 53, 1, 0, 0, 0, 0, 0, 0, 4,
233, 245, 239, 255, 1, 0, 0, 0, 133, 111, 74, 131, 163, 96, 0, 0, 2, 10, 202, 144, 125,
19, 48, 89, 133, 49, 10, 10, 67, 91, 111, 10, 74, 131, 96, 0, 163, 131, 255, 255, 255,
255, 255, 255, 255, 255, 255, 1, 153, 0, 0, 246, 255, 255, 255, 157, 157, 157, 157,
157, 157, 157, 157, 157, 157, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 48, 254, 208,
];
let _ = Automerge::load(bytes);
}
#[test]
fn load_broken_list() {
enum Action {
InsertText(usize, char),
DelText(usize),
}
use Action::*;
let actions = [
InsertText(0, 'a'),
InsertText(0, 'b'),
DelText(1),
InsertText(0, 'c'),
DelText(1),
DelText(0),
InsertText(0, 'd'),
InsertText(0, 'e'),
InsertText(1, 'f'),
DelText(2),
DelText(1),
InsertText(0, 'g'),
DelText(1),
DelText(0),
InsertText(0, 'h'),
InsertText(1, 'i'),
DelText(1),
DelText(0),
InsertText(0, 'j'),
InsertText(0, 'k'),
DelText(1),
DelText(0),
InsertText(0, 'l'),
DelText(0),
InsertText(0, 'm'),
InsertText(0, 'n'),
DelText(1),
DelText(0),
InsertText(0, 'o'),
DelText(0),
InsertText(0, 'p'),
InsertText(1, 'q'),
InsertText(1, 'r'),
InsertText(1, 's'),
InsertText(3, 't'),
InsertText(5, 'u'),
InsertText(0, 'v'),
InsertText(3, 'w'),
InsertText(4, 'x'),
InsertText(0, 'y'),
InsertText(6, 'z'),
InsertText(11, '1'),
InsertText(0, '2'),
InsertText(0, '3'),
InsertText(0, '4'),
InsertText(13, '5'),
InsertText(11, '6'),
InsertText(17, '7'),
];
let mut doc = Automerge::new();
let mut tx = doc.transaction();
let list = tx.set_object(ROOT, "list", ObjType::List).unwrap();
for action in actions {
match action {
Action::InsertText(index, c) => {
println!("inserting {} at {}", c, index);
tx.insert(&list, index, c).unwrap();
}
Action::DelText(index) => {
println!("deleting at {} ", index);
tx.del(&list, index).unwrap();
}
}
}
tx.commit();
let bytes = doc.save();
println!("doc2 time");
let mut doc2 = Automerge::load(&bytes).unwrap();
let bytes2 = doc2.save();
assert_eq!(doc.text(&list).unwrap(), doc2.text(&list).unwrap());
assert_eq!(doc.queue, doc2.queue);
assert_eq!(doc.history, doc2.history);
assert_eq!(doc.history_index, doc2.history_index);
assert_eq!(doc.states, doc2.states);
assert_eq!(doc.deps, doc2.deps);
assert_eq!(doc.saved, doc2.saved);
assert_eq!(doc.ops, doc2.ops);
assert_eq!(doc.max_op, doc2.max_op);
assert_eq!(bytes, bytes2);
}
#[test]
fn load_broken_list_short() {
// breaks when the B constant in OpSet is 3
enum Action {
InsertText(usize, char),
DelText(usize),
}
use Action::*;
let actions = [
InsertText(0, 'a'),
InsertText(1, 'b'),
DelText(1),
InsertText(1, 'c'),
InsertText(2, 'd'),
InsertText(2, 'e'),
InsertText(0, 'f'),
DelText(4),
InsertText(4, 'g'),
];
let mut doc = Automerge::new();
let mut tx = doc.transaction();
let list = tx.set_object(ROOT, "list", ObjType::List).unwrap();
for action in actions {
match action {
Action::InsertText(index, c) => {
println!("inserting {} at {}", c, index);
tx.insert(&list, index, c).unwrap();
}
Action::DelText(index) => {
println!("deleting at {} ", index);
tx.del(&list, index).unwrap();
}
}
}
tx.commit();
let bytes = doc.save();
println!("doc2 time");
let mut doc2 = Automerge::load(&bytes).unwrap();
let bytes2 = doc2.save();
assert_eq!(doc.text(&list).unwrap(), doc2.text(&list).unwrap());
assert_eq!(doc.queue, doc2.queue);
assert_eq!(doc.history, doc2.history);
assert_eq!(doc.history_index, doc2.history_index);
assert_eq!(doc.states, doc2.states);
assert_eq!(doc.deps, doc2.deps);
assert_eq!(doc.saved, doc2.saved);
assert_eq!(doc.ops, doc2.ops);
assert_eq!(doc.max_op, doc2.max_op);
assert_eq!(bytes, bytes2);
}
#[test]
fn compute_list_indexes_correctly_when_list_element_is_split_across_tree_nodes() {
let max = B as u64 * 2;
let actor1 = ActorId::from(b"aaaa");
let mut doc1 = AutoCommit::new().with_actor(actor1.clone());
let actor2 = ActorId::from(b"bbbb");
let mut doc2 = AutoCommit::new().with_actor(actor2.clone());
let list = doc1.set_object(ROOT, "list", ObjType::List).unwrap();
doc1.insert(&list, 0, 0).unwrap();
doc2.load_incremental(&doc1.save_incremental()).unwrap();
for i in 1..=max {
doc1.set(&list, 0, i).unwrap()
}
for i in 1..=max {
doc2.set(&list, 0, i).unwrap()
}
let change1 = doc1.save_incremental();
let change2 = doc2.save_incremental();
doc2.load_incremental(&change1).unwrap();
doc1.load_incremental(&change2).unwrap();
assert_eq!(doc1.length(&list), 1);
assert_eq!(doc2.length(&list), 1);
assert_eq!(
doc1.values(&list, 0).unwrap(),
vec![
(max.into(), ExId::Id(max + 2, actor1.clone(), 0)),
(max.into(), ExId::Id(max + 2, actor2.clone(), 1))
]
);
assert_eq!(
doc2.values(&list, 0).unwrap(),
vec![
(max.into(), ExId::Id(max + 2, actor1, 0)),
(max.into(), ExId::Id(max + 2, actor2, 1))
]
);
assert!(doc1.value(&list, 1).unwrap().is_none());
assert!(doc2.value(&list, 1).unwrap().is_none());
}
}

10
automerge/src/change.rs
View file

@ -23,6 +23,7 @@ use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use std::fmt::Debug;
use std::io::{Read, Write};
use std::num::NonZeroU64;
use tracing::instrument;
const MAGIC_BYTES: [u8; 4] = [0x85, 0x6f, 0x4a, 0x83];
@ -319,8 +320,8 @@ pub struct Change {
pub hash: amp::ChangeHash,
/// The index of this change in the changes from this actor.
pub seq: u64,
/// The start operation index.
pub start_op: u64,
/// The start operation index. Starts at 1.
pub start_op: NonZeroU64,
/// The time that this change was committed.
pub time: i64,
/// The message of this change.
@ -357,7 +358,7 @@ impl Change {
}
pub fn max_op(&self) -> u64 {
self.start_op + (self.len() as u64) - 1
self.start_op.get() + (self.len() as u64) - 1
}
pub fn message(&self) -> Option<String> {
@ -928,7 +929,8 @@ fn doc_changes_to_uncompressed_changes<'a>(
actor_id: actors[change.actor].clone(),
seq: change.seq,
time: change.time,
start_op: change.max_op - change.ops.len() as u64 + 1,
// SAFETY: this unwrap is safe as we always add 1
start_op: NonZeroU64::new(change.max_op - change.ops.len() as u64 + 1).unwrap(),
hash: None,
message: change.message,
operations: change

2
automerge/src/columnar.rs
View file

@ -946,7 +946,7 @@ impl ChangeEncoder {
self.seq.append_value(change.seq);
// FIXME iterops.count is crazy slow
self.max_op
.append_value(change.start_op + change.iter_ops().count() as u64 - 1);
.append_value(change.start_op.get() + change.iter_ops().count() as u64 - 1);
self.time.append_value(change.time as u64);
self.message.append_value(change.message());
self.deps_num.append_value(change.deps.len());

10
automerge/src/decoding.rs
View file

@ -1,4 +1,5 @@
use core::fmt::Debug;
use std::num::NonZeroU64;
use std::{borrow::Cow, io, io::Read, str};
use crate::error;
@ -353,6 +354,15 @@ impl Decodable for u64 {
}
}
impl Decodable for NonZeroU64 {
fn decode<R>(bytes: &mut R) -> Option<Self>
where
R: Read,
{
NonZeroU64::new(leb128::read::unsigned(bytes).ok()?)
}
}
impl Decodable for Vec<u8> {
fn decode<R>(bytes: &mut R) -> Option<Self>
where

9
automerge/src/encoding.rs
View file

@ -3,6 +3,7 @@ use std::{
io,
io::{Read, Write},
mem,
num::NonZeroU64,
};
use flate2::{bufread::DeflateEncoder, Compression};
@ -240,7 +241,7 @@ where
}
pub(crate) trait Encodable {
fn encode_with_actors_to_vec(&self, actors: &mut Vec<ActorId>) -> io::Result<Vec<u8>> {
fn encode_with_actors_to_vec(&self, actors: &mut [ActorId]) -> io::Result<Vec<u8>> {
let mut buf = Vec::new();
self.encode_with_actors(&mut buf, actors)?;
Ok(buf)
@ -291,6 +292,12 @@ impl Encodable for u64 {
}
}
impl Encodable for NonZeroU64 {
fn encode<R: Write>(&self, buf: &mut R) -> io::Result<usize> {
leb128::write::unsigned(buf, self.get())
}
}
impl Encodable for f64 {
fn encode<R: Write>(&self, buf: &mut R) -> io::Result<usize> {
let bytes = self.to_le_bytes();

22
automerge/src/error.rs
View file

@ -1,6 +1,6 @@
use crate::decoding;
use crate::types::{ActorId, ScalarValue};
use crate::value::DataType;
use crate::{decoding, encoding};
use thiserror::Error;
#[derive(Error, Debug)]
@ -9,10 +9,10 @@ pub enum AutomergeError {
InvalidOpId(String),
#[error("obj id not from this document `{0}`")]
ForeignObjId(String),
#[error("there was an ecoding problem")]
Encoding,
#[error("there was a decoding problem")]
Decoding,
#[error("there was an encoding problem: {0}")]
Encoding(#[from] encoding::Error),
#[error("there was a decoding problem: {0}")]
Decoding(#[from] decoding::Error),
#[error("key must not be an empty string")]
EmptyStringKey,
#[error("invalid seq {0}")]
@ -25,18 +25,6 @@ pub enum AutomergeError {
Fail,
}
impl From<std::io::Error> for AutomergeError {
fn from(_: std::io::Error) -> Self {
AutomergeError::Encoding
}
}
impl From<decoding::Error> for AutomergeError {
fn from(_: decoding::Error) -> Self {
AutomergeError::Decoding
}
}
#[cfg(feature = "wasm")]
impl From<AutomergeError> for wasm_bindgen::JsValue {
fn from(err: AutomergeError) -> Self {

9
automerge/src/indexed_cache.rs
View file

@ -9,6 +9,15 @@ pub(crate) struct IndexedCache<T> {
lookup: HashMap<T, usize>,
}
impl<T> PartialEq for IndexedCache<T>
where
T: PartialEq,
{
fn eq(&self, other: &Self) -> bool {
self.cache == other.cache
}
}
impl<T> IndexedCache<T>
where
T: Clone + Eq + Hash + Ord,

6
automerge/src/legacy/mod.rs
View file

@ -1,6 +1,8 @@
mod serde_impls;
mod utility_impls;
use std::num::NonZeroU64;
pub(crate) use crate::types::{ActorId, ChangeHash, ObjType, OpType, ScalarValue};
pub(crate) use crate::value::DataType;
@ -246,9 +248,9 @@ pub struct Change {
pub hash: Option<ChangeHash>,
/// The index of this change in the changes from this actor.
pub seq: u64,
/// The start operation index.
/// The start operation index. Starts at 1.
#[serde(rename = "startOp")]
pub start_op: u64,
pub start_op: NonZeroU64,
/// The time that this change was committed.
pub time: i64,
/// The message of this change.

3
automerge/src/lib.rs
View file

@ -50,6 +50,9 @@ mod visualisation;
pub use crate::automerge::Automerge;
pub use autocommit::AutoCommit;
pub use change::Change;
pub use decoding::Error as DecodingError;
pub use decoding::InvalidChangeError;
pub use encoding::Error as EncodingError;
pub use error::AutomergeError;
pub use exid::ExId as ObjId;
pub use keys::Keys;

16
automerge/src/op_set.rs
View file

@ -11,10 +11,13 @@ use std::collections::HashMap;
pub(crate) const B: usize = 16;
pub(crate) type OpSet = OpSetInternal<B>;
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct OpSetInternal<const B: usize> {
/// The map of objects to their type and ops.
trees: HashMap<ObjId, (ObjType, OpTreeInternal<B>), FxBuildHasher>,
/// The number of operations in the opset.
length: usize,
/// Metadata about the operations in this opset.
pub m: OpSetMetadata,
}
@ -132,14 +135,7 @@ impl<'a, const B: usize> IntoIterator for &'a OpSetInternal<B> {
type IntoIter = Iter<'a, B>;
fn into_iter(self) -> Self::IntoIter {
let mut objs: Vec<_> = self.trees.keys().collect();
objs.sort_by(|a, b| self.m.lamport_cmp(a.0, b.0));
Iter {
inner: self,
index: 0,
objs,
sub_index: 0,
}
self.iter()
}
}
@ -170,7 +166,7 @@ impl<'a, const B: usize> Iterator for Iter<'a, B> {
}
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct OpSetMetadata {
pub actors: IndexedCache<ActorId>,
pub props: IndexedCache<String>,

43
automerge/src/query.rs
View file

@ -76,21 +76,26 @@ pub(crate) enum QueryResult {
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Index {
pub len: usize,
/// The map of visible elements to the number of operations targetting them.
pub visible: HashMap<ElemId, usize, FxBuildHasher>,
/// Set of opids found in this node and below.
pub ops: HashSet<OpId, FxBuildHasher>,
}
impl Index {
pub fn new() -> Self {
Index {
len: 0,
visible: Default::default(),
ops: Default::default(),
}
}
pub fn has(&self, e: &Option<ElemId>) -> bool {
/// Get the number of visible elements in this index.
pub fn visible_len(&self) -> usize {
self.visible.len()
}
pub fn has_visible(&self, e: &Option<ElemId>) -> bool {
if let Some(seen) = e {
self.visible.contains_key(seen)
} else {
@ -109,7 +114,6 @@ impl Index {
match (new.visible(), old.visible(), new.elemid()) {
(false, true, Some(elem)) => match self.visible.get(&elem).copied() {
Some(n) if n == 1 => {
self.len -= 1;
self.visible.remove(&elem);
}
Some(n) => {
@ -117,15 +121,7 @@ impl Index {
}
None => panic!("remove overun in index"),
},
(true, false, Some(elem)) => match self.visible.get(&elem).copied() {
Some(n) => {
self.visible.insert(elem, n + 1);
}
None => {
self.len += 1;
self.visible.insert(elem, 1);
}
},
(true, false, Some(elem)) => *self.visible.entry(elem).or_default() += 1,
_ => {}
}
}
@ -134,15 +130,7 @@ impl Index {
self.ops.insert(op.id);
if op.visible() {
if let Some(elem) = op.elemid() {
match self.visible.get(&elem).copied() {
Some(n) => {
self.visible.insert(elem, n + 1);
}
None => {
self.len += 1;
self.visible.insert(elem, 1);
}
}
*self.visible.entry(elem).or_default() += 1;
}
}
}
@ -153,7 +141,6 @@ impl Index {
if let Some(elem) = op.elemid() {
match self.visible.get(&elem).copied() {
Some(n) if n == 1 => {
self.len -= 1;
self.visible.remove(&elem);
}
Some(n) => {
@ -170,15 +157,7 @@ impl Index {
self.ops.insert(*id);
}
for (elem, n) in other.visible.iter() {
match self.visible.get(elem).cloned() {
None => {
self.visible.insert(*elem, 1);
self.len += 1;
}
Some(m) => {
self.visible.insert(*elem, m + n);
}
}
*self.visible.entry(*elem).or_default() += n;
}
}
}

46
automerge/src/query/insert.rs
View file

@ -5,18 +5,23 @@ use crate::types::{ElemId, Key, Op, HEAD};
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct InsertNth<const B: usize> {
pub(crate) struct InsertNth {
/// the index in the realised list that we want to insert at
target: usize,
/// the number of visible operations seen
seen: usize,
//pub pos: usize,
/// the number of operations (including non-visible) that we have seen
n: usize,
valid: Option<usize>,
/// last_seen is the target elemid of the last `seen` operation.
/// It is used to avoid double counting visible elements (which arise through conflicts) that are split across nodes.
last_seen: Option<ElemId>,
last_insert: Option<ElemId>,
last_valid_insert: Option<ElemId>,
}
impl<const B: usize> InsertNth<B> {
impl InsertNth {
pub fn new(target: usize) -> Self {
let (valid, last_valid_insert) = if target == 0 {
(Some(0), Some(HEAD))
@ -56,23 +61,32 @@ impl<const B: usize> InsertNth<B> {
}
}
impl<const B: usize> TreeQuery<B> for InsertNth<B> {
impl<const B: usize> TreeQuery<B> for InsertNth {
fn query_node(&mut self, child: &OpTreeNode<B>) -> QueryResult {
let mut num_vis = child.index.len;
if num_vis > 0 {
if child.index.has(&self.last_seen) {
num_vis -= 1;
}
if self.seen + num_vis >= self.target {
QueryResult::Descend
} else {
self.n += child.len();
self.seen += num_vis;
self.last_seen = child.last().elemid();
QueryResult::Next
}
// if this node has some visible elements then we may find our target within
let mut num_vis = child.index.visible_len();
if child.index.has_visible(&self.last_seen) {
num_vis -= 1;
}
if self.seen + num_vis >= self.target {
// our target is within this node
QueryResult::Descend
} else {
// our target is not in this node so try the next one
self.n += child.len();
self.seen += num_vis;
// We have updated seen by the number of visible elements in this index, before we skip it.
// We also need to keep track of the last elemid that we have seen (and counted as seen).
// We can just use the elemid of the last op in this node as either:
// - the insert was at a previous node and this is a long run of overwrites so last_seen should already be set correctly
// - the visible op is in this node and the elemid references it so it can be set here
// - the visible op is in a future node and so it will be counted as seen there
let last_elemid = child.last().elemid();
if child.index.has_visible(&last_elemid) {
self.last_seen = last_elemid;
}
QueryResult::Next
}
}

8
automerge/src/query/len.rs
View file

@ -3,19 +3,19 @@ use crate::query::{QueryResult, TreeQuery};
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Len<const B: usize> {
pub(crate) struct Len {
pub len: usize,
}
impl<const B: usize> Len<B> {
impl Len {
pub fn new() -> Self {
Len { len: 0 }
}
}
impl<const B: usize> TreeQuery<B> for Len<B> {
impl<const B: usize> TreeQuery<B> for Len {
fn query_node(&mut self, child: &OpTreeNode<B>) -> QueryResult {
self.len = child.index.len;
self.len = child.index.visible_len();
QueryResult::Finish
}
}

6
automerge/src/query/len_at.rs
View file

@ -3,7 +3,7 @@ use crate::types::{Clock, ElemId, Op};
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct LenAt<const B: usize> {
pub(crate) struct LenAt {
pub len: usize,
clock: Clock,
pos: usize,
@ -11,7 +11,7 @@ pub(crate) struct LenAt<const B: usize> {
window: VisWindow,
}
impl<const B: usize> LenAt<B> {
impl LenAt {
pub fn new(clock: Clock) -> Self {
LenAt {
clock,
@ -23,7 +23,7 @@ impl<const B: usize> LenAt<B> {
}
}
impl<const B: usize> TreeQuery<B> for LenAt<B> {
impl<const B: usize> TreeQuery<B> for LenAt {
fn query_element(&mut self, op: &Op) -> QueryResult {
if op.insert {
self.last = None;

55
automerge/src/query/nth.rs
View file

@ -5,17 +5,18 @@ use crate::types::{ElemId, Key, Op};
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Nth<const B: usize> {
pub(crate) struct Nth {
target: usize,
seen: usize,
/// last_seen is the target elemid of the last `seen` operation.
/// It is used to avoid double counting visible elements (which arise through conflicts) that are split across nodes.
last_seen: Option<ElemId>,
last_elem: Option<ElemId>,
pub ops: Vec<Op>,
pub ops_pos: Vec<usize>,
pub pos: usize,
}
impl<const B: usize> Nth<B> {
impl Nth {
pub fn new(target: usize) -> Self {
Nth {
target,
@ -24,12 +25,13 @@ impl<const B: usize> Nth<B> {
ops: vec![],
ops_pos: vec![],
pos: 0,
last_elem: None,
}
}
/// Get the key
pub fn key(&self) -> Result<Key, AutomergeError> {
if let Some(e) = self.last_elem {
// the query collects the ops so we can use that to get the key they all use
if let Some(e) = self.ops.first().and_then(|op| op.elemid()) {
Ok(Key::Seq(e))
} else {
Err(AutomergeError::InvalidIndex(self.target))
@ -37,26 +39,30 @@ impl<const B: usize> Nth<B> {
}
}
impl<const B: usize> TreeQuery<B> for Nth<B> {
impl<const B: usize> TreeQuery<B> for Nth {
fn query_node(&mut self, child: &OpTreeNode<B>) -> QueryResult {
let mut num_vis = child.index.len;
if num_vis > 0 {
// num vis is the number of keys in the index
// minus one if we're counting last_seen
// let mut num_vis = s.keys().count();
if child.index.has(&self.last_seen) {
num_vis -= 1;
}
if self.seen + num_vis > self.target {
QueryResult::Descend
} else {
self.pos += child.len();
self.seen += num_vis;
self.last_seen = child.last().elemid();
QueryResult::Next
}
let mut num_vis = child.index.visible_len();
if child.index.has_visible(&self.last_seen) {
num_vis -= 1;
}
if self.seen + num_vis > self.target {
QueryResult::Descend
} else {
// skip this node as no useful ops in it
self.pos += child.len();
self.seen += num_vis;
// We have updated seen by the number of visible elements in this index, before we skip it.
// We also need to keep track of the last elemid that we have seen (and counted as seen).
// We can just use the elemid of the last op in this node as either:
// - the insert was at a previous node and this is a long run of overwrites so last_seen should already be set correctly
// - the visible op is in this node and the elemid references it so it can be set here
// - the visible op is in a future node and so it will be counted as seen there
let last_elemid = child.last().elemid();
if child.index.has_visible(&last_elemid) {
self.last_seen = last_elemid;
}
QueryResult::Next
}
}
@ -65,13 +71,14 @@ impl<const B: usize> TreeQuery<B> for Nth<B> {
if element.insert {
if self.seen > self.target {
return QueryResult::Finish;
};
self.last_elem = element.elemid();
}
// we have a new potentially visible element so reset last_seen
self.last_seen = None
}
let visible = element.visible();
if visible && self.last_seen.is_none() {
self.seen += 1;
// we have a new visible element
self.last_seen = element.elemid()
}
if self.seen == self.target + 1 && visible {

9
automerge/src/query/nth_at.rs
View file

@ -3,19 +3,18 @@ use crate::types::{Clock, ElemId, Op};
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct NthAt<const B: usize> {
pub(crate) struct NthAt {
clock: Clock,
target: usize,
seen: usize,
last_seen: Option<ElemId>,
last_elem: Option<ElemId>,
window: VisWindow,
pub ops: Vec<Op>,
pub ops_pos: Vec<usize>,
pub pos: usize,
}
impl<const B: usize> NthAt<B> {
impl NthAt {
pub fn new(target: usize, clock: Clock) -> Self {
NthAt {
clock,
@ -25,19 +24,17 @@ impl<const B: usize> NthAt<B> {
ops: vec![],
ops_pos: vec![],
pos: 0,
last_elem: None,
window: Default::default(),
}
}
}
impl<const B: usize> TreeQuery<B> for NthAt<B> {
impl<const B: usize> TreeQuery<B> for NthAt {
fn query_element(&mut self, element: &Op) -> QueryResult {
if element.insert {
if self.seen > self.target {
return QueryResult::Finish;
};
self.last_elem = element.elemid();
self.last_seen = None
}
let visible = self.window.visible_at(element, self.pos, &self.clock);

15
automerge/src/query/seek_op.rs
View file

@ -5,14 +5,18 @@ use std::cmp::Ordering;
use std::fmt::Debug;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SeekOp<const B: usize> {
pub(crate) struct SeekOp {
/// the op we are looking for
op: Op,
/// The position to insert at
pub pos: usize,
/// The indices of ops that this op overwrites
pub succ: Vec<usize>,
/// whether a position has been found
found: bool,
}
impl<const B: usize> SeekOp<B> {
impl SeekOp {
pub fn new(op: &Op) -> Self {
SeekOp {
op: op.clone(),
@ -35,7 +39,7 @@ impl<const B: usize> SeekOp<B> {
}
}
impl<const B: usize> TreeQuery<B> for SeekOp<B> {
impl<const B: usize> TreeQuery<B> for SeekOp {
fn query_node_with_metadata(
&mut self,
child: &OpTreeNode<B>,
@ -45,7 +49,7 @@ impl<const B: usize> TreeQuery<B> for SeekOp<B> {
return QueryResult::Descend;
}
match self.op.key {
Key::Seq(e) if e == HEAD => {
Key::Seq(HEAD) => {
while self.pos < child.len() {
let op = child.get(self.pos).unwrap();
if op.insert && m.lamport_cmp(op.id, self.op.id) == Ordering::Less {
@ -56,7 +60,7 @@ impl<const B: usize> TreeQuery<B> for SeekOp<B> {
QueryResult::Finish
}
Key::Seq(e) => {
if self.found || child.index.ops.contains(&e.0) {
if child.index.ops.contains(&e.0) {
QueryResult::Descend
} else {
self.pos += child.len();
@ -94,6 +98,7 @@ impl<const B: usize> TreeQuery<B> for SeekOp<B> {
self.pos += 1;
QueryResult::Next
} else {
// we have already found the target
if self.op.overwrites(e) {
self.succ.push(self.pos);
}

4
automerge/src/query/seek_op_with_patch.rs
View file

@ -98,13 +98,13 @@ impl<const B: usize> TreeQuery<B> for SeekOpWithPatch<B> {
// elements it contains. However, it could happen that a visible element is
// split across two tree nodes. To avoid double-counting in this situation, we
// subtract one if the last visible element also appears in this tree node.
let mut num_vis = child.index.len;
let mut num_vis = child.index.visible_len();
if num_vis > 0 {
// FIXME: I think this is wrong: we should subtract one only if this
// subtree contains a *visible* (i.e. empty succs) operation for the list
// element with elemId `last_seen`; this will subtract one even if all
// values for this list element have been deleted in this subtree.
if child.index.has(&self.last_seen) {
if child.index.has_visible(&self.last_seen) {
num_vis -= 1;
}
self.seen += num_vis;

10
automerge/src/transaction/inner.rs
View file

@ -1,3 +1,5 @@
use std::num::NonZeroU64;
use crate::automerge::Actor;
use crate::exid::ExId;
use crate::query::{self, OpIdSearch};
@ -9,7 +11,7 @@ use crate::{AutomergeError, ObjType, OpType, ScalarValue};
pub struct TransactionInner {
pub(crate) actor: usize,
pub(crate) seq: u64,
pub(crate) start_op: u64,
pub(crate) start_op: NonZeroU64,
pub(crate) time: i64,
pub(crate) message: Option<String>,
pub(crate) extra_bytes: Vec<u8>,
@ -39,7 +41,7 @@ impl TransactionInner {
self.time = t;
}
let num_ops = self.operations.len();
let num_ops = self.pending_ops();
let change = export_change(self, &doc.ops.m.actors, &doc.ops.m.props);
let hash = change.hash;
doc.update_history(change, num_ops);
@ -56,7 +58,7 @@ impl TransactionInner {
doc.actor = Actor::Unused(actor);
}
let num = self.operations.len();
let num = self.pending_ops();
// remove in reverse order so sets are removed before makes etc...
for (obj, op) in self.operations.iter().rev() {
for pred_id in &op.pred {
@ -123,7 +125,7 @@ impl TransactionInner {
}
fn next_id(&mut self) -> OpId {
OpId(self.start_op + self.operations.len() as u64, self.actor)
OpId(self.start_op.get() + self.pending_ops() as u64, self.actor)
}
fn insert_local_op(

17
automerge/src/types.rs
View file

@ -90,6 +90,23 @@ impl From<Vec<u8>> for ActorId {
}
}
impl<const N: usize> From<[u8; N]> for ActorId {
fn from(array: [u8; N]) -> Self {
ActorId::from(&array)
}
}
impl<const N: usize> From<&[u8; N]> for ActorId {
fn from(slice: &[u8; N]) -> Self {
let inner = if let Ok(arr) = ArrayVec::try_from(slice.as_slice()) {
TinyVec::Inline(arr)
} else {
TinyVec::Heap(slice.to_vec())
};
ActorId(inner)
}
}
impl FromStr for ActorId {
type Err = error::InvalidActorId;