From 175596beee77d51dbff8083b3b9d77a2ed871bcb Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 12:03:10 +0100 Subject: [PATCH 01/19] Change trees to objects and use objectdata struct --- automerge/src/lib.rs | 1 + automerge/src/object_data.rs | 33 +++++++++++++++++++ automerge/src/op_set.rs | 63 ++++++++++++++++-------------------- 3 files changed, 62 insertions(+), 35 deletions(-) create mode 100644 automerge/src/object_data.rs diff --git a/automerge/src/lib.rs b/automerge/src/lib.rs index bb39de25..f4888506 100644 --- a/automerge/src/lib.rs +++ b/automerge/src/lib.rs @@ -67,6 +67,7 @@ mod indexed_cache; mod keys; mod keys_at; mod legacy; +mod object_data; mod op_observer; mod op_set; mod op_tree; diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs new file mode 100644 index 00000000..5d4ebdd1 --- /dev/null +++ b/automerge/src/object_data.rs @@ -0,0 +1,33 @@ +use crate::types::ObjId; +use crate::ObjType; + +use crate::op_tree::OpTreeInternal; + +/// Stores the data for an object. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ObjectData { + /// The type of this object. + pub(crate) typ: ObjType, + /// The operations pertaining to this object. + pub(crate) ops: OpTreeInternal, + /// The id of the parent object, root has no parent. + pub parent: Option, +} + +impl ObjectData { + pub fn root() -> Self { + ObjectData { + typ: ObjType::Map, + ops: Default::default(), + parent: None, + } + } + + pub fn new(typ: ObjType, parent: Option) -> Self { + ObjectData { + typ, + ops: Default::default(), + parent, + } + } +} diff --git a/automerge/src/op_set.rs b/automerge/src/op_set.rs index 7928fce9..fa371b72 100644 --- a/automerge/src/op_set.rs +++ b/automerge/src/op_set.rs @@ -1,7 +1,7 @@ use crate::clock::Clock; use crate::exid::ExId; use crate::indexed_cache::IndexedCache; -use crate::op_tree::OpTree; +use crate::object_data::ObjectData; use crate::query::{self, OpIdSearch, TreeQuery}; use crate::types::{self, ActorId, Key, ObjId, Op, OpId, OpType}; use crate::{ObjType, OpObserver}; @@ -14,8 +14,8 @@ pub(crate) type OpSet = OpSetInternal; #[derive(Debug, Clone, PartialEq)] pub(crate) struct OpSetInternal { - /// The map of objects to their type and ops. - trees: HashMap, + /// The map of objects to their data. + objects: HashMap, /// The number of operations in the opset. length: usize, /// Metadata about the operations in this opset. @@ -25,9 +25,9 @@ pub(crate) struct OpSetInternal { impl OpSetInternal { pub(crate) fn new() -> Self { let mut trees: HashMap<_, _, _> = Default::default(); - trees.insert(ObjId::root(), OpTree::new()); + trees.insert(ObjId::root(), ObjectData::root()); OpSetInternal { - trees, + objects: trees, length: 0, m: OpSetMetadata { actors: IndexedCache::new(), @@ -45,7 +45,7 @@ impl OpSetInternal { } pub(crate) fn iter(&self) -> Iter<'_> { - let mut objs: Vec<_> = self.trees.keys().collect(); + let mut objs: Vec<_> = self.objects.keys().collect(); objs.sort_by(|a, b| self.m.lamport_cmp(a.0, b.0)); Iter { inner: self, @@ -56,22 +56,22 @@ impl OpSetInternal { } pub(crate) fn parent_object(&self, obj: &ObjId) -> Option<(ObjId, Key)> { - let parent = self.trees.get(obj)?.parent?; + let parent = self.objects.get(obj)?.parent?; let key = self.search(&parent, OpIdSearch::new(obj.0)).key().unwrap(); Some((parent, key)) } pub(crate) fn keys(&self, obj: ObjId) -> Option> { - if let Some(tree) = self.trees.get(&obj) { - tree.internal.keys() + if let Some(object) = self.objects.get(&obj) { + object.ops.keys() } else { None } } pub(crate) fn keys_at(&self, obj: ObjId, clock: Clock) -> Option> { - if let Some(tree) = self.trees.get(&obj) { - tree.internal.keys_at(clock) + if let Some(object) = self.objects.get(&obj) { + object.ops.keys_at(clock) } else { None } @@ -82,7 +82,7 @@ impl OpSetInternal { obj: ObjId, range: R, ) -> Option> { - if let Some(tree) = self.trees.get(&obj) { + if let Some(tree) = self.objects.get(&obj) { tree.internal.range(range, &self.m) } else { None @@ -95,7 +95,7 @@ impl OpSetInternal { range: R, clock: Clock, ) -> Option> { - if let Some(tree) = self.trees.get(&obj) { + if let Some(tree) = self.objects.get(&obj) { tree.internal.range_at(range, &self.m, clock) } else { None @@ -106,8 +106,8 @@ impl OpSetInternal { where Q: TreeQuery<'a>, { - if let Some(tree) = self.trees.get(obj) { - tree.internal.search(query, &self.m) + if let Some(object) = self.objects.get(obj) { + object.ops.search(query, &self.m) } else { query } @@ -117,18 +117,18 @@ impl OpSetInternal { where F: FnMut(&mut Op), { - if let Some(tree) = self.trees.get_mut(obj) { - tree.internal.update(index, f) + if let Some(object) = self.objects.get_mut(obj) { + object.ops.update(index, f) } } pub(crate) fn remove(&mut self, obj: &ObjId, index: usize) -> Op { // this happens on rollback - be sure to go back to the old state - let tree = self.trees.get_mut(obj).unwrap(); + let object = self.objects.get_mut(obj).unwrap(); self.length -= 1; - let op = tree.internal.remove(index); + let op = object.ops.remove(index); if let OpType::Make(_) = &op.action { - self.trees.remove(&op.id.into()); + self.objects.remove(&op.id.into()); } op } @@ -139,19 +139,12 @@ impl OpSetInternal { pub(crate) fn insert(&mut self, index: usize, obj: &ObjId, element: Op) { if let OpType::Make(typ) = element.action { - self.trees.insert( - element.id.into(), - OpTree { - internal: Default::default(), - objtype: typ, - parent: Some(*obj), - }, - ); + self.objects + .insert(element.id.into(), ObjectData::new(typ, Some(*obj))); } - if let Some(tree) = self.trees.get_mut(obj) { - //let tree = self.trees.get_mut(&element.obj).unwrap(); - tree.internal.insert(index, element); + if let Some(object) = self.objects.get_mut(obj) { + object.ops.insert(index, element); self.length += 1; } } @@ -244,13 +237,13 @@ impl OpSetInternal { } pub(crate) fn object_type(&self, id: &ObjId) -> Option { - self.trees.get(id).map(|tree| tree.objtype) + self.objects.get(id).map(|object| object.typ) } #[cfg(feature = "optree-visualisation")] pub(crate) fn visualise(&self) -> String { let mut out = Vec::new(); - let graph = super::visualisation::GraphVisualisation::construct(&self.trees, &self.m); + let graph = super::visualisation::GraphVisualisation::construct(&self.objects, &self.m); dot::render(&graph, &mut out).unwrap(); String::from_utf8_lossy(&out[..]).to_string() } @@ -285,8 +278,8 @@ impl<'a> Iterator for Iter<'a> { fn next(&mut self) -> Option { let mut result = None; for obj in self.objs.iter().skip(self.index) { - let tree = self.inner.trees.get(obj)?; - result = tree.internal.get(self.sub_index).map(|op| (*obj, op)); + let object = self.inner.objects.get(obj)?; + result = object.ops.get(self.sub_index).map(|op| (*obj, op)); if result.is_some() { self.sub_index += 1; break; From 0d84123ad736662640ffd85bf9ae26e8e03be46c Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 12:34:19 +0100 Subject: [PATCH 02/19] Use methods on container --- automerge/src/object_data.rs | 48 +++++++++++++++++++++++++++++++++--- automerge/src/op_set.rs | 16 ++++++------ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 5d4ebdd1..b0b92237 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -1,15 +1,19 @@ +use crate::clock::Clock; +use crate::query::TreeQuery; use crate::types::ObjId; +use crate::types::Op; use crate::ObjType; +use crate::{query::Keys, query::KeysAt, ObjType}; -use crate::op_tree::OpTreeInternal; +use crate::op_tree::{OpSetMetadata, OpTreeInternal}; /// Stores the data for an object. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObjectData { /// The type of this object. - pub(crate) typ: ObjType, + typ: ObjType, /// The operations pertaining to this object. - pub(crate) ops: OpTreeInternal, + ops: OpTreeInternal, /// The id of the parent object, root has no parent. pub parent: Option, } @@ -30,4 +34,42 @@ impl ObjectData { parent, } } + + pub fn keys(&self) -> Option { + self.ops.keys() + } + + pub fn keys_at(&self, clock: Clock) -> Option { + self.ops.keys_at(clock) + } + + pub fn search(&self, query: Q, metadata: &OpSetMetadata) -> Q + where + Q: TreeQuery, + { + self.ops.search(query, metadata) + } + + pub fn replace(&mut self, index: usize, f: F) + where + F: FnMut(&mut Op), + { + self.ops.replace(index, f) + } + + pub fn remove(&mut self, index: usize) -> Op { + self.ops.remove(index) + } + + pub fn insert(&mut self, index: usize, op: Op) { + self.ops.insert(index, op) + } + + pub fn typ(&self) -> ObjType { + self.typ + } + + pub fn get(&self, index: usize) -> Option<&Op> { + self.ops.get(index) + } } diff --git a/automerge/src/op_set.rs b/automerge/src/op_set.rs index fa371b72..443397e4 100644 --- a/automerge/src/op_set.rs +++ b/automerge/src/op_set.rs @@ -63,7 +63,7 @@ impl OpSetInternal { pub(crate) fn keys(&self, obj: ObjId) -> Option> { if let Some(object) = self.objects.get(&obj) { - object.ops.keys() + object.keys() } else { None } @@ -71,7 +71,7 @@ impl OpSetInternal { pub(crate) fn keys_at(&self, obj: ObjId, clock: Clock) -> Option> { if let Some(object) = self.objects.get(&obj) { - object.ops.keys_at(clock) + object.keys_at(clock) } else { None } @@ -107,7 +107,7 @@ impl OpSetInternal { Q: TreeQuery<'a>, { if let Some(object) = self.objects.get(obj) { - object.ops.search(query, &self.m) + object.search(query, &self.m) } else { query } @@ -118,7 +118,7 @@ impl OpSetInternal { F: FnMut(&mut Op), { if let Some(object) = self.objects.get_mut(obj) { - object.ops.update(index, f) + object.update(index, f) } } @@ -126,7 +126,7 @@ impl OpSetInternal { // this happens on rollback - be sure to go back to the old state let object = self.objects.get_mut(obj).unwrap(); self.length -= 1; - let op = object.ops.remove(index); + let op = object.remove(index); if let OpType::Make(_) = &op.action { self.objects.remove(&op.id.into()); } @@ -144,7 +144,7 @@ impl OpSetInternal { } if let Some(object) = self.objects.get_mut(obj) { - object.ops.insert(index, element); + object.insert(index, element); self.length += 1; } } @@ -237,7 +237,7 @@ impl OpSetInternal { } pub(crate) fn object_type(&self, id: &ObjId) -> Option { - self.objects.get(id).map(|object| object.typ) + self.objects.get(id).map(|object| object.typ()) } #[cfg(feature = "optree-visualisation")] @@ -279,7 +279,7 @@ impl<'a> Iterator for Iter<'a> { let mut result = None; for obj in self.objs.iter().skip(self.index) { let object = self.inner.objects.get(obj)?; - result = object.ops.get(self.sub_index).map(|op| (*obj, op)); + result = object.get(self.sub_index).map(|op| (*obj, op)); if result.is_some() { self.sub_index += 1; break; From e2b44091e2f5f023bad1b5f09793c4ebe966ba06 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 12:48:18 +0100 Subject: [PATCH 03/19] Use an enum for Maps vs seqs on objectdata --- automerge/src/object_data.rs | 86 +++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index b0b92237..eac28ec0 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -7,29 +7,78 @@ use crate::{query::Keys, query::KeysAt, ObjType}; use crate::op_tree::{OpSetMetadata, OpTreeInternal}; +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum MapType { + Map, + Table, +} + +impl From for ObjType { + fn from(m: MapType) -> Self { + match m { + MapType::Map => ObjType::Map, + MapType::Table => ObjType::Table, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum SeqType { + List, + Text, +} + +impl From for ObjType { + fn from(s: SeqType) -> Self { + match s { + SeqType::List => ObjType::List, + SeqType::Text => ObjType::Text, + } + } +} + /// Stores the data for an object. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObjectData { - /// The type of this object. - typ: ObjType, + internal: ObjectDataInternal, /// The operations pertaining to this object. ops: OpTreeInternal, /// The id of the parent object, root has no parent. pub parent: Option, } +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ObjectDataInternal { + Map { + /// The type of this object. + typ: MapType, + }, + Seq { + /// The type of this object. + typ: SeqType, + }, +} + impl ObjectData { pub fn root() -> Self { ObjectData { - typ: ObjType::Map, + internal: ObjectDataInternal::Map { typ: MapType::Map }, ops: Default::default(), parent: None, } } pub fn new(typ: ObjType, parent: Option) -> Self { + let internal = match typ { + ObjType::Map => ObjectDataInternal::Map { typ: MapType::Map }, + ObjType::Table => ObjectDataInternal::Map { + typ: MapType::Table, + }, + ObjType::List => ObjectDataInternal::Seq { typ: SeqType::List }, + ObjType::Text => ObjectDataInternal::Seq { typ: SeqType::Text }, + }; ObjectData { - typ, + internal, ops: Default::default(), parent, } @@ -43,33 +92,50 @@ impl ObjectData { self.ops.keys_at(clock) } + fn ops(&self) -> &OpTreeInternal { + match self { + ObjectData::Map { typ: _, ops } => ops, + ObjectData::Seq { typ: _, ops } => ops, + } + } + + fn ops_mut(&mut self) -> &mut OpTreeInternal { + match self { + ObjectData::Map { typ: _, ops } => ops, + ObjectData::Seq { typ: _, ops } => ops, + } + } + pub fn search(&self, query: Q, metadata: &OpSetMetadata) -> Q where Q: TreeQuery, { - self.ops.search(query, metadata) + self.ops().search(query, metadata) } pub fn replace(&mut self, index: usize, f: F) where F: FnMut(&mut Op), { - self.ops.replace(index, f) + self.ops_mut().replace(index, f) } pub fn remove(&mut self, index: usize) -> Op { - self.ops.remove(index) + self.ops_mut().remove(index) } pub fn insert(&mut self, index: usize, op: Op) { - self.ops.insert(index, op) + self.ops_mut().insert(index, op) } pub fn typ(&self) -> ObjType { - self.typ + match self { + ObjectData::Map { typ, ops: _ } => (*typ).into(), + ObjectData::Seq { typ, ops: _ } => (*typ).into(), + } } pub fn get(&self, index: usize) -> Option<&Op> { - self.ops.get(index) + self.ops().get(index) } } From 148e52545b56e53c9ef804900358a72f8fb29e29 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 12:48:34 +0100 Subject: [PATCH 04/19] Group imports --- automerge/src/object_data.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index eac28ec0..e64ccc87 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -1,12 +1,11 @@ use crate::clock::Clock; +use crate::op_tree::{OpSetMetadata, OpTreeInternal}; use crate::query::TreeQuery; use crate::types::ObjId; use crate::types::Op; use crate::ObjType; use crate::{query::Keys, query::KeysAt, ObjType}; -use crate::op_tree::{OpSetMetadata, OpTreeInternal}; - #[derive(Debug, Clone, Copy, PartialEq)] pub(crate) enum MapType { Map, From 682f60c77421baa8cb6c360da64e42900139c953 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 12:50:48 +0100 Subject: [PATCH 05/19] Add cache structs --- automerge/src/object_data.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index e64ccc87..3ad79483 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -21,6 +21,9 @@ impl From for ObjType { } } +#[derive(Debug, Default, Clone, PartialEq)] +pub(crate) struct MapOpsCache {} + #[derive(Debug, Clone, Copy, PartialEq)] pub(crate) enum SeqType { List, @@ -36,6 +39,9 @@ impl From for ObjType { } } +#[derive(Debug, Default, Clone, PartialEq)] +pub(crate) struct SeqOpsCache {} + /// Stores the data for an object. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObjectData { @@ -51,17 +57,22 @@ pub(crate) enum ObjectDataInternal { Map { /// The type of this object. typ: MapType, + cache: MapOpsCache, }, Seq { /// The type of this object. typ: SeqType, + cache: SeqOpsCache, }, } impl ObjectData { pub fn root() -> Self { ObjectData { - internal: ObjectDataInternal::Map { typ: MapType::Map }, + internal: ObjectDataInternal::Map { + typ: MapType::Map, + cache: Default::default(), + }, ops: Default::default(), parent: None, } @@ -93,15 +104,15 @@ impl ObjectData { fn ops(&self) -> &OpTreeInternal { match self { - ObjectData::Map { typ: _, ops } => ops, - ObjectData::Seq { typ: _, ops } => ops, + ObjectData::Map { ops, .. } => ops, + ObjectData::Seq { ops, .. } => ops, } } fn ops_mut(&mut self) -> &mut OpTreeInternal { match self { - ObjectData::Map { typ: _, ops } => ops, - ObjectData::Seq { typ: _, ops } => ops, + ObjectData::Map { ops, .. } => ops, + ObjectData::Seq { ops, .. } => ops, } } @@ -129,8 +140,8 @@ impl ObjectData { pub fn typ(&self) -> ObjType { match self { - ObjectData::Map { typ, ops: _ } => (*typ).into(), - ObjectData::Seq { typ, ops: _ } => (*typ).into(), + ObjectData::Map { typ, .. } => (*typ).into(), + ObjectData::Seq { typ, .. } => (*typ).into(), } } From 49fad13843337a79b6955e5869b09e443a1eedae Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 13:35:02 +0100 Subject: [PATCH 06/19] Add caching infrastructure --- automerge/src/object_data.rs | 61 +++++++++++++++++++++++++++++++++--- automerge/src/query.rs | 19 +++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 3ad79483..7d8c8a15 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -1,3 +1,5 @@ +use std::sync::{Arc, Mutex}; + use crate::clock::Clock; use crate::op_tree::{OpSetMetadata, OpTreeInternal}; use crate::query::TreeQuery; @@ -57,15 +59,47 @@ pub(crate) enum ObjectDataInternal { Map { /// The type of this object. typ: MapType, - cache: MapOpsCache, + cache: Arc>, }, Seq { /// The type of this object. typ: SeqType, - cache: SeqOpsCache, + cache: Arc>, }, } +impl PartialEq for ObjectData { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::Map { + typ: l_typ, + ops: l_ops, + cache: _, + }, + Self::Map { + typ: r_typ, + ops: r_ops, + cache: _, + }, + ) => l_typ == r_typ && l_ops == r_ops, + ( + Self::Seq { + typ: l_typ, + ops: l_ops, + cache: _, + }, + Self::Seq { + typ: r_typ, + ops: r_ops, + cache: _, + }, + ) => l_typ == r_typ && l_ops == r_ops, + _ => false, + } + } +} + impl ObjectData { pub fn root() -> Self { ObjectData { @@ -116,11 +150,30 @@ impl ObjectData { } } - pub fn search(&self, query: Q, metadata: &OpSetMetadata) -> Q + pub fn search(&self, mut query: Q, metadata: &OpSetMetadata) -> Q where Q: TreeQuery, { - self.ops().search(query, metadata) + match self { + ObjectData::Map { cache, ops, .. } => { + let mut cache = cache.lock().unwrap(); + if !query.cache_lookup_map(&cache) { + let query = ops.search(query, metadata); + query.cache_update_map(&mut cache); + return query; + } + query + } + ObjectData::Seq { cache, ops, .. } => { + let mut cache = cache.lock().unwrap(); + if !query.cache_lookup_seq(&cache) { + let query = ops.search(query, metadata); + query.cache_update_seq(&mut cache); + return query; + } + query + } + } } pub fn replace(&mut self, index: usize, f: F) diff --git a/automerge/src/query.rs b/automerge/src/query.rs index 06225885..a98be65a 100644 --- a/automerge/src/query.rs +++ b/automerge/src/query.rs @@ -1,3 +1,4 @@ +use crate::object_data::{MapOpsCache, SeqOpsCache}; use crate::op_tree::{OpSetMetadata, OpTreeNode}; use crate::types::{Clock, Counter, ElemId, Op, OpId, OpType, ScalarValue}; use fxhash::FxBuildHasher; @@ -50,6 +51,24 @@ pub(crate) struct CounterData { } pub(crate) trait TreeQuery<'a> { + fn cache_lookup_map(&mut self, _cache: &MapOpsCache) -> bool { + // by default we haven't found something in the cache + false + } + + fn cache_update_map(&self, _cache: &mut MapOpsCache) { + // by default we don't have anything to update in the cache + } + + fn cache_lookup_seq(&mut self, _cache: &SeqOpsCache) -> bool { + // by default we haven't found something in the cache + false + } + + fn cache_update_seq(&self, _cache: &mut SeqOpsCache) { + // by default we don't have anything to update in the cache + } + #[inline(always)] fn query_node_with_metadata( &mut self, From 4dbe29ad8fcaf9e17df716c414259218e5a7a5a9 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 13:43:16 +0100 Subject: [PATCH 07/19] Add cache functionality on the caches --- automerge/src/object_data.rs | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 7d8c8a15..598cdd16 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -26,6 +26,17 @@ impl From for ObjType { #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct MapOpsCache {} +impl MapOpsCache { + fn lookup(&self, query: &mut Q) -> bool { + query.cache_lookup_map(self) + } + + fn update(&mut self, query: &Q) { + query.cache_update_map(self); + // TODO: fixup the cache (reordering etc.) + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub(crate) enum SeqType { List, @@ -42,7 +53,21 @@ impl From for ObjType { } #[derive(Debug, Default, Clone, PartialEq)] -pub(crate) struct SeqOpsCache {} +pub(crate) struct SeqOpsCache { + // list of previous insert positions + lru: Vec<(usize, usize)>, +} + +impl SeqOpsCache { + fn lookup(&self, query: &mut Q) -> bool { + query.cache_lookup_seq(self) + } + + fn update(&mut self, query: &Q) { + query.cache_update_seq(self); + // TODO: fixup the cache (reordering etc.) + } +} /// Stores the data for an object. #[derive(Debug, Clone, PartialEq)] @@ -157,18 +182,18 @@ impl ObjectData { match self { ObjectData::Map { cache, ops, .. } => { let mut cache = cache.lock().unwrap(); - if !query.cache_lookup_map(&cache) { + if !cache.lookup(&mut query) { let query = ops.search(query, metadata); - query.cache_update_map(&mut cache); + cache.update(&query); return query; } query } ObjectData::Seq { cache, ops, .. } => { let mut cache = cache.lock().unwrap(); - if !query.cache_lookup_seq(&cache) { + if !cache.lookup(&mut query) { let query = ops.search(query, metadata); - query.cache_update_seq(&mut cache); + cache.update(&query); return query; } query From 537af55d5cfd91701f2cfff3733c65437d7caa72 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 14:20:40 +0100 Subject: [PATCH 08/19] Fixup update --- automerge/src/object_data.rs | 6 +++--- automerge/src/op_set.rs | 2 +- automerge/src/op_tree.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 598cdd16..934fd8d3 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -201,11 +201,11 @@ impl ObjectData { } } - pub fn replace(&mut self, index: usize, f: F) + pub fn update(&mut self, index: usize, f: F) where - F: FnMut(&mut Op), + F: FnOnce(&mut Op), { - self.ops_mut().replace(index, f) + self.ops_mut().update(index, f) } pub fn remove(&mut self, index: usize) -> Op { diff --git a/automerge/src/op_set.rs b/automerge/src/op_set.rs index 443397e4..fc70e4f6 100644 --- a/automerge/src/op_set.rs +++ b/automerge/src/op_set.rs @@ -115,7 +115,7 @@ impl OpSetInternal { pub(crate) fn replace(&mut self, obj: &ObjId, index: usize, f: F) where - F: FnMut(&mut Op), + F: FnOnce(&mut Op), { if let Some(object) = self.objects.get_mut(obj) { object.update(index, f) diff --git a/automerge/src/op_tree.rs b/automerge/src/op_tree.rs index 908522d5..f820b58c 100644 --- a/automerge/src/op_tree.rs +++ b/automerge/src/op_tree.rs @@ -167,7 +167,7 @@ impl OpTreeInternal { // this replaces get_mut() because it allows the indexes to update correctly pub(crate) fn update(&mut self, index: usize, f: F) where - F: FnMut(&mut Op), + F: FnOnce(&mut Op), { if self.len() > index { self.root_node.as_mut().unwrap().update(index, f); From 0ae73981acb423819cc83125c8273af5cc238b31 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 14:51:03 +0100 Subject: [PATCH 09/19] Fixup visualisation --- automerge/src/visualisation.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/automerge/src/visualisation.rs b/automerge/src/visualisation.rs index 5e6dae6f..fbf387a0 100644 --- a/automerge/src/visualisation.rs +++ b/automerge/src/visualisation.rs @@ -1,3 +1,4 @@ +use crate::object_data::ObjectData; use crate::types::ObjId; use fxhash::FxHasher; use std::{borrow::Cow, collections::HashMap, hash::BuildHasherDefault}; @@ -42,16 +43,12 @@ pub(crate) struct GraphVisualisation<'a> { impl<'a> GraphVisualisation<'a> { pub(super) fn construct( - trees: &'a HashMap< - crate::types::ObjId, - crate::op_tree::OpTree, - BuildHasherDefault, - >, + objects: &'a HashMap>, metadata: &'a crate::op_set::OpSetMetadata, ) -> GraphVisualisation<'a> { let mut nodes = HashMap::new(); - for (obj_id, tree) in trees { - if let Some(root_node) = &tree.internal.root_node { + for (obj_id, object_data) in objects { + if let Some(root_node) = &object_data.ops().root_node { let tree_id = Self::construct_nodes(root_node, obj_id, &mut nodes, metadata); let obj_tree_id = NodeId::default(); nodes.insert( From bf546b2ab4a5f090a8813575da7c64ffbb2d0188 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 14:52:47 +0100 Subject: [PATCH 10/19] Cache inserts --- automerge/src/object_data.rs | 18 +++++++++--------- automerge/src/op_set.rs | 6 +++--- automerge/src/query/insert.rs | 24 ++++++++++++++++++++++-- automerge/src/transaction/inner.rs | 2 +- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 934fd8d3..29d64349 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -5,6 +5,7 @@ use crate::op_tree::{OpSetMetadata, OpTreeInternal}; use crate::query::TreeQuery; use crate::types::ObjId; use crate::types::Op; +use crate::types::{Op, OpId}; use crate::ObjType; use crate::{query::Keys, query::KeysAt, ObjType}; @@ -54,8 +55,9 @@ impl From for ObjType { #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SeqOpsCache { - // list of previous insert positions - lru: Vec<(usize, usize)>, + // last insertion (list index, tree index, opid to be inserted) + // TODO: invalidation + pub(crate) last: Option<(usize, usize, OpId)>, } impl SeqOpsCache { @@ -161,7 +163,7 @@ impl ObjectData { self.ops.keys_at(clock) } - fn ops(&self) -> &OpTreeInternal { + pub(crate) fn ops(&self) -> &OpTreeInternal { match self { ObjectData::Map { ops, .. } => ops, ObjectData::Seq { ops, .. } => ops, @@ -183,19 +185,17 @@ impl ObjectData { ObjectData::Map { cache, ops, .. } => { let mut cache = cache.lock().unwrap(); if !cache.lookup(&mut query) { - let query = ops.search(query, metadata); - cache.update(&query); - return query; + query = ops.search(query, metadata); } + cache.update(&query); query } ObjectData::Seq { cache, ops, .. } => { let mut cache = cache.lock().unwrap(); if !cache.lookup(&mut query) { - let query = ops.search(query, metadata); - cache.update(&query); - return query; + query = ops.search(query, metadata); } + cache.update(&query); query } } diff --git a/automerge/src/op_set.rs b/automerge/src/op_set.rs index fc70e4f6..8e22f1a8 100644 --- a/automerge/src/op_set.rs +++ b/automerge/src/op_set.rs @@ -24,10 +24,10 @@ pub(crate) struct OpSetInternal { impl OpSetInternal { pub(crate) fn new() -> Self { - let mut trees: HashMap<_, _, _> = Default::default(); - trees.insert(ObjId::root(), ObjectData::root()); + let mut objects: HashMap<_, _, _> = Default::default(); + objects.insert(ObjId::root(), ObjectData::root()); OpSetInternal { - objects: trees, + objects, length: 0, m: OpSetMetadata { actors: IndexedCache::new(), diff --git a/automerge/src/query/insert.rs b/automerge/src/query/insert.rs index 6f69474c..3fede336 100644 --- a/automerge/src/query/insert.rs +++ b/automerge/src/query/insert.rs @@ -1,13 +1,15 @@ use crate::error::AutomergeError; use crate::op_tree::OpTreeNode; use crate::query::{QueryResult, TreeQuery}; -use crate::types::{ElemId, Key, Op, HEAD}; +use crate::types::{ElemId, Key, Op, OpId, HEAD}; use std::fmt::Debug; #[derive(Debug, Clone, PartialEq)] pub(crate) struct InsertNth { /// the index in the realised list that we want to insert at target: usize, + /// OpId of the op we are trying to find a location for. + id: OpId, /// the number of visible operations seen seen: usize, //pub pos: usize, @@ -22,7 +24,7 @@ pub(crate) struct InsertNth { } impl InsertNth { - pub(crate) fn new(target: usize) -> Self { + pub(crate) fn new(target: usize, opid: OpId) -> Self { let (valid, last_valid_insert) = if target == 0 { (Some(0), Some(HEAD)) } else { @@ -30,6 +32,7 @@ impl InsertNth { }; InsertNth { target, + id: opid, seen: 0, n: 0, valid, @@ -62,6 +65,23 @@ impl InsertNth { } impl<'a> TreeQuery<'a> for InsertNth { + fn cache_lookup_seq(&mut self, cache: &crate::object_data::SeqOpsCache) -> bool { + if let Some((last_target_index, last_tree_index, last_id)) = cache.last { + if last_target_index + 1 == self.target { + // we can use the cached value + let key = ElemId(last_id); + self.last_valid_insert = Some(key); + self.n = last_tree_index + 1; + return true; + } + } + false + } + + fn cache_update_seq(&self, cache: &mut crate::object_data::SeqOpsCache) { + cache.last = Some((self.target, self.pos(), self.id)); + } + fn query_node(&mut self, child: &OpTreeNode) -> QueryResult { // if this node has some visible elements then we may find our target within let mut num_vis = child.index.visible_len(); diff --git a/automerge/src/transaction/inner.rs b/automerge/src/transaction/inner.rs index ebfb20ce..a9c6bed7 100644 --- a/automerge/src/transaction/inner.rs +++ b/automerge/src/transaction/inner.rs @@ -210,7 +210,7 @@ impl TransactionInner { ) -> Result { let id = self.next_id(); - let query = doc.ops.search(&obj, query::InsertNth::new(index)); + let query = doc.ops.search(&obj, query::InsertNth::new(index, id)); let key = query.key()?; From 5250ad4840ff2f56bbd5beb956dddeb2233c7aa0 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 15:19:50 +0100 Subject: [PATCH 11/19] Track the type of the last cached result too --- automerge/src/object_data.rs | 4 ++-- automerge/src/query/insert.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 29d64349..19393db1 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -55,9 +55,9 @@ impl From for ObjType { #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SeqOpsCache { - // last insertion (list index, tree index, opid to be inserted) + // last insertion (list index, tree index, whether the last op was an insert, opid to be inserted) // TODO: invalidation - pub(crate) last: Option<(usize, usize, OpId)>, + pub(crate) last: Option<(usize, usize, bool, OpId)>, } impl SeqOpsCache { diff --git a/automerge/src/query/insert.rs b/automerge/src/query/insert.rs index 3fede336..f5fecf9b 100644 --- a/automerge/src/query/insert.rs +++ b/automerge/src/query/insert.rs @@ -66,8 +66,8 @@ impl InsertNth { impl<'a> TreeQuery<'a> for InsertNth { fn cache_lookup_seq(&mut self, cache: &crate::object_data::SeqOpsCache) -> bool { - if let Some((last_target_index, last_tree_index, last_id)) = cache.last { - if last_target_index + 1 == self.target { + if let Some((last_target_index, last_tree_index, insert, last_id)) = cache.last { + if insert && last_target_index + 1 == self.target { // we can use the cached value let key = ElemId(last_id); self.last_valid_insert = Some(key); @@ -79,7 +79,7 @@ impl<'a> TreeQuery<'a> for InsertNth { } fn cache_update_seq(&self, cache: &mut crate::object_data::SeqOpsCache) { - cache.last = Some((self.target, self.pos(), self.id)); + cache.last = Some((self.target, self.pos(), true, self.id)); } fn query_node(&mut self, child: &OpTreeNode) -> QueryResult { From e51137fc284768f3df9552e2f1e611a22ab4c1b6 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 31 Mar 2022 15:31:00 +0100 Subject: [PATCH 12/19] Invalidate cache on nth --- automerge/src/query/nth.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/automerge/src/query/nth.rs b/automerge/src/query/nth.rs index 3924fc62..f6d47215 100644 --- a/automerge/src/query/nth.rs +++ b/automerge/src/query/nth.rs @@ -40,6 +40,14 @@ impl<'a> Nth<'a> { } impl<'a> TreeQuery<'a> for Nth<'a> { + fn cache_lookup_seq(&mut self, _cache: &crate::object_data::SeqOpsCache) -> bool { + false + } + + fn cache_update_seq(&self, cache: &mut crate::object_data::SeqOpsCache) { + cache.last = None; + } + fn query_node(&mut self, child: &OpTreeNode) -> QueryResult { let mut num_vis = child.index.visible_len(); if child.index.has_visible(&self.last_seen) { From 4a7924dc60e78f2124c9cc390b4e2dd00340e558 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Sat, 9 Apr 2022 10:36:23 +0100 Subject: [PATCH 13/19] Fixup after rebase --- automerge/src/object_data.rs | 124 ++++++++++++++++++++------------- automerge/src/op_set.rs | 4 +- automerge/src/op_tree.rs | 26 +++---- automerge/src/visualisation.rs | 2 +- 4 files changed, 90 insertions(+), 66 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 19393db1..23e61bfd 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -1,12 +1,12 @@ +use std::ops::RangeBounds; use std::sync::{Arc, Mutex}; use crate::clock::Clock; use crate::op_tree::{OpSetMetadata, OpTreeInternal}; -use crate::query::TreeQuery; +use crate::query::{self, TreeQuery}; use crate::types::ObjId; -use crate::types::Op; use crate::types::{Op, OpId}; -use crate::ObjType; +use crate::Prop; use crate::{query::Keys, query::KeysAt, ObjType}; #[derive(Debug, Clone, Copy, PartialEq)] @@ -28,11 +28,11 @@ impl From for ObjType { pub(crate) struct MapOpsCache {} impl MapOpsCache { - fn lookup(&self, query: &mut Q) -> bool { + fn lookup<'a, Q: TreeQuery<'a>>(&self, query: &mut Q) -> bool { query.cache_lookup_map(self) } - fn update(&mut self, query: &Q) { + fn update<'a, Q: TreeQuery<'a>>(&mut self, query: &Q) { query.cache_update_map(self); // TODO: fixup the cache (reordering etc.) } @@ -61,11 +61,11 @@ pub(crate) struct SeqOpsCache { } impl SeqOpsCache { - fn lookup(&self, query: &mut Q) -> bool { + fn lookup<'a, Q: TreeQuery<'a>>(&self, query: &mut Q) -> bool { query.cache_lookup_seq(self) } - fn update(&mut self, query: &Q) { + fn update<'a, Q: TreeQuery<'a>>(&mut self, query: &Q) { query.cache_update_seq(self); // TODO: fixup the cache (reordering etc.) } @@ -76,12 +76,12 @@ impl SeqOpsCache { pub(crate) struct ObjectData { internal: ObjectDataInternal, /// The operations pertaining to this object. - ops: OpTreeInternal, + pub(crate) ops: OpTreeInternal, /// The id of the parent object, root has no parent. pub parent: Option, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub(crate) enum ObjectDataInternal { Map { /// The type of this object. @@ -95,34 +95,37 @@ pub(crate) enum ObjectDataInternal { }, } -impl PartialEq for ObjectData { - fn eq(&self, other: &Self) -> bool { +impl PartialEq for ObjectDataInternal { + fn eq(&self, other: &ObjectDataInternal) -> bool { match (self, other) { ( - Self::Map { - typ: l_typ, - ops: l_ops, + ObjectDataInternal::Map { + typ: typ1, cache: _, }, - Self::Map { - typ: r_typ, - ops: r_ops, + ObjectDataInternal::Map { + typ: typ2, cache: _, }, - ) => l_typ == r_typ && l_ops == r_ops, + ) => typ1 == typ2, ( - Self::Seq { - typ: l_typ, - ops: l_ops, + ObjectDataInternal::Map { typ: _, cache: _ }, + ObjectDataInternal::Seq { typ: _, cache: _ }, + ) => false, + ( + ObjectDataInternal::Seq { typ: _, cache: _ }, + ObjectDataInternal::Map { typ: _, cache: _ }, + ) => false, + ( + ObjectDataInternal::Seq { + typ: typ1, cache: _, }, - Self::Seq { - typ: r_typ, - ops: r_ops, + ObjectDataInternal::Seq { + typ: typ2, cache: _, }, - ) => l_typ == r_typ && l_ops == r_ops, - _ => false, + ) => typ1 == typ2, } } } @@ -141,12 +144,22 @@ impl ObjectData { pub fn new(typ: ObjType, parent: Option) -> Self { let internal = match typ { - ObjType::Map => ObjectDataInternal::Map { typ: MapType::Map }, + ObjType::Map => ObjectDataInternal::Map { + typ: MapType::Map, + cache: Default::default(), + }, ObjType::Table => ObjectDataInternal::Map { typ: MapType::Table, + cache: Default::default(), + }, + ObjType::List => ObjectDataInternal::Seq { + typ: SeqType::List, + cache: Default::default(), + }, + ObjType::Text => ObjectDataInternal::Seq { + typ: SeqType::Text, + cache: Default::default(), }, - ObjType::List => ObjectDataInternal::Seq { typ: SeqType::List }, - ObjType::Text => ObjectDataInternal::Seq { typ: SeqType::Text }, }; ObjectData { internal, @@ -163,26 +176,33 @@ impl ObjectData { self.ops.keys_at(clock) } - pub(crate) fn ops(&self) -> &OpTreeInternal { - match self { - ObjectData::Map { ops, .. } => ops, - ObjectData::Seq { ops, .. } => ops, - } + pub fn range<'a, R: RangeBounds>( + &'a self, + range: R, + meta: &'a OpSetMetadata, + ) -> Option> { + self.ops.range(range, meta) } - fn ops_mut(&mut self) -> &mut OpTreeInternal { - match self { - ObjectData::Map { ops, .. } => ops, - ObjectData::Seq { ops, .. } => ops, - } + pub fn range_at<'a, R: RangeBounds>( + &'a self, + range: R, + meta: &'a OpSetMetadata, + clock: Clock, + ) -> Option> { + self.ops.range_at(range, meta, clock) } - pub fn search(&self, mut query: Q, metadata: &OpSetMetadata) -> Q + pub fn search<'a, 'b: 'a, Q>(&'b self, mut query: Q, metadata: &OpSetMetadata) -> Q where - Q: TreeQuery, + Q: TreeQuery<'a>, { match self { - ObjectData::Map { cache, ops, .. } => { + ObjectData { + ops, + internal: ObjectDataInternal::Map { cache, .. }, + .. + } => { let mut cache = cache.lock().unwrap(); if !cache.lookup(&mut query) { query = ops.search(query, metadata); @@ -190,7 +210,11 @@ impl ObjectData { cache.update(&query); query } - ObjectData::Seq { cache, ops, .. } => { + ObjectData { + ops, + internal: ObjectDataInternal::Seq { cache, .. }, + .. + } => { let mut cache = cache.lock().unwrap(); if !cache.lookup(&mut query) { query = ops.search(query, metadata); @@ -205,25 +229,25 @@ impl ObjectData { where F: FnOnce(&mut Op), { - self.ops_mut().update(index, f) + self.ops.update(index, f) } pub fn remove(&mut self, index: usize) -> Op { - self.ops_mut().remove(index) + self.ops.remove(index) } pub fn insert(&mut self, index: usize, op: Op) { - self.ops_mut().insert(index, op) + self.ops.insert(index, op) } pub fn typ(&self) -> ObjType { - match self { - ObjectData::Map { typ, .. } => (*typ).into(), - ObjectData::Seq { typ, .. } => (*typ).into(), + match &self.internal { + ObjectDataInternal::Map { typ, .. } => (*typ).into(), + ObjectDataInternal::Seq { typ, .. } => (*typ).into(), } } pub fn get(&self, index: usize) -> Option<&Op> { - self.ops().get(index) + self.ops.get(index) } } diff --git a/automerge/src/op_set.rs b/automerge/src/op_set.rs index 8e22f1a8..80a5d77e 100644 --- a/automerge/src/op_set.rs +++ b/automerge/src/op_set.rs @@ -83,7 +83,7 @@ impl OpSetInternal { range: R, ) -> Option> { if let Some(tree) = self.objects.get(&obj) { - tree.internal.range(range, &self.m) + tree.range(range, &self.m) } else { None } @@ -96,7 +96,7 @@ impl OpSetInternal { clock: Clock, ) -> Option> { if let Some(tree) = self.objects.get(&obj) { - tree.internal.range_at(range, &self.m, clock) + tree.range_at(range, &self.m, clock) } else { None } diff --git a/automerge/src/op_tree.rs b/automerge/src/op_tree.rs index f820b58c..63281edf 100644 --- a/automerge/src/op_tree.rs +++ b/automerge/src/op_tree.rs @@ -692,36 +692,36 @@ mod tests { #[test] fn insert() { - let mut t: OpTree = OpTree::new(); + let mut t = OpTreeInternal::new(); - t.internal.insert(0, op()); - t.internal.insert(1, op()); - t.internal.insert(0, op()); - t.internal.insert(0, op()); - t.internal.insert(0, op()); - t.internal.insert(3, op()); - t.internal.insert(4, op()); + t.insert(0, op()); + t.insert(1, op()); + t.insert(0, op()); + t.insert(0, op()); + t.insert(0, op()); + t.insert(3, op()); + t.insert(4, op()); } #[test] fn insert_book() { - let mut t: OpTree = OpTree::new(); + let mut t = OpTreeInternal::new(); for i in 0..100 { - t.internal.insert(i % 2, op()); + t.insert(i % 2, op()); } } #[test] fn insert_book_vec() { - let mut t: OpTree = OpTree::new(); + let mut t = OpTreeInternal::new(); let mut v = Vec::new(); for i in 0..100 { - t.internal.insert(i % 3, op()); + t.insert(i % 3, op()); v.insert(i % 3, op()); - assert_eq!(v, t.internal.iter().cloned().collect::>()) + assert_eq!(v, t.iter().cloned().collect::>()) } } } diff --git a/automerge/src/visualisation.rs b/automerge/src/visualisation.rs index fbf387a0..6107eeee 100644 --- a/automerge/src/visualisation.rs +++ b/automerge/src/visualisation.rs @@ -48,7 +48,7 @@ impl<'a> GraphVisualisation<'a> { ) -> GraphVisualisation<'a> { let mut nodes = HashMap::new(); for (obj_id, object_data) in objects { - if let Some(root_node) = &object_data.ops().root_node { + if let Some(root_node) = &object_data.ops.root_node { let tree_id = Self::construct_nodes(root_node, obj_id, &mut nodes, metadata); let obj_tree_id = NodeId::default(); nodes.insert( From 57aad148da79ee5779bf4fdf93ab02b9aa57b192 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Sat, 9 Apr 2022 17:35:52 +0100 Subject: [PATCH 14/19] Refactor ObjectData --- automerge/src/object_data.rs | 120 ++++++----------------------------- 1 file changed, 21 insertions(+), 99 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 23e61bfd..bd4896d7 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -9,21 +9,6 @@ use crate::types::{Op, OpId}; use crate::Prop; use crate::{query::Keys, query::KeysAt, ObjType}; -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) enum MapType { - Map, - Table, -} - -impl From for ObjType { - fn from(m: MapType) -> Self { - match m { - MapType::Map => ObjType::Map, - MapType::Table => ObjType::Table, - } - } -} - #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct MapOpsCache {} @@ -38,21 +23,6 @@ impl MapOpsCache { } } -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) enum SeqType { - List, - Text, -} - -impl From for ObjType { - fn from(s: SeqType) -> Self { - match s { - SeqType::List => ObjType::List, - SeqType::Text => ObjType::Text, - } - } -} - #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SeqOpsCache { // last insertion (list index, tree index, whether the last op was an insert, opid to be inserted) @@ -74,7 +44,9 @@ impl SeqOpsCache { /// Stores the data for an object. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObjectData { - internal: ObjectDataInternal, + cache: ObjectDataCache, + /// The type of this object. + typ: ObjType, /// The operations pertaining to this object. pub(crate) ops: OpTreeInternal, /// The id of the parent object, root has no parent. @@ -82,50 +54,18 @@ pub(crate) struct ObjectData { } #[derive(Debug, Clone)] -pub(crate) enum ObjectDataInternal { - Map { - /// The type of this object. - typ: MapType, - cache: Arc>, - }, - Seq { - /// The type of this object. - typ: SeqType, - cache: Arc>, - }, +pub(crate) enum ObjectDataCache { + Map(Arc>), + Seq(Arc>), } -impl PartialEq for ObjectDataInternal { - fn eq(&self, other: &ObjectDataInternal) -> bool { +impl PartialEq for ObjectDataCache { + fn eq(&self, other: &ObjectDataCache) -> bool { match (self, other) { - ( - ObjectDataInternal::Map { - typ: typ1, - cache: _, - }, - ObjectDataInternal::Map { - typ: typ2, - cache: _, - }, - ) => typ1 == typ2, - ( - ObjectDataInternal::Map { typ: _, cache: _ }, - ObjectDataInternal::Seq { typ: _, cache: _ }, - ) => false, - ( - ObjectDataInternal::Seq { typ: _, cache: _ }, - ObjectDataInternal::Map { typ: _, cache: _ }, - ) => false, - ( - ObjectDataInternal::Seq { - typ: typ1, - cache: _, - }, - ObjectDataInternal::Seq { - typ: typ2, - cache: _, - }, - ) => typ1 == typ2, + (ObjectDataCache::Map(_), ObjectDataCache::Map(_)) => true, + (ObjectDataCache::Map(_), ObjectDataCache::Seq(_)) => false, + (ObjectDataCache::Seq(_), ObjectDataCache::Map(_)) => false, + (ObjectDataCache::Seq(_), ObjectDataCache::Seq(_)) => true, } } } @@ -133,10 +73,8 @@ impl PartialEq for ObjectDataInternal { impl ObjectData { pub fn root() -> Self { ObjectData { - internal: ObjectDataInternal::Map { - typ: MapType::Map, - cache: Default::default(), - }, + cache: ObjectDataCache::Map(Default::default()), + typ: ObjType::Map, ops: Default::default(), parent: None, } @@ -144,25 +82,12 @@ impl ObjectData { pub fn new(typ: ObjType, parent: Option) -> Self { let internal = match typ { - ObjType::Map => ObjectDataInternal::Map { - typ: MapType::Map, - cache: Default::default(), - }, - ObjType::Table => ObjectDataInternal::Map { - typ: MapType::Table, - cache: Default::default(), - }, - ObjType::List => ObjectDataInternal::Seq { - typ: SeqType::List, - cache: Default::default(), - }, - ObjType::Text => ObjectDataInternal::Seq { - typ: SeqType::Text, - cache: Default::default(), - }, + ObjType::Map | ObjType::Table => ObjectDataCache::Map(Default::default()), + ObjType::List | ObjType::Text => ObjectDataCache::Seq(Default::default()), }; ObjectData { - internal, + cache: internal, + typ, ops: Default::default(), parent, } @@ -200,7 +125,7 @@ impl ObjectData { match self { ObjectData { ops, - internal: ObjectDataInternal::Map { cache, .. }, + cache: ObjectDataCache::Map(cache), .. } => { let mut cache = cache.lock().unwrap(); @@ -212,7 +137,7 @@ impl ObjectData { } ObjectData { ops, - internal: ObjectDataInternal::Seq { cache, .. }, + cache: ObjectDataCache::Seq(cache), .. } => { let mut cache = cache.lock().unwrap(); @@ -241,10 +166,7 @@ impl ObjectData { } pub fn typ(&self) -> ObjType { - match &self.internal { - ObjectDataInternal::Map { typ, .. } => (*typ).into(), - ObjectDataInternal::Seq { typ, .. } => (*typ).into(), - } + self.typ } pub fn get(&self, index: usize) -> Option<&Op> { From f25500f81bc3d74d99718ab47df796a78850d3f5 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Tue, 19 Apr 2022 08:14:48 +0100 Subject: [PATCH 15/19] Add map bench --- automerge/Cargo.toml | 5 +++++ automerge/benches/map.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 automerge/benches/map.rs diff --git a/automerge/Cargo.toml b/automerge/Cargo.toml index ae95fa4e..c56ee24d 100644 --- a/automerge/Cargo.toml +++ b/automerge/Cargo.toml @@ -43,3 +43,8 @@ proptest = { version = "^1.0.0", default-features = false, features = ["std"] } serde_json = { version = "^1.0.73", features=["float_roundtrip"], default-features=true } maplit = { version = "^1.0" } decorum = "0.3.1" +criterion = "0.3.5" + +[[bench]] +name = "map" +harness = false diff --git a/automerge/benches/map.rs b/automerge/benches/map.rs new file mode 100644 index 00000000..1048d08b --- /dev/null +++ b/automerge/benches/map.rs @@ -0,0 +1,29 @@ +use automerge::{transaction::Transactable, Automerge, ROOT}; +use criterion::{criterion_group, criterion_main, Criterion}; + +fn query_doc(doc: &Automerge, key: &str, rounds: u32) { + for _ in 0..rounds { + doc.get(ROOT, key).unwrap(); + } +} + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("map"); + + let rounds = 10_000; + let mut doc = Automerge::new(); + let mut tx = doc.transaction(); + for i in 0..rounds { + tx.put(ROOT, i.to_string(), vec![0, 1, 2, 3, 4, 5]).unwrap(); + } + tx.commit(); + + group.bench_function("query", |b| { + b.iter(|| query_doc(&doc, &(rounds - 1).to_string(), rounds)) + }); + + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); From d9b35c16a2078c8b112be3d0d1c080fcc9511c66 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Tue, 19 Apr 2022 17:20:12 +0100 Subject: [PATCH 16/19] Add initial work caching prop --- automerge/src/object_data.rs | 6 ++++-- automerge/src/query/prop.rs | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index bd4896d7..17f8ef2b 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -4,13 +4,15 @@ use std::sync::{Arc, Mutex}; use crate::clock::Clock; use crate::op_tree::{OpSetMetadata, OpTreeInternal}; use crate::query::{self, TreeQuery}; -use crate::types::ObjId; +use crate::types::{Key, ObjId}; use crate::types::{Op, OpId}; use crate::Prop; use crate::{query::Keys, query::KeysAt, ObjType}; #[derive(Debug, Default, Clone, PartialEq)] -pub(crate) struct MapOpsCache {} +pub(crate) struct MapOpsCache { + pub(crate) last: Option<(Key, usize)>, +} impl MapOpsCache { fn lookup<'a, Q: TreeQuery<'a>>(&self, query: &mut Q) -> bool { diff --git a/automerge/src/query/prop.rs b/automerge/src/query/prop.rs index 7fcb8559..3609ffe5 100644 --- a/automerge/src/query/prop.rs +++ b/automerge/src/query/prop.rs @@ -9,6 +9,7 @@ pub(crate) struct Prop<'a> { pub(crate) ops: Vec<&'a Op>, pub(crate) ops_pos: Vec, pub(crate) pos: usize, + start: Option, } impl<'a> Prop<'a> { @@ -18,17 +19,36 @@ impl<'a> Prop<'a> { ops: vec![], ops_pos: vec![], pos: 0, + start: None, } } } impl<'a> TreeQuery<'a> for Prop<'a> { + fn cache_lookup_map(&mut self, cache: &crate::object_data::MapOpsCache) -> bool { + if let Some((last_key, last_pos)) = cache.last { + if last_key == self.key { + self.start = Some(last_pos); + } + } + // don't have all of the result yet + false + } + + fn cache_update_map(&self, cache: &mut crate::object_data::MapOpsCache) { + cache.last = Some((self.key, self.pos)) + } + fn query_node_with_metadata( &mut self, child: &'a OpTreeNode, m: &OpSetMetadata, ) -> QueryResult { - let start = binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)); + let start = if let Some(start) = self.start { + start + } else { + binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)) + }; self.pos = start; for pos in start..child.len() { let op = child.get(pos).unwrap(); From 7d20572c494702ab55cd230ed6d3575bce015682 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Tue, 19 Apr 2022 21:36:10 +0100 Subject: [PATCH 17/19] Update map benches --- automerge/benches/map.rs | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/automerge/benches/map.rs b/automerge/benches/map.rs index 1048d08b..19141d29 100644 --- a/automerge/benches/map.rs +++ b/automerge/benches/map.rs @@ -1,9 +1,24 @@ use automerge::{transaction::Transactable, Automerge, ROOT}; use criterion::{criterion_group, criterion_main, Criterion}; -fn query_doc(doc: &Automerge, key: &str, rounds: u32) { +fn query_single(doc: &Automerge, rounds: u32) { for _ in 0..rounds { - doc.get(ROOT, key).unwrap(); + // repeatedly get the last key + doc.get(ROOT, (rounds - 1).to_string()).unwrap(); + } +} + +fn query_range(doc: &Automerge, rounds: u32) { + for i in 0..rounds { + doc.get(ROOT, i.to_string()).unwrap(); + } +} + +fn put_doc(doc: &mut Automerge, rounds: u32) { + for i in 0..rounds { + let mut tx = doc.transaction(); + tx.put(ROOT, i.to_string(), "value").unwrap(); + tx.commit(); } } @@ -12,14 +27,18 @@ fn bench(c: &mut Criterion) { let rounds = 10_000; let mut doc = Automerge::new(); - let mut tx = doc.transaction(); - for i in 0..rounds { - tx.put(ROOT, i.to_string(), vec![0, 1, 2, 3, 4, 5]).unwrap(); - } - tx.commit(); + put_doc(&mut doc, rounds); - group.bench_function("query", |b| { - b.iter(|| query_doc(&doc, &(rounds - 1).to_string(), rounds)) + group.bench_function("query single", |b| b.iter(|| query_single(&doc, rounds))); + + group.bench_function("query range", |b| b.iter(|| query_range(&doc, rounds))); + + group.bench_function("put", |b| { + b.iter_batched( + Automerge::new, + |mut doc| put_doc(&mut doc, rounds), + criterion::BatchSize::LargeInput, + ) }); group.finish(); From 633e05a84735619716d5958872dc41c9a51915b7 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Tue, 19 Apr 2022 21:36:20 +0100 Subject: [PATCH 18/19] Update prop query caching --- automerge/src/query.rs | 2 + automerge/src/query/insert_prop.rs | 68 ++++++++++++++++++++++++++++++ automerge/src/query/prop.rs | 5 ++- automerge/src/transaction/inner.rs | 4 +- 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 automerge/src/query/insert_prop.rs diff --git a/automerge/src/query.rs b/automerge/src/query.rs index a98be65a..790c88d3 100644 --- a/automerge/src/query.rs +++ b/automerge/src/query.rs @@ -8,6 +8,7 @@ use std::fmt::Debug; mod elem_id_pos; mod insert; +mod insert_prop; mod keys; mod keys_at; mod len; @@ -26,6 +27,7 @@ mod seek_op_with_patch; pub(crate) use elem_id_pos::ElemIdPos; pub(crate) use insert::InsertNth; +pub(crate) use insert_prop::InsertProp; pub(crate) use keys::Keys; pub(crate) use keys_at::KeysAt; pub(crate) use len::Len; diff --git a/automerge/src/query/insert_prop.rs b/automerge/src/query/insert_prop.rs new file mode 100644 index 00000000..4f6131ce --- /dev/null +++ b/automerge/src/query/insert_prop.rs @@ -0,0 +1,68 @@ +use crate::op_tree::{OpSetMetadata, OpTreeNode}; +use crate::query::{binary_search_by, QueryResult, TreeQuery}; +use crate::types::{Key, Op}; +use std::fmt::Debug; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct InsertProp<'a> { + key: Key, + pub ops: Vec<&'a Op>, + pub ops_pos: Vec, + pub pos: usize, + start: Option, +} + +impl<'a> InsertProp<'a> { + pub fn new(prop: usize) -> Self { + InsertProp { + key: Key::Map(prop), + ops: vec![], + ops_pos: vec![], + pos: 0, + start: None, + } + } +} + +impl<'a> TreeQuery<'a> for InsertProp<'a> { + fn cache_lookup_map(&mut self, cache: &crate::object_data::MapOpsCache) -> bool { + if let Some((last_key, last_pos)) = cache.last { + if last_key == self.key { + self.start = Some(last_pos); + } + } + // don't have all of the result yet + false + } + + fn cache_update_map(&self, cache: &mut crate::object_data::MapOpsCache) { + cache.last = None + } + + fn query_node_with_metadata( + &mut self, + child: &'a OpTreeNode, + m: &OpSetMetadata, + ) -> QueryResult { + let start = if let Some(start) = self.start { + debug_assert!(binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)) >= start); + start + } else { + binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)) + }; + self.start = Some(start); + self.pos = start; + for pos in start..child.len() { + let op = child.get(pos).unwrap(); + if op.key != self.key { + break; + } + if op.visible() { + self.ops.push(op); + self.ops_pos.push(pos); + } + self.pos += 1; + } + QueryResult::Finish + } +} diff --git a/automerge/src/query/prop.rs b/automerge/src/query/prop.rs index 3609ffe5..8cbfb358 100644 --- a/automerge/src/query/prop.rs +++ b/automerge/src/query/prop.rs @@ -1,7 +1,6 @@ use crate::op_tree::{OpSetMetadata, OpTreeNode}; use crate::query::{binary_search_by, QueryResult, TreeQuery}; use crate::types::{Key, Op}; -use std::fmt::Debug; #[derive(Debug, Clone, PartialEq)] pub(crate) struct Prop<'a> { @@ -36,7 +35,7 @@ impl<'a> TreeQuery<'a> for Prop<'a> { } fn cache_update_map(&self, cache: &mut crate::object_data::MapOpsCache) { - cache.last = Some((self.key, self.pos)) + cache.last = self.start.map(|start| (self.key, start)); } fn query_node_with_metadata( @@ -45,10 +44,12 @@ impl<'a> TreeQuery<'a> for Prop<'a> { m: &OpSetMetadata, ) -> QueryResult { let start = if let Some(start) = self.start { + debug_assert!(binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)) >= start); start } else { binary_search_by(child, |op| m.key_cmp(&op.key, &self.key)) }; + self.start = Some(start); self.pos = start; for pos in start..child.len() { let op = child.get(pos).unwrap(); diff --git a/automerge/src/transaction/inner.rs b/automerge/src/transaction/inner.rs index a9c6bed7..de8980e5 100644 --- a/automerge/src/transaction/inner.rs +++ b/automerge/src/transaction/inner.rs @@ -254,8 +254,8 @@ impl TransactionInner { } let id = self.next_id(); - let prop_index = doc.ops.m.props.cache(prop.clone()); - let query = doc.ops.search(&obj, query::Prop::new(prop_index)); + let prop_index = doc.ops.m.props.cache(prop); + let query = doc.ops.search(&obj, query::InsertProp::new(prop)); // no key present to delete if query.ops.is_empty() && action == OpType::Delete { From 295d9a9c22d217da8b19a4c0dff709895e5841b0 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Thu, 28 Apr 2022 14:07:25 +0100 Subject: [PATCH 19/19] Fixup --- automerge/src/object_data.rs | 27 +++++++++++++-------------- automerge/src/op_tree.rs | 23 +---------------------- automerge/src/query/insert_prop.rs | 8 ++++---- automerge/src/transaction/inner.rs | 4 ++-- 4 files changed, 20 insertions(+), 42 deletions(-) diff --git a/automerge/src/object_data.rs b/automerge/src/object_data.rs index 17f8ef2b..98b86f9c 100644 --- a/automerge/src/object_data.rs +++ b/automerge/src/object_data.rs @@ -6,7 +6,6 @@ use crate::op_tree::{OpSetMetadata, OpTreeInternal}; use crate::query::{self, TreeQuery}; use crate::types::{Key, ObjId}; use crate::types::{Op, OpId}; -use crate::Prop; use crate::{query::Keys, query::KeysAt, ObjType}; #[derive(Debug, Default, Clone, PartialEq)] @@ -52,7 +51,7 @@ pub(crate) struct ObjectData { /// The operations pertaining to this object. pub(crate) ops: OpTreeInternal, /// The id of the parent object, root has no parent. - pub parent: Option, + pub(crate) parent: Option, } #[derive(Debug, Clone)] @@ -73,7 +72,7 @@ impl PartialEq for ObjectDataCache { } impl ObjectData { - pub fn root() -> Self { + pub(crate) fn root() -> Self { ObjectData { cache: ObjectDataCache::Map(Default::default()), typ: ObjType::Map, @@ -82,7 +81,7 @@ impl ObjectData { } } - pub fn new(typ: ObjType, parent: Option) -> Self { + pub(crate) fn new(typ: ObjType, parent: Option) -> Self { let internal = match typ { ObjType::Map | ObjType::Table => ObjectDataCache::Map(Default::default()), ObjType::List | ObjType::Text => ObjectDataCache::Seq(Default::default()), @@ -95,15 +94,15 @@ impl ObjectData { } } - pub fn keys(&self) -> Option { + pub(crate) fn keys(&self) -> Option> { self.ops.keys() } - pub fn keys_at(&self, clock: Clock) -> Option { + pub(crate) fn keys_at(&self, clock: Clock) -> Option> { self.ops.keys_at(clock) } - pub fn range<'a, R: RangeBounds>( + pub(crate) fn range<'a, R: RangeBounds>( &'a self, range: R, meta: &'a OpSetMetadata, @@ -111,7 +110,7 @@ impl ObjectData { self.ops.range(range, meta) } - pub fn range_at<'a, R: RangeBounds>( + pub(crate) fn range_at<'a, R: RangeBounds>( &'a self, range: R, meta: &'a OpSetMetadata, @@ -120,7 +119,7 @@ impl ObjectData { self.ops.range_at(range, meta, clock) } - pub fn search<'a, 'b: 'a, Q>(&'b self, mut query: Q, metadata: &OpSetMetadata) -> Q + pub(crate) fn search<'a, 'b: 'a, Q>(&'b self, mut query: Q, metadata: &OpSetMetadata) -> Q where Q: TreeQuery<'a>, { @@ -152,26 +151,26 @@ impl ObjectData { } } - pub fn update(&mut self, index: usize, f: F) + pub(crate) fn update(&mut self, index: usize, f: F) where F: FnOnce(&mut Op), { self.ops.update(index, f) } - pub fn remove(&mut self, index: usize) -> Op { + pub(crate) fn remove(&mut self, index: usize) -> Op { self.ops.remove(index) } - pub fn insert(&mut self, index: usize, op: Op) { + pub(crate) fn insert(&mut self, index: usize, op: Op) { self.ops.insert(index, op) } - pub fn typ(&self) -> ObjType { + pub(crate) fn typ(&self) -> ObjType { self.typ } - pub fn get(&self, index: usize) -> Option<&Op> { + pub(crate) fn get(&self, index: usize) -> Option<&Op> { self.ops.get(index) } } diff --git a/automerge/src/op_tree.rs b/automerge/src/op_tree.rs index 63281edf..79e110af 100644 --- a/automerge/src/op_tree.rs +++ b/automerge/src/op_tree.rs @@ -6,36 +6,15 @@ use std::{ }; pub(crate) use crate::op_set::OpSetMetadata; +use crate::types::{Op, OpId}; use crate::{ clock::Clock, query::{self, Index, QueryResult, TreeQuery}, }; -use crate::{ - types::{ObjId, Op, OpId}, - ObjType, -}; use std::collections::HashSet; pub(crate) const B: usize = 16; -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct OpTree { - pub(crate) internal: OpTreeInternal, - pub(crate) objtype: ObjType, - /// The id of the parent object, root has no parent. - pub(crate) parent: Option, -} - -impl OpTree { - pub(crate) fn new() -> Self { - Self { - internal: Default::default(), - objtype: ObjType::Map, - parent: None, - } - } -} - #[derive(Clone, Debug)] pub(crate) struct OpTreeInternal { pub(crate) root_node: Option, diff --git a/automerge/src/query/insert_prop.rs b/automerge/src/query/insert_prop.rs index 4f6131ce..73078d58 100644 --- a/automerge/src/query/insert_prop.rs +++ b/automerge/src/query/insert_prop.rs @@ -6,14 +6,14 @@ use std::fmt::Debug; #[derive(Debug, Clone, PartialEq)] pub(crate) struct InsertProp<'a> { key: Key, - pub ops: Vec<&'a Op>, - pub ops_pos: Vec, - pub pos: usize, + pub(crate) ops: Vec<&'a Op>, + pub(crate) ops_pos: Vec, + pub(crate) pos: usize, start: Option, } impl<'a> InsertProp<'a> { - pub fn new(prop: usize) -> Self { + pub(crate) fn new(prop: usize) -> Self { InsertProp { key: Key::Map(prop), ops: vec![], diff --git a/automerge/src/transaction/inner.rs b/automerge/src/transaction/inner.rs index de8980e5..7a42a5ee 100644 --- a/automerge/src/transaction/inner.rs +++ b/automerge/src/transaction/inner.rs @@ -254,8 +254,8 @@ impl TransactionInner { } let id = self.next_id(); - let prop_index = doc.ops.m.props.cache(prop); - let query = doc.ops.search(&obj, query::InsertProp::new(prop)); + let prop_index = doc.ops.m.props.cache(prop.clone()); + let query = doc.ops.search(&obj, query::InsertProp::new(prop_index)); // no key present to delete if query.ops.is_empty() && action == OpType::Delete {