Add tests and don't load data into changes

commonspace/object/tree/objecttree/changebuilder.go

@@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.
 	return c.ChangeBuilder.Marshall(ch)
 }
 
+type emptyDataChangeBuilder struct {
+	ChangeBuilder
+}
+
+func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) {
+	panic("should not be called")
+}
+
+func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) {
+	panic("should not be called")
+}
+
 type ChangeBuilder interface {
 	Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error)
 	Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error)
@@ -59,13 +71,28 @@ type ChangeBuilder interface {
 	Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error)
 }
 
+type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change
+
 type changeBuilder struct {
 	rootChange *treechangeproto.RawTreeChangeWithId
 	keys       crypto.KeyStorage
+	newChange  newChangeFunc
+}
+
+func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder {
+	return &emptyDataChangeBuilder{&changeBuilder{
+		rootChange: rootChange,
+		keys:       keys,
+		newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change {
+			c := NewChange(id, identity, ch, signature)
+			c.Data = nil
+			return c
+		},
+	}}
 }
 
 func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder {
-	return &changeBuilder{keys: keys, rootChange: rootChange}
+	return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange}
 }
 
 func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) {
@@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
 	if err != nil {
 		return
 	}
-	ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature)
+	ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature)
 	rawIdChange = &treechangeproto.RawTreeChangeWithId{
 		RawChange: marshalledRawChange,
 		Id:        id,
@@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
 	if err != nil {
 		return
 	}
-	ch = NewChange(id, key, unmarshalled, raw.Signature)
+	ch = c.newChange(id, key, unmarshalled, raw.Signature)
 	return
 }
 

commonspace/object/tree/objecttree/objecttree.go

@@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string)
 		}
 	}
 
-	if commonSnapshot == ot.tree.RootId() {
+	return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads)
-		return ot.getChangesFromTree(theirHeads)
-	} else {
-		return ot.getChangesFromDB(commonSnapshot, theirHeads)
-	}
-}
-
-func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) {
-	return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads)
-}
-
-func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) {
-	return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads)
 }
 
 func (ot *objectTree) snapshotPathIsActual() bool {

commonspace/object/tree/objecttree/objecttreefactory.go

@@ -51,6 +51,22 @@ func verifiableTreeDeps(
 	}
 }
 
+func emptyDataTreeDeps(
+	rootChange *treechangeproto.RawTreeChangeWithId,
+	treeStorage treestorage.TreeStorage,
+	aclList list.AclList) objectTreeDeps {
+	changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange)
+	treeBuilder := newTreeBuilder(treeStorage, changeBuilder)
+	return objectTreeDeps{
+		changeBuilder:   changeBuilder,
+		treeBuilder:     treeBuilder,
+		treeStorage:     treeStorage,
+		validator:       newTreeValidator(),
+		rawChangeLoader: newStorageLoader(treeStorage, changeBuilder),
+		aclList:         aclList,
+	}
+}
+
 func nonVerifiableTreeDeps(
 	rootChange *treechangeproto.RawTreeChangeWithId,
 	treeStorage treestorage.TreeStorage,
@@ -80,6 +96,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList)
 	return createObjectTreeRoot(payload, 0, nil, aclList)
 }
 
+func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
+	rootChange, err := treeStorage.Root()
+	if err != nil {
+		return nil, err
+	}
+	deps := emptyDataTreeDeps(rootChange, treeStorage, aclList)
+	return buildObjectTree(deps)
+}
+
+func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
+	root, _ := treeStorage.Root()
+	changeBuilder := &nonVerifiableChangeBuilder{
+		ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root),
+	}
+	deps := objectTreeDeps{
+		changeBuilder:   changeBuilder,
+		treeBuilder:     newTreeBuilder(treeStorage, changeBuilder),
+		treeStorage:     treeStorage,
+		rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
+		validator:       &noOpTreeValidator{},
+		aclList:         aclList,
+	}
+
+	return buildObjectTree(deps)
+}
+
+func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
+	root, _ := treeStorage.Root()
+	changeBuilder := &nonVerifiableChangeBuilder{
+		ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root),
+	}
+	deps := objectTreeDeps{
+		changeBuilder:   changeBuilder,
+		treeBuilder:     newTreeBuilder(treeStorage, changeBuilder),
+		treeStorage:     treeStorage,
+		rawChangeLoader: newStorageLoader(treeStorage, changeBuilder),
+		validator:       &noOpTreeValidator{},
+		aclList:         aclList,
+	}
+
+	return buildObjectTree(deps)
+}
+
 func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
 	rootChange, err := treeStorage.Root()
 	if err != nil {

commonspace/object/tree/objecttree/rawloader.go

@@ -9,8 +9,9 @@ import (
 )
 
 type rawChangeLoader struct {
-	treeStorage   treestorage.TreeStorage
+	treeStorage       treestorage.TreeStorage
-	changeBuilder ChangeBuilder
+	changeBuilder     ChangeBuilder
+	alwaysFromStorage bool
 
 	// buffers
 	idStack []string
@@ -23,6 +24,12 @@ type rawCacheEntry struct {
 	position  int
 }
 
+func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader {
+	loader := newRawChangeLoader(treeStorage, changeBuilder)
+	loader.alwaysFromStorage = true
+	return loader
+}
+
 func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader {
 	return &rawChangeLoader{
 		treeStorage:   treeStorage,
@@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang
 	}
 }
 
-func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
+func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
+	if commonSnapshot == t.root.Id && !r.alwaysFromStorage {
+		return r.loadFromTree(t, breakpoints)
+	} else {
+		return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints)
+	}
+}
+
+func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
 	var stack []*Change
 	for _, h := range t.headIds {
 		stack = append(stack, t.attached[h])
@@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech
 	return convert(results)
 }
 
-func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
+func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
 	// resetting cache
 	r.cache = make(map[string]rawCacheEntry)
 	defer func() {

commonspace/object/tree/objecttree/testutils.go

@@ -2,7 +2,6 @@ package objecttree
 
 import (
 	"fmt"
-	"github.com/anytypeio/any-sync/commonspace/object/acl/list"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
 	"github.com/anytypeio/any-sync/util/crypto"
@@ -112,20 +111,3 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees
 	treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
 	return treeStorage
 }
-
-func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
-	root, _ := treeStorage.Root()
-	changeBuilder := &nonVerifiableChangeBuilder{
-		ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root),
-	}
-	deps := objectTreeDeps{
-		changeBuilder:   changeBuilder,
-		treeBuilder:     newTreeBuilder(treeStorage, changeBuilder),
-		treeStorage:     treeStorage,
-		rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
-		validator:       &noOpTreeValidator{},
-		aclList:         aclList,
-	}
-
-	return buildObjectTree(deps)
-}

commonspace/object/tree/synctree/syncprotocol_test.go

@@ -2,7 +2,6 @@ package synctree
 
 import (
 	"context"
-	"fmt"
 	"github.com/anytypeio/any-sync/commonspace/object/accountdata"
 	"github.com/anytypeio/any-sync/commonspace/object/acl/list"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@@ -60,7 +59,24 @@ func TestEmptyClientGetsFullHistory(t *testing.T) {
 	require.Len(t, fullResponseMsg.changes, 2)
 }
 
-func TestRandomTreeMerge(t *testing.T) {
+func TestRandomMerge(t *testing.T) {
+	var (
+		rnd      = rand.New(rand.NewSource(time.Now().Unix()))
+		levels   = 20
+		perLevel = 20
+	)
+	testTreeMerge(t, levels, perLevel, func() bool {
+		return true
+	})
+	testTreeMerge(t, levels, perLevel, func() bool {
+		return false
+	})
+	testTreeMerge(t, levels, perLevel, func() bool {
+		return rnd.Intn(10) > 8
+	})
+}
+
+func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) {
 	treeId := "treeId"
 	spaceId := "spaceId"
 	keys, err := accountdata.NewRandom()
@@ -68,17 +84,15 @@ func TestRandomTreeMerge(t *testing.T) {
 	aclList, err := list.NewTestDerivedAcl(spaceId, keys)
 	storage := createStorage(treeId, aclList)
 	changeCreator := objecttree.NewMockChangeCreator()
-	rnd := rand.New(rand.NewSource(time.Now().Unix()))
 	params := genParams{
 		prefix:     "peer1",
 		aclId:      aclList.Id(),
 		startIdx:   0,
-		levels:     10,
+		levels:     levels,
+		perLevel:   perlevel,
 		snapshotId: treeId,
 		prevHeads:  []string{treeId},
-		isSnapshot: func() bool {
+		isSnapshot: isSnapshot,
-			return rnd.Intn(100) > 80
-		},
 	}
 	initialRes := genChanges(changeCreator, params)
 	err = storage.TransactionAdd(initialRes.changes, initialRes.heads)
@@ -99,20 +113,20 @@ func TestRandomTreeMerge(t *testing.T) {
 		NewHeads:   initialRes.heads,
 		RawChanges: initialRes.changes,
 	})
-	time.Sleep(1000 * time.Millisecond)
+	time.Sleep(100 * time.Millisecond)
 	firstHeads := fx.handlers["peer1"].tree().Heads()
 	secondHeads := fx.handlers["peer2"].tree().Heads()
 	require.True(t, slice.UnsortedEquals(firstHeads, secondHeads))
 	params = genParams{
-		prefix:     "peer1",
+		prefix:   "peer1",
-		aclId:      aclList.Id(),
+		aclId:    aclList.Id(),
-		startIdx:   11,
+		startIdx: levels,
-		levels:     10,
+		levels:   levels,
+		perLevel: perlevel,
+
 		snapshotId: initialRes.snapshotId,
 		prevHeads:  initialRes.heads,
-		isSnapshot: func() bool {
+		isSnapshot: isSnapshot,
-			return rnd.Intn(100) > 80
-		},
 	}
 	peer1Res := genChanges(changeCreator, params)
 	params.prefix = "peer2"
@@ -125,10 +139,15 @@ func TestRandomTreeMerge(t *testing.T) {
 		NewHeads:   peer2Res.heads,
 		RawChanges: peer2Res.changes,
 	})
-	time.Sleep(1000 * time.Millisecond)
+	time.Sleep(100 * time.Millisecond)
 	fx.stop()
-	firstHeads = fx.handlers["peer1"].tree().Heads()
+	firstTree := fx.handlers["peer1"].tree()
-	secondHeads = fx.handlers["peer2"].tree().Heads()
+	secondTree := fx.handlers["peer2"].tree()
-	fmt.Println(firstHeads)
+	firstHeads = firstTree.Heads()
-	fmt.Println(secondHeads)
+	secondHeads = secondTree.Heads()
+	require.True(t, slice.UnsortedEquals(firstHeads, secondHeads))
+	require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...)))
+	firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage)
+	secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage)
+	require.True(t, firstStorage.Equal(secondStorage))
 }

commonspace/object/tree/synctree/utils_test.go

@@ -3,7 +3,6 @@ package synctree
 import (
 	"context"
 	"fmt"
-	"github.com/anytypeio/any-sync/commonspace/object/accountdata"
 	"github.com/anytypeio/any-sync/commonspace/object/acl/list"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@@ -275,7 +274,7 @@ func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage
 }
 
 func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) {
-	return objecttree.BuildTestableTree(aclList, storage)
+	return objecttree.BuildEmptyDataTestableTree(aclList, storage)
 }
 
 type fixtureDeps struct {
@@ -345,6 +344,7 @@ type genParams struct {
 	aclId      string
 	startIdx   int
 	levels     int
+	perLevel   int
 	snapshotId string
 	prevHeads  []string
 	isSnapshot func() bool
@@ -374,7 +374,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
 			snapshotId = newId
 			continue
 		}
-		perLevel := rnd.Intn(10)
+		perLevel := rnd.Intn(params.perLevel)
 		if perLevel == 0 {
 			perLevel = 1
 		}
@@ -383,7 +383,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
 			usedIds  = map[string]struct{}{}
 		)
 		for j := 0; j < perLevel; j++ {
-			// if we didn't connect with all prev ones
 			prevConns := rnd.Intn(len(prevHeads))
 			if prevConns == 0 {
 				prevConns = 1
@@ -391,6 +390,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
 			rnd.Shuffle(len(prevHeads), func(i, j int) {
 				prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i]
 			})
+			// if we didn't connect with all prev ones
 			if j == perLevel-1 && len(usedIds) != len(prevHeads) {
 				var unusedIds []string
 				for _, id := range prevHeads {
@@ -417,30 +417,3 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
 	res.snapshotId = snapshotId
 	return
 }
-
-func TestGenChanges(t *testing.T) {
-	treeId := "treeId"
-	spaceId := "spaceId"
-	keys, err := accountdata.NewRandom()
-	require.NoError(t, err)
-	aclList, err := list.NewTestDerivedAcl(spaceId, keys)
-	storage := createStorage(treeId, aclList)
-	creator := objecttree.NewMockChangeCreator()
-	rnd := rand.New(rand.NewSource(time.Now().Unix()))
-	params := genParams{
-		prefix:     "peerId",
-		aclId:      aclList.Id(),
-		startIdx:   0,
-		levels:     10,
-		snapshotId: treeId,
-		prevHeads:  []string{treeId},
-		isSnapshot: func() bool {
-			return rnd.Intn(100) > 80
-		},
-	}
-	res := genChanges(creator, params)
-	storage.TransactionAdd(res.changes, res.heads)
-	tr, err := createTestTree(aclList, storage)
-	require.NoError(t, err)
-	fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser))
-}

commonspace/object/tree/treestorage/inmemory.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
+	"github.com/anytypeio/any-sync/util/slice"
 	"sync"
 )
 
@@ -105,3 +106,21 @@ func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage {
 	other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes)
 	return other.(*InMemoryTreeStorage)
 }
+
+func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool {
+	if !slice.UnsortedEquals(t.heads, other.heads) {
+		return false
+	}
+	if len(t.changes) != len(other.changes) {
+		return false
+	}
+	for k, v := range t.changes {
+		if otherV, exists := other.changes[k]; exists {
+			if otherV.Id == v.Id {
+				continue
+			}
+		}
+		return false
+	}
+	return true
+}