benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tests and don't load data into changes

Browse Source
This commit is contained in:
mcrakhman 2023-04-17 23:51:17 +02:00
parent 125d8b4626
commit 678ec256bf
No known key found for this signature in database
GPG Key ID: DED12CFEF5B8396B
8 changed files with 171 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
commonspace/object/tree/objecttree/changebuilder.go
View File

@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.
return c.ChangeBuilder.Marshall(ch)
}
type emptyDataChangeBuilder struct {
ChangeBuilder
}
func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) {
panic("should not be called")
}
func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) {
panic("should not be called")
}
type ChangeBuilder interface {
Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error)
Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error)
@ -59,13 +71,28 @@ type ChangeBuilder interface {
Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error)
}
type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change
type changeBuilder struct {
rootChange *treechangeproto.RawTreeChangeWithId
keys crypto.KeyStorage
newChange newChangeFunc
}
func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder {
return &emptyDataChangeBuilder{&changeBuilder{
rootChange: rootChange,
keys: keys,
newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change {
c := NewChange(id, identity, ch, signature)
c.Data = nil
return c
},
}}
}
func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder {
return &changeBuilder{keys: keys, rootChange: rootChange}
return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange}
}
func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) {
@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
if err != nil {
return
}
ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature)
ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature)
rawIdChange = &treechangeproto.RawTreeChangeWithId{
RawChange: marshalledRawChange,
Id: id,
@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
if err != nil {
return
}
ch = NewChange(id, key, unmarshalled, raw.Signature)
ch = c.newChange(id, key, unmarshalled, raw.Signature)
return
}

14
commonspace/object/tree/objecttree/objecttree.go
View File

@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string)
}
}
if commonSnapshot == ot.tree.RootId() {
return ot.getChangesFromTree(theirHeads)
} else {
return ot.getChangesFromDB(commonSnapshot, theirHeads)
}
}
func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) {
return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads)
}
func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) {
return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads)
return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads)
}
func (ot *objectTree) snapshotPathIsActual() bool {

59
commonspace/object/tree/objecttree/objecttreefactory.go
View File

@ -51,6 +51,22 @@ func verifiableTreeDeps(
}
}
func emptyDataTreeDeps(
rootChange *treechangeproto.RawTreeChangeWithId,
treeStorage treestorage.TreeStorage,
aclList list.AclList) objectTreeDeps {
changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange)
treeBuilder := newTreeBuilder(treeStorage, changeBuilder)
return objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: treeBuilder,
treeStorage: treeStorage,
validator: newTreeValidator(),
rawChangeLoader: newStorageLoader(treeStorage, changeBuilder),
aclList: aclList,
}
}
func nonVerifiableTreeDeps(
rootChange *treechangeproto.RawTreeChangeWithId,
treeStorage treestorage.TreeStorage,
@ -80,6 +96,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList)
return createObjectTreeRoot(payload, 0, nil, aclList)
}
func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
rootChange, err := treeStorage.Root()
if err != nil {
return nil, err
}
deps := emptyDataTreeDeps(rootChange, treeStorage, aclList)
return buildObjectTree(deps)
}
func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
root, _ := treeStorage.Root()
changeBuilder := &nonVerifiableChangeBuilder{
ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root),
}
deps := objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: newTreeBuilder(treeStorage, changeBuilder),
treeStorage: treeStorage,
rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
validator: &noOpTreeValidator{},
aclList: aclList,
}
return buildObjectTree(deps)
}
func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
root, _ := treeStorage.Root()
changeBuilder := &nonVerifiableChangeBuilder{
ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root),
}
deps := objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: newTreeBuilder(treeStorage, changeBuilder),
treeStorage: treeStorage,
rawChangeLoader: newStorageLoader(treeStorage, changeBuilder),
validator: &noOpTreeValidator{},
aclList: aclList,
}
return buildObjectTree(deps)
}
func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
rootChange, err := treeStorage.Root()
if err != nil {

23
commonspace/object/tree/objecttree/rawloader.go
View File

@ -9,8 +9,9 @@ import (
)
type rawChangeLoader struct {
treeStorage treestorage.TreeStorage
changeBuilder ChangeBuilder
treeStorage treestorage.TreeStorage
changeBuilder ChangeBuilder
alwaysFromStorage bool
// buffers
idStack []string
@ -23,6 +24,12 @@ type rawCacheEntry struct {
position int
}
func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader {
loader := newRawChangeLoader(treeStorage, changeBuilder)
loader.alwaysFromStorage = true
return loader
}
func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader {
return &rawChangeLoader{
treeStorage: treeStorage,
@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang
}
}
func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
if commonSnapshot == t.root.Id && !r.alwaysFromStorage {
return r.loadFromTree(t, breakpoints)
} else {
return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints)
}
}
func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
var stack []*Change
for _, h := range t.headIds {
stack = append(stack, t.attached[h])
@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech
return convert(results)
}
func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
// resetting cache
r.cache = make(map[string]rawCacheEntry)
defer func() {

18
commonspace/object/tree/objecttree/testutils.go
View File

@ -2,7 +2,6 @@ package objecttree
import (
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/util/crypto"
@ -112,20 +111,3 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees
treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
return treeStorage
}
func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) {
root, _ := treeStorage.Root()
changeBuilder := &nonVerifiableChangeBuilder{
ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root),
}
deps := objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: newTreeBuilder(treeStorage, changeBuilder),
treeStorage: treeStorage,
rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
validator: &noOpTreeValidator{},
aclList: aclList,
}
return buildObjectTree(deps)
}

59
commonspace/object/tree/synctree/syncclient_test.go → commonspace/object/tree/synctree/syncprotocol_test.go
View File

@ -2,7 +2,6 @@ package synctree
import (
"context"
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/accountdata"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -60,7 +59,24 @@ func TestEmptyClientGetsFullHistory(t *testing.T) {
require.Len(t, fullResponseMsg.changes, 2)
}
func TestRandomTreeMerge(t *testing.T) {
func TestRandomMerge(t *testing.T) {
var (
rnd = rand.New(rand.NewSource(time.Now().Unix()))
levels = 20
perLevel = 20
)
testTreeMerge(t, levels, perLevel, func() bool {
return true
})
testTreeMerge(t, levels, perLevel, func() bool {
return false
})
testTreeMerge(t, levels, perLevel, func() bool {
return rnd.Intn(10) > 8
})
}
func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) {
treeId := "treeId"
spaceId := "spaceId"
keys, err := accountdata.NewRandom()
@ -68,17 +84,15 @@ func TestRandomTreeMerge(t *testing.T) {
aclList, err := list.NewTestDerivedAcl(spaceId, keys)
storage := createStorage(treeId, aclList)
changeCreator := objecttree.NewMockChangeCreator()
rnd := rand.New(rand.NewSource(time.Now().Unix()))
params := genParams{
prefix: "peer1",
aclId: aclList.Id(),
startIdx: 0,
levels: 10,
levels: levels,
perLevel: perlevel,
snapshotId: treeId,
prevHeads: []string{treeId},
isSnapshot: func() bool {
return rnd.Intn(100) > 80
},
isSnapshot: isSnapshot,
}
initialRes := genChanges(changeCreator, params)
err = storage.TransactionAdd(initialRes.changes, initialRes.heads)
@ -99,20 +113,20 @@ func TestRandomTreeMerge(t *testing.T) {
NewHeads: initialRes.heads,
RawChanges: initialRes.changes,
})
time.Sleep(1000 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
firstHeads := fx.handlers["peer1"].tree().Heads()
secondHeads := fx.handlers["peer2"].tree().Heads()
require.True(t, slice.UnsortedEquals(firstHeads, secondHeads))
params = genParams{
prefix: "peer1",
aclId: aclList.Id(),
startIdx: 11,
levels: 10,
prefix: "peer1",
aclId: aclList.Id(),
startIdx: levels,
levels: levels,
perLevel: perlevel,
snapshotId: initialRes.snapshotId,
prevHeads: initialRes.heads,
isSnapshot: func() bool {
return rnd.Intn(100) > 80
},
isSnapshot: isSnapshot,
}
peer1Res := genChanges(changeCreator, params)
params.prefix = "peer2"
@ -125,10 +139,15 @@ func TestRandomTreeMerge(t *testing.T) {
NewHeads: peer2Res.heads,
RawChanges: peer2Res.changes,
})
time.Sleep(1000 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
fx.stop()
firstHeads = fx.handlers["peer1"].tree().Heads()
secondHeads = fx.handlers["peer2"].tree().Heads()
fmt.Println(firstHeads)
fmt.Println(secondHeads)
firstTree := fx.handlers["peer1"].tree()
secondTree := fx.handlers["peer2"].tree()
firstHeads = firstTree.Heads()
secondHeads = secondTree.Heads()
require.True(t, slice.UnsortedEquals(firstHeads, secondHeads))
require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...)))
firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage)
secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage)
require.True(t, firstStorage.Equal(secondStorage))
}

35
commonspace/object/tree/synctree/utils_test.go
View File

@ -3,7 +3,6 @@ package synctree
import (
"context"
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/accountdata"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@ -275,7 +274,7 @@ func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage
}
func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) {
return objecttree.BuildTestableTree(aclList, storage)
return objecttree.BuildEmptyDataTestableTree(aclList, storage)
}
type fixtureDeps struct {
@ -345,6 +344,7 @@ type genParams struct {
aclId string
startIdx int
levels int
perLevel int
snapshotId string
prevHeads []string
isSnapshot func() bool
@ -374,7 +374,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
snapshotId = newId
continue
}
perLevel := rnd.Intn(10)
perLevel := rnd.Intn(params.perLevel)
if perLevel == 0 {
perLevel = 1
}
@ -383,7 +383,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
usedIds = map[string]struct{}{}
)
for j := 0; j < perLevel; j++ {
// if we didn't connect with all prev ones
prevConns := rnd.Intn(len(prevHeads))
if prevConns == 0 {
prevConns = 1
@ -391,6 +390,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
rnd.Shuffle(len(prevHeads), func(i, j int) {
prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i]
})
// if we didn't connect with all prev ones
if j == perLevel-1 && len(usedIds) != len(prevHeads) {
var unusedIds []string
for _, id := range prevHeads {
@ -417,30 +417,3 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge
res.snapshotId = snapshotId
return
}
func TestGenChanges(t *testing.T) {
treeId := "treeId"
spaceId := "spaceId"
keys, err := accountdata.NewRandom()
require.NoError(t, err)
aclList, err := list.NewTestDerivedAcl(spaceId, keys)
storage := createStorage(treeId, aclList)
creator := objecttree.NewMockChangeCreator()
rnd := rand.New(rand.NewSource(time.Now().Unix()))
params := genParams{
prefix: "peerId",
aclId: aclList.Id(),
startIdx: 0,
levels: 10,
snapshotId: treeId,
prevHeads: []string{treeId},
isSnapshot: func() bool {
return rnd.Intn(100) > 80
},
}
res := genChanges(creator, params)
storage.TransactionAdd(res.changes, res.heads)
tr, err := createTestTree(aclList, storage)
require.NoError(t, err)
fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser))
}

19
commonspace/object/tree/treestorage/inmemory.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/util/slice"
"sync"
)
@ -105,3 +106,21 @@ func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage {
other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes)
return other.(*InMemoryTreeStorage)
}
func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool {
if !slice.UnsortedEquals(t.heads, other.heads) {
return false
}
if len(t.changes) != len(other.changes) {
return false
}
for k, v := range t.changes {
if otherV, exists := other.changes[k]; exists {
if otherV.Id == v.Id {
continue
}
}
return false
}
return true
}