diff --git a/commonspace/commongetter.go b/commonspace/commongetter.go index 6807265b..74356281 100644 --- a/commonspace/commongetter.go +++ b/commonspace/commongetter.go @@ -4,20 +4,20 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "sync/atomic" ) type commonGetter struct { - treegetter.TreeGetter + treemanager.TreeManager spaceId string reservedObjects []syncobjectgetter.SyncObject spaceIsClosed *atomic.Bool } -func newCommonGetter(spaceId string, getter treegetter.TreeGetter, spaceIsClosed *atomic.Bool) *commonGetter { +func newCommonGetter(spaceId string, getter treemanager.TreeManager, spaceIsClosed *atomic.Bool) *commonGetter { return &commonGetter{ - TreeGetter: getter, + TreeManager: getter, spaceId: spaceId, spaceIsClosed: spaceIsClosed, } @@ -34,7 +34,7 @@ func (c *commonGetter) GetTree(ctx context.Context, spaceId, treeId string) (obj if obj := c.getReservedObject(treeId); obj != nil { return obj.(objecttree.ObjectTree), nil } - return c.TreeGetter.GetTree(ctx, spaceId, treeId) + return c.TreeManager.GetTree(ctx, spaceId, treeId) } func (c *commonGetter) getReservedObject(id string) syncobjectgetter.SyncObject { @@ -53,7 +53,7 @@ func (c *commonGetter) GetObject(ctx context.Context, objectId string) (obj sync if obj := c.getReservedObject(objectId); obj != nil { return obj, nil } - t, err := c.TreeGetter.GetTree(ctx, c.spaceId, objectId) + t, err := c.TreeManager.GetTree(ctx, c.spaceId, objectId) if err != nil { return } diff --git a/commonspace/config.go b/commonspace/config.go index befa3ac8..e5485068 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,6 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + KeepTreeDataInMemory bool `yaml:"keepTreeDataInMemory"` } diff --git a/commonspace/headsync/diffsyncer.go b/commonspace/headsync/diffsyncer.go index 9d3434c1..1b8a15dc 100644 --- a/commonspace/headsync/diffsyncer.go +++ b/commonspace/headsync/diffsyncer.go @@ -7,7 +7,7 @@ import ( "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -30,7 +30,7 @@ func newDiffSyncer( spaceId string, diff ldiff.Diff, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, storage spacestorage.SpaceStorage, clientFactory spacesyncproto.ClientFactory, syncStatus syncstatus.StatusUpdater, @@ -53,7 +53,7 @@ type diffSyncer struct { spaceId string diff ldiff.Diff peerManager peermanager.PeerManager - cache treegetter.TreeGetter + cache treemanager.TreeManager storage spacestorage.SpaceStorage clientFactory spacesyncproto.ClientFactory log logger.CtxLogger diff --git a/commonspace/headsync/diffsyncer_test.go b/commonspace/headsync/diffsyncer_test.go index ae40e5a7..e6558b61 100644 --- a/commonspace/headsync/diffsyncer_test.go +++ b/commonspace/headsync/diffsyncer_test.go @@ -12,7 +12,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" mock_treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage/mock_treestorage" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager/mock_peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -109,7 +109,7 @@ func TestDiffSyncer_Sync(t *testing.T) { diffMock := mock_ldiff.NewMockDiff(ctrl) peerManagerMock := mock_peermanager.NewMockPeerManager(ctrl) - cacheMock := mock_treegetter.NewMockTreeGetter(ctrl) + cacheMock := mock_treemanager.NewMockTreeManager(ctrl) stMock := mock_spacestorage.NewMockSpaceStorage(ctrl) clientMock := mock_spacesyncproto.NewMockDRPCSpaceSyncClient(ctrl) factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient { diff --git a/commonspace/headsync/headsync.go b/commonspace/headsync/headsync.go index 27a62733..27ca8b5e 100644 --- a/commonspace/headsync/headsync.go +++ b/commonspace/headsync/headsync.go @@ -6,7 +6,7 @@ import ( "github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -59,7 +59,7 @@ func NewHeadSync( configuration nodeconf.NodeConf, storage spacestorage.SpaceStorage, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, syncStatus syncstatus.StatusUpdater, credentialProvider credentialprovider.CredentialProvider, log logger.CtxLogger) HeadSync { diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index cb22d249..b3dc65a1 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto. return c.ChangeBuilder.Marshall(ch) } +type emptyDataChangeBuilder struct { + ChangeBuilder +} + +func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + +func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + type ChangeBuilder interface { Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) @@ -59,13 +71,28 @@ type ChangeBuilder interface { Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error) } +type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change + type changeBuilder struct { rootChange *treechangeproto.RawTreeChangeWithId keys crypto.KeyStorage + newChange newChangeFunc +} + +func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { + return &emptyDataChangeBuilder{&changeBuilder{ + rootChange: rootChange, + keys: keys, + newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { + c := NewChange(id, identity, ch, nil) + c.Data = nil + return c + }, + }} } func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { - return &changeBuilder{keys: keys, rootChange: rootChange} + return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange} } func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) { @@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange * if err != nil { return } - ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature) + ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature) rawIdChange = &treechangeproto.RawTreeChangeWithId{ RawChange: marshalledRawChange, Id: id, @@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, if err != nil { return } - ch = NewChange(id, key, unmarshalled, raw.Signature) + ch = c.newChange(id, key, unmarshalled, raw.Signature) return } diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 430329c2..5f2d98e4 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string) } } - if commonSnapshot == ot.tree.RootId() { - return ot.getChangesFromTree(theirHeads) - } else { - return ot.getChangesFromDB(commonSnapshot, theirHeads) - } -} - -func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads) -} - -func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads) + return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads) } func (ot *objectTree) snapshotPathIsActual() bool { diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index dbb31321..7ec9fac9 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -2,83 +2,19 @@ package objecttree import ( "context" - "crypto/rand" "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" - "github.com/anytypeio/any-sync/util/crypto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "testing" ) -type mockKeyStorage struct { - key crypto.PubKey -} - -func newKeyStorage() mockKeyStorage { - _, pk, _ := crypto.GenerateEd25519Key(rand.Reader) - return mockKeyStorage{pk} -} - -func (m mockKeyStorage) PubKeyFromProto(protoBytes []byte) (crypto.PubKey, error) { - return m.key, nil -} - -type mockChangeCreator struct{} - -func (c *mockChangeCreator) createRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.RootChange{ - AclHeadId: aclId, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.TreeChange{ - TreeHeadIds: prevIds, - AclHeadId: aclId, - SnapshotBaseId: snapshotId, - ChangesData: nil, - IsSnapshot: isSnapshot, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { - root := c.createRoot(treeId, aclHeadId) - treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) - return treeStorage -} - type testTreeContext struct { aclList list.AclList treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder - changeCreator *mockChangeCreator + changeCreator *MockChangeCreator objTree ObjectTree } @@ -91,12 +27,12 @@ func prepareAclList(t *testing.T) list.AclList { return aclList } -func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) +func prepareTreeDeps(aclList list.AclList) (*MockChangeCreator, objectTreeDeps) { + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, @@ -110,23 +46,9 @@ func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) } func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - // check build - objTree, err := buildObjectTree(deps) + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) + objTree, err := BuildTestableTree(aclList, treeStorage) require.NoError(t, err, "building tree should be without error") // check tree iterate @@ -140,7 +62,6 @@ func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { return testTreeContext{ aclList: aclList, treeStorage: treeStorage, - changeBuilder: changeBuilder, changeCreator: changeCreator, objTree: objTree, } @@ -156,8 +77,8 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -201,7 +122,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("0", aclList.Head().Id, "", true, ""), + changeCreator.CreateRaw("0", aclList.Head().Id, "", true, ""), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -225,7 +146,33 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + } + payload := RawChangesPayload{ + NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, + RawChanges: rawChanges, + } + res, err := objTree.AddRawChanges(context.Background(), payload) + require.NoError(t, err, "adding changes should be without error") + + // check result + assert.Equal(t, []string{"0"}, res.OldHeads) + assert.Equal(t, []string{"0"}, res.Heads) + assert.Equal(t, 0, len(res.Added)) + assert.Equal(t, Nothing, res.Mode) + + // check tree heads + assert.Equal(t, []string{"0"}, objTree.Heads()) + }) + + t.Run("add not connected changes", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + objTree := ctx.objTree + + // this change could in theory replace current snapshot, we should prevent that + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Head().Id, "0", true, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -251,10 +198,10 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -301,9 +248,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -325,12 +272,12 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload := RawChangesPayload{ @@ -404,13 +351,13 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), // main difference from tree example - changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), } payload := RawChangesPayload{ @@ -485,9 +432,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -499,9 +446,9 @@ func TestObjectTree(t *testing.T) { require.Equal(t, "3", objTree.Root().Id) rawChanges = []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload = RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -545,12 +492,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -576,12 +523,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -623,4 +570,40 @@ func TestObjectTree(t *testing.T) { assert.Equal(t, []string{"0"}, iterChangesId) assert.Equal(t, "0", hTree.Root().Id) }) + + t.Run("validate correct tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.NoError(t, err) + }) + + t.Run("fail to validate not connected tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.Equal(t, ErrHasInvalidChanges, err) + }) } diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index b3b757fb..a913d183 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -33,7 +33,11 @@ type objectTreeDeps struct { aclList list.AclList } -func defaultObjectTreeDeps( +type BuildObjectTreeFunc = func(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) + +var defaultObjectTreeDeps = verifiableTreeDeps + +func verifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { @@ -49,11 +53,27 @@ func defaultObjectTreeDeps( } } +func emptyDataTreeDeps( + rootChange *treechangeproto.RawTreeChangeWithId, + treeStorage treestorage.TreeStorage, + aclList list.AclList) objectTreeDeps { + changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange) + treeBuilder := newTreeBuilder(treeStorage, changeBuilder) + return objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: treeBuilder, + treeStorage: treeStorage, + validator: newTreeValidator(), + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + aclList: aclList, + } +} + func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { - changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(nil, rootChange)} + changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(newMockKeyStorage(), rootChange)} treeBuilder := newTreeBuilder(treeStorage, changeBuilder) return objectTreeDeps{ changeBuilder: changeBuilder, @@ -78,6 +98,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) return createObjectTreeRoot(payload, 0, nil, aclList) } +func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { + rootChange, err := treeStorage.Root() + if err != nil { + return nil, err + } + deps := emptyDataTreeDeps(rootChange, treeStorage, aclList) + return buildObjectTree(deps) +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + +func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { rootChange, err := treeStorage.Root() if err != nil { diff --git a/commonspace/object/tree/objecttree/objecttreevalidator.go b/commonspace/object/tree/objecttree/objecttreevalidator.go index 937dcc96..e2816246 100644 --- a/commonspace/object/tree/objecttree/objecttreevalidator.go +++ b/commonspace/object/tree/objecttree/objecttreevalidator.go @@ -1,10 +1,12 @@ package objecttree import ( + "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/util/slice" ) type ObjectTreeValidator interface { @@ -88,11 +90,23 @@ func (v *objectTreeValidator) validateChange(tree *Tree, aclList list.AclList, c } func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) { - treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes) + treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil) if err != nil { return } - - _, err = BuildObjectTree(treeStorage, aclList) + tree, err := BuildObjectTree(treeStorage, aclList) + if err != nil { + return + } + res, err := tree.AddRawChanges(context.Background(), RawChangesPayload{ + NewHeads: payload.Heads, + RawChanges: payload.Changes, + }) + if err != nil { + return + } + if !slice.UnsortedEquals(res.Heads, payload.Heads) { + return ErrHasInvalidChanges + } return } diff --git a/commonspace/object/tree/objecttree/rawloader.go b/commonspace/object/tree/objecttree/rawloader.go index cae32cb3..c9234b5b 100644 --- a/commonspace/object/tree/objecttree/rawloader.go +++ b/commonspace/object/tree/objecttree/rawloader.go @@ -9,8 +9,9 @@ import ( ) type rawChangeLoader struct { - treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder + treeStorage treestorage.TreeStorage + changeBuilder ChangeBuilder + alwaysFromStorage bool // buffers idStack []string @@ -23,6 +24,12 @@ type rawCacheEntry struct { position int } +func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { + loader := newRawChangeLoader(treeStorage, changeBuilder) + loader.alwaysFromStorage = true + return loader +} + func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { return &rawChangeLoader{ treeStorage: treeStorage, @@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang } } -func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { + if commonSnapshot == t.root.Id && !r.alwaysFromStorage { + return r.loadFromTree(t, breakpoints) + } else { + return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints) + } +} + +func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { var stack []*Change for _, h := range t.headIds { stack = append(stack, t.attached[h]) @@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech return convert(results) } -func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { // resetting cache r.cache = make(map[string]rawCacheEntry) defer func() { diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go new file mode 100644 index 00000000..d18eb859 --- /dev/null +++ b/commonspace/object/tree/objecttree/testutils.go @@ -0,0 +1,117 @@ +package objecttree + +import ( + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/util/crypto" + libcrypto "github.com/libp2p/go-libp2p/core/crypto" +) + +type mockPubKey struct { +} + +const mockKeyValue = "mockKey" + +func (m mockPubKey) Equals(key crypto.Key) bool { + return true +} + +func (m mockPubKey) Raw() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Encrypt(message []byte) ([]byte, error) { + return message, nil +} + +func (m mockPubKey) Verify(data []byte, sig []byte) (bool, error) { + return true, nil +} + +func (m mockPubKey) Marshall() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Storage() []byte { + return []byte(mockKeyValue) +} + +func (m mockPubKey) Account() string { + return mockKeyValue +} + +func (m mockPubKey) Network() string { + return mockKeyValue +} + +func (m mockPubKey) PeerId() string { + return mockKeyValue +} + +func (m mockPubKey) LibP2P() (libcrypto.PubKey, error) { + return nil, fmt.Errorf("can't be converted in libp2p") +} + +type mockKeyStorage struct { +} + +func newMockKeyStorage() mockKeyStorage { + return mockKeyStorage{} +} + +func (m mockKeyStorage) PubKeyFromProto(protoBytes []byte) (crypto.PubKey, error) { + return mockPubKey{}, nil +} + +type MockChangeCreator struct{} + +func NewMockChangeCreator() *MockChangeCreator { + return &MockChangeCreator{} +} + +func (c *MockChangeCreator) CreateRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.RootChange{ + AclHeadId: aclId, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.TreeChange{ + TreeHeadIds: prevIds, + AclHeadId: aclId, + SnapshotBaseId: snapshotId, + ChangesData: nil, + IsSnapshot: isSnapshot, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { + root := c.CreateRoot(treeId, aclHeadId) + treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) + return treeStorage +} diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 3ab6cf1b..07eb8878 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncClient,SyncTree,ReceiveQueue,HeadNotifiable) +// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncTree,ReceiveQueue,HeadNotifiable) // Package mock_synctree is a generated GoMock package. package mock_synctree @@ -18,115 +18,6 @@ import ( gomock "github.com/golang/mock/gomock" ) -// MockSyncClient is a mock of SyncClient interface. -type MockSyncClient struct { - ctrl *gomock.Controller - recorder *MockSyncClientMockRecorder -} - -// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. -type MockSyncClientMockRecorder struct { - mock *MockSyncClient -} - -// NewMockSyncClient creates a new mock instance. -func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { - mock := &MockSyncClient{ctrl: ctrl} - mock.recorder = &MockSyncClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { - return m.recorder -} - -// Broadcast mocks base method. -func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Broadcast indicates an expected call of Broadcast. -func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) -} - -// CreateFullSyncRequest mocks base method. -func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. -func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) -} - -// CreateFullSyncResponse mocks base method. -func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. -func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) -} - -// CreateHeadUpdate mocks base method. -func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. -func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) -} - -// CreateNewTreeRequest mocks base method. -func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateNewTreeRequest") - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. -func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) -} - -// SendWithReply mocks base method. -func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendWithReply indicates an expected call of SendWithReply. -func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) -} - // MockSyncTree is a mock of SyncTree interface. type MockSyncTree struct { ctrl *gomock.Controller diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/object/tree/synctree/syncclient.go deleted file mode 100644 index 5a62c632..00000000 --- a/commonspace/object/tree/synctree/syncclient.go +++ /dev/null @@ -1,66 +0,0 @@ -//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncClient,SyncTree,ReceiveQueue,HeadNotifiable -package synctree - -import ( - "context" - "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/objectsync" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/nodeconf" -) - -type SyncClient interface { - RequestFactory - Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) - SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) -} - -type syncClient struct { - objectsync.MessagePool - RequestFactory - spaceId string - configuration nodeconf.NodeConf -} - -func newSyncClient( - spaceId string, - pool objectsync.MessagePool, - factory RequestFactory, - configuration nodeconf.NodeConf) SyncClient { - return &syncClient{ - MessagePool: pool, - RequestFactory: factory, - configuration: configuration, - spaceId: spaceId, - } -} - -func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") - if err != nil { - return - } - return s.MessagePool.Broadcast(ctx, objMsg) -} - -func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) - if err != nil { - return - } - return s.MessagePool.SendPeer(ctx, peerId, objMsg) -} - -func marshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { - payload, err := message.Marshal() - if err != nil { - return - } - objMsg = &spacesyncproto.ObjectSyncMessage{ - ReplyId: replyId, - Payload: payload, - ObjectId: objectId, - SpaceId: spaceId, - } - return -} diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go new file mode 100644 index 00000000..d7c7a935 --- /dev/null +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -0,0 +1,159 @@ +package synctree + +import ( + "context" + "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/util/slice" + "github.com/stretchr/testify/require" + "math/rand" + "testing" + "time" +) + +func TestEmptyClientGetsFullHistory(t *testing.T) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + require.NoError(t, err) + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"peer2"}, + "peer2": []string{"peer1"}, + }, + emptyTrees: []string{"peer2"}, + } + fx := newProtocolFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, true, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + require.Equal(t, []string{"1"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + + var fullResponseMsg msgDescription + for _, msg := range logMsgs { + descr := msg.description() + if descr.name == "FullSyncResponse" { + fullResponseMsg = descr + } + } + // that means that we got not only the last snapshot, but also the first one + require.Len(t, fullResponseMsg.changes, 2) +} + +func TestTreeFuzzyMerge(t *testing.T) { + var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) + levels = 20 + perLevel = 20 + rounds = 10 + ) + for i := 0; i < rounds; i++ { + testTreeMerge(t, levels, perLevel, func() bool { + return true + }) + testTreeMerge(t, levels, perLevel, func() bool { + return false + }) + testTreeMerge(t, levels, perLevel, func() bool { + return rnd.Intn(10) > 8 + }) + levels += 2 + } +} + +func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + params := genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: 0, + levels: levels, + perLevel: perlevel, + snapshotId: treeId, + prevHeads: []string{treeId}, + isSnapshot: isSnapshot, + } + // generating initial tree + initialRes := genChanges(changeCreator, params) + err = storage.TransactionAdd(initialRes.changes, initialRes.heads) + require.NoError(t, err) + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"node1"}, + "peer2": []string{"node1"}, + "node1": []string{"peer1", "peer2"}, + }, + emptyTrees: []string{"peer2", "node1"}, + } + fx := newProtocolFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: initialRes.heads, + RawChanges: initialRes.changes, + }) + time.Sleep(50 * time.Millisecond) + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + params = genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: levels, + levels: levels, + perLevel: perlevel, + + snapshotId: initialRes.snapshotId, + prevHeads: initialRes.heads, + isSnapshot: isSnapshot, + } + // generating different additions to the tree for different peers + peer1Res := genChanges(changeCreator, params) + params.prefix = "peer2" + peer2Res := genChanges(changeCreator, params) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer1Res.heads, + RawChanges: peer1Res.changes, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer2Res.heads, + RawChanges: peer2Res.changes, + }) + time.Sleep(50 * time.Millisecond) + fx.stop() + firstTree := fx.handlers["peer1"].tree() + secondTree := fx.handlers["peer2"].tree() + firstHeads = firstTree.Heads() + secondHeads = secondTree.Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...))) + firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage) + secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage) + require.True(t, firstStorage.Equal(secondStorage)) +} diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 69d077e5..dc87a3c4 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncTree,ReceiveQueue,HeadNotifiable package synctree import ( @@ -43,7 +44,7 @@ type SyncTree interface { type syncTree struct { objecttree.ObjectTree synchandler.SyncHandler - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater notifiable HeadNotifiable listener updatelistener.UpdateListener @@ -54,16 +55,13 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var buildObjectTree = objecttree.BuildObjectTree -var createSyncClient = newSyncClient - type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) } type BuildDeps struct { SpaceId string - ObjectSync objectsync.ObjectSync + SyncClient objectsync.SyncClient Configuration nodeconf.NodeConf HeadNotifiable HeadNotifiable Listener updatelistener.UpdateListener @@ -73,6 +71,7 @@ type BuildDeps struct { OnClose func(id string) SyncStatus syncstatus.StatusUpdater PeerGetter ResponsiblePeersGetter + BuildObjectTree objecttree.BuildObjectTreeFunc WaitTreeRemoteSync bool } @@ -94,15 +93,11 @@ func PutSyncTree(ctx context.Context, payload treestorage.TreeStorageCreatePaylo } func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t SyncTree, err error) { - objTree, err := buildObjectTree(deps.TreeStorage, deps.AclList) + objTree, err := deps.BuildObjectTree(deps.TreeStorage, deps.AclList) if err != nil { return } - syncClient := createSyncClient( - deps.SpaceId, - deps.ObjectSync.MessagePool(), - sharedFactory, - deps.Configuration) + syncClient := deps.SyncClient syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, @@ -243,7 +238,7 @@ func (s *syncTree) SyncWithPeer(ctx context.Context, peerId string) (err error) s.Lock() defer s.Unlock() headUpdate := s.syncClient.CreateHeadUpdate(s, nil) - return s.syncClient.SendWithReply(ctx, peerId, headUpdate, "") + return s.syncClient.SendWithReply(ctx, peerId, headUpdate.RootChange.Id, headUpdate, "") } func (s *syncTree) afterBuild() { diff --git a/commonspace/object/tree/synctree/synctree_test.go b/commonspace/object/tree/synctree/synctree_test.go index 7b1b804d..9a9255ec 100644 --- a/commonspace/object/tree/synctree/synctree_test.go +++ b/commonspace/object/tree/synctree/synctree_test.go @@ -4,11 +4,11 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener/mock_updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/nodeconf" "github.com/golang/mock/gomock" @@ -18,7 +18,7 @@ import ( type syncTreeMatcher struct { objTree objecttree.ObjectTree - client SyncClient + client objectsync.SyncClient listener updatelistener.UpdateListener } @@ -34,8 +34,8 @@ func (s syncTreeMatcher) String() string { return "" } -func syncClientFuncCreator(client SyncClient) func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.NodeConf) SyncClient { - return func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.NodeConf) SyncClient { +func syncClientFuncCreator(client objectsync.SyncClient) func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.NodeConf) objectsync.SyncClient { + return func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.NodeConf) objectsync.SyncClient { return client } } @@ -46,7 +46,7 @@ func Test_BuildSyncTree(t *testing.T) { defer ctrl.Finish() updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) tr := &syncTree{ ObjectTree: objTreeMock, diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index de79d0de..dab63e92 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -4,6 +4,7 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" @@ -15,7 +16,7 @@ import ( type syncTreeHandler struct { objTree objecttree.ObjectTree - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater handlerLock sync.Mutex spaceId string @@ -24,7 +25,7 @@ type syncTreeHandler struct { const maxQueueSize = 5 -func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { +func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient objectsync.SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { return &syncTreeHandler{ objTree: objTree, syncClient: syncClient, @@ -81,15 +82,21 @@ func (s *syncTreeHandler) handleHeadUpdate( fullRequest *treechangeproto.TreeSyncMessage isEmptyUpdate = len(update.Changes) == 0 objTree = s.objTree + treeId = objTree.Id() ) - - log := log.With(zap.Strings("heads", objTree.Heads()), zap.String("treeId", objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("update heads", update.Heads), + zap.String("treeId", treeId), + zap.String("spaceId", s.spaceId), + zap.Int("len(update changes)", len(update.Changes))) log.DebugCtx(ctx, "received head update message") defer func() { if err != nil { log.With(zap.Error(err)).Debug("head update finished with error") } else if fullRequest != nil { + cnt := fullRequest.Content.GetFullSyncRequest() + log = log.With(zap.Strings("request heads", cnt.Heads), zap.Int("len(request changes)", len(cnt.Changes))) log.DebugCtx(ctx, "sending full sync request") } else { if !isEmptyUpdate { @@ -112,7 +119,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } if s.alreadyHasHeads(objTree, update.Heads) { @@ -136,7 +143,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } func (s *syncTreeHandler) handleFullSyncRequest( @@ -148,22 +155,25 @@ func (s *syncTreeHandler) handleFullSyncRequest( fullResponse *treechangeproto.TreeSyncMessage header = s.objTree.Header() objTree = s.objTree + treeId = s.objTree.Id() ) log := log.With(zap.String("senderId", senderId), - zap.Strings("heads", request.Heads), - zap.String("treeId", s.objTree.Id()), + zap.Strings("request heads", request.Heads), + zap.String("treeId", treeId), zap.String("replyId", replyId), - zap.String("spaceId", s.spaceId)) + zap.String("spaceId", s.spaceId), + zap.Int("len(request changes)", len(request.Changes))) log.DebugCtx(ctx, "received full sync request message") defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(err, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treeId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) return } else if fullResponse != nil { + cnt := fullResponse.Content.GetFullSyncResponse() + log = log.With(zap.Strings("response heads", cnt.Heads), zap.Int("len(response changes)", len(cnt.Changes))) log.DebugCtx(ctx, "full sync response sent") } }() @@ -182,7 +192,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( return } - return s.syncClient.SendWithReply(ctx, senderId, fullResponse, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullResponse, replyId) } func (s *syncTreeHandler) handleFullSyncResponse( @@ -191,8 +201,13 @@ func (s *syncTreeHandler) handleFullSyncResponse( response *treechangeproto.TreeFullSyncResponse) (err error) { var ( objTree = s.objTree + treeId = s.objTree.Id() ) - log := log.With(zap.Strings("heads", response.Heads), zap.String("treeId", s.objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("heads", response.Heads), + zap.String("treeId", treeId), + zap.String("spaceId", s.spaceId), + zap.Int("len(changes)", len(response.Changes))) log.DebugCtx(ctx, "received full sync response message") defer func() { diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index 052479d2..401f9989 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -3,13 +3,14 @@ package synctree import ( "context" "fmt" + "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "sync" "testing" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/golang/mock/gomock" @@ -54,7 +55,7 @@ func (t *testObjTreeMock) TryRLock() bool { type syncHandlerFixture struct { ctrl *gomock.Controller - syncClientMock *mock_synctree.MockSyncClient + syncClientMock *mock_objectsync.MockSyncClient objectTreeMock *testObjTreeMock receiveQueueMock ReceiveQueue @@ -63,7 +64,7 @@ type syncHandlerFixture struct { func newSyncHandlerFixture(t *testing.T) *syncHandlerFixture { ctrl := gomock.NewController(t) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objectTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) receiveQueue := newReceiveQueue(5) @@ -89,6 +90,13 @@ func (fx *syncHandlerFixture) stop() { func TestSyncHandler_HandleHeadUpdate(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullRequest := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncRequest{ + FullSyncRequest: &treechangeproto.TreeFullSyncRequest{}, + }, + }, + } t.Run("head update non empty all heads added", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -102,7 +110,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).Times(2) @@ -113,7 +121,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, nil) - fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2", "h1"}) fx.objectTreeMock.EXPECT().HasChanges(gomock.Eq([]string{"h1"})).Return(true) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) @@ -132,8 +139,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -148,7 +154,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -166,7 +172,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -187,15 +193,14 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -213,7 +218,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -226,6 +231,13 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullResponse := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncResponse{ + FullSyncResponse: &treechangeproto.TreeFullSyncResponse{}, + }, + }, + } t.Run("full sync request with change", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -239,8 +251,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Header().Return(nil) @@ -255,7 +266,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -273,8 +284,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -285,7 +295,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -303,9 +313,8 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") objectMsg.RequestId = replyId - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -313,7 +322,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq(replyId)) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq(replyId)) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -331,7 +340,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -348,7 +357,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, fmt.Errorf("")) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Any(), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Any(), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.Error(t, err) @@ -372,7 +381,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). @@ -405,7 +414,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 561d850c..7c401a5e 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" "github.com/gogo/protobuf/proto" "go.uber.org/zap" "time" @@ -45,13 +46,8 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e } func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() - objMsg, err := marshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") - if err != nil { - return - } - - resp, err := t.deps.ObjectSync.MessagePool().SendSync(ctx, peerId, objMsg) + newTreeRequest := t.deps.SyncClient.CreateNewTreeRequest() + resp, err := t.deps.SyncClient.SendSync(ctx, peerId, t.treeId, newTreeRequest) if err != nil { return } @@ -117,9 +113,16 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage. if err != nil { return } - if resp.GetContent().GetFullSyncResponse() == nil { - err = fmt.Errorf("expected to get full sync response, but got something else") + switch { + case resp.GetContent().GetErrorResponse() != nil: + errResp := resp.GetContent().GetErrorResponse() + err = rpcerr.Err(errResp.ErrCode) return + case resp.GetContent().GetFullSyncResponse() == nil: + err = treechangeproto.ErrUnexpected + return + default: + break } fullSyncResp := resp.GetContent().GetFullSyncResponse() diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go new file mode 100644 index 00000000..d0b207b4 --- /dev/null +++ b/commonspace/object/tree/synctree/utils_test.go @@ -0,0 +1,448 @@ +package synctree + +import ( + "context" + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/commonspace/syncstatus" + "github.com/anytypeio/any-sync/net/peer" + "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" + "math/rand" + "sync" + "testing" + "time" +) + +// protocolMsg is a message used in sync protocol tests +type protocolMsg struct { + msg *spacesyncproto.ObjectSyncMessage + senderId string + receiverId string + userMsg *objecttree.RawChangesPayload +} + +// msgDescription is a representation of message used for checking the results of the test +type msgDescription struct { + name string + from string + to string + heads []string + changes []*treechangeproto.RawTreeChangeWithId +} + +func (p *protocolMsg) description() (descr msgDescription) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err := proto.Unmarshal(p.msg.Payload, unmarshalled) + if err != nil { + panic(err) + } + descr = msgDescription{ + from: p.senderId, + to: p.receiverId, + } + switch { + case unmarshalled.GetContent().GetHeadUpdate() != nil: + cnt := unmarshalled.GetContent().GetHeadUpdate() + descr.name = "HeadUpdate" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes + case unmarshalled.GetContent().GetFullSyncRequest() != nil: + cnt := unmarshalled.GetContent().GetFullSyncRequest() + descr.name = "FullSyncRequest" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes + case unmarshalled.GetContent().GetFullSyncResponse() != nil: + cnt := unmarshalled.GetContent().GetFullSyncResponse() + descr.name = "FullSyncResponse" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes + } + return +} + +// messageLog saves all messages that were sent during sync test +type messageLog struct { + batcher *mb.MB[protocolMsg] +} + +func newMessageLog() *messageLog { + return &messageLog{batcher: mb.New[protocolMsg](0)} +} + +func (m *messageLog) addMessage(msg protocolMsg) { + m.batcher.Add(context.Background(), msg) +} + +// testSyncHandler is the wrapper around individual tree to test sync protocol +type testSyncHandler struct { + synchandler.SyncHandler + batcher *mb.MB[protocolMsg] + peerId string + aclList list.AclList + log *messageLog + syncClient objectsync.SyncClient +} + +// createSyncHandler creates a sync handler when a tree is already created +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { + factory := objectsync.NewRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newTestSyncHandler(peerId, handler) +} + +// createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { + factory := objectsync.NewRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) + + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + +func newTestSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *testSyncHandler { + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } +} + +func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if h.SyncHandler != nil { + return h.SyncHandler.HandleMessage(ctx, senderId, request) + } + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(request.Payload, unmarshalled) + if err != nil { + return + } + if unmarshalled.Content.GetFullSyncResponse() == nil { + newTreeRequest := objectsync.NewRequestFactory().CreateNewTreeRequest() + var objMsg *spacesyncproto.ObjectSyncMessage + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return h.manager().SendPeer(context.Background(), senderId, objMsg) + } + fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() + treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, []string{unmarshalled.RootChange.Id}, nil) + tree, err := createTestTree(h.aclList, treeStorage) + if err != nil { + return + } + netTree := &broadcastTree{ + ObjectTree: tree, + SyncClient: h.syncClient, + } + res, err := netTree.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: fullSyncResponse.Heads, + RawChanges: fullSyncResponse.Changes, + }) + if err != nil { + return + } + h.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, h.syncClient, syncstatus.NewNoOpSyncStatus()) + var objMsg *spacesyncproto.ObjectSyncMessage + newTreeRequest := objectsync.NewRequestFactory().CreateHeadUpdate(netTree, res.Added) + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return h.manager().Broadcast(context.Background(), objMsg) +} + +func (h *testSyncHandler) manager() *testMessagePool { + if h.SyncHandler != nil { + return h.SyncHandler.(*syncTreeHandler).syncClient.MessagePool().(*testMessagePool) + } + return h.syncClient.MessagePool().(*testMessagePool) +} + +func (h *testSyncHandler) tree() *broadcastTree { + return h.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +} + +func (h *testSyncHandler) send(ctx context.Context, msg protocolMsg) (err error) { + return h.batcher.Add(ctx, msg) +} + +func (h *testSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + h.batcher.Add(ctx, protocolMsg{userMsg: &changes}) +} + +func (h *testSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for { + res, err := h.batcher.WaitOne(ctx) + if err != nil { + return + } + if res.userMsg != nil { + h.tree().Lock() + userRes, err := h.tree().AddRawChanges(ctx, *res.userMsg) + require.NoError(t, err) + fmt.Println("user add result", userRes.Heads) + h.tree().Unlock() + continue + } + err = h.HandleMessage(ctx, res.senderId, res.msg) + if err != nil { + fmt.Println("error handling message", err.Error()) + continue + } + } + }() +} + +// testMessagePool captures all other handlers and sends messages to them +type testMessagePool struct { + peerId string + handlers map[string]*testSyncHandler + log *messageLog +} + +func newTestMessagePool(peerId string, log *messageLog) *testMessagePool { + return &testMessagePool{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} +} + +func (m *testMessagePool) addHandler(peerId string, handler *testSyncHandler) { + m.handlers[peerId] = handler +} + +func (m *testMessagePool) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := protocolMsg{ + msg: msg, + senderId: m.peerId, + receiverId: peerId, + } + m.log.addMessage(pMsg) + return m.handlers[peerId].send(context.Background(), pMsg) +} + +func (m *testMessagePool) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + for _, handler := range m.handlers { + pMsg := protocolMsg{ + msg: msg, + senderId: m.peerId, + receiverId: handler.peerId, + } + m.log.addMessage(pMsg) + handler.send(context.Background(), pMsg) + } + return +} + +func (m *testMessagePool) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") +} + +func (m *testMessagePool) LastUsage() time.Time { + panic("should not be called") +} + +func (m *testMessagePool) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + panic("should not be called") +} + +func (m *testMessagePool) SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { + panic("should not be called") +} + +// broadcastTree is the tree that broadcasts changes to everyone when changes are added +// it is a simplified version of SyncTree which is easier to use in the test environment +type broadcastTree struct { + objecttree.ObjectTree + objectsync.SyncClient +} + +func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { + res, err := b.ObjectTree.AddRawChanges(ctx, changes) + if err != nil { + return objecttree.AddResult{}, err + } + upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) + b.SyncClient.Broadcast(ctx, upd) + return res, nil +} + +func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { + changeCreator := objecttree.NewMockChangeCreator() + st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) + return st +} + +func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { + return objecttree.BuildEmptyDataTestableTree(aclList, storage) +} + +type fixtureDeps struct { + aclList list.AclList + initStorage *treestorage.InMemoryTreeStorage + connectionMap map[string][]string + emptyTrees []string +} + +// protocolFixture is the test environment for sync protocol tests +type protocolFixture struct { + handlers map[string]*testSyncHandler + log *messageLog + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc +} + +func newProtocolFixture(t *testing.T, spaceId string, deps fixtureDeps) *protocolFixture { + var ( + handlers = map[string]*testSyncHandler{} + log = newMessageLog() + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(context.Background()) + ) + + for peerId := range deps.connectionMap { + var handler *testSyncHandler + if slices.Contains(deps.emptyTrees, peerId) { + handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) + } else { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler = createSyncHandler(peerId, spaceId, testTree, log) + } + handlers[peerId] = handler + } + for peerId, connectionMap := range deps.connectionMap { + handler := handlers[peerId] + manager := handler.manager() + for _, connectionId := range connectionMap { + manager.addHandler(connectionId, handlers[connectionId]) + } + } + return &protocolFixture{ + handlers: handlers, + log: log, + wg: &wg, + ctx: ctx, + cancel: cancel, + } +} + +func (p *protocolFixture) run(t *testing.T) { + for _, handler := range p.handlers { + handler.run(p.ctx, t, p.wg) + } +} + +func (p *protocolFixture) stop() { + p.cancel() + p.wg.Wait() +} + +// genParams is the parameters for genChanges +type genParams struct { + // prefix is the prefix which is added to change id + prefix string + aclId string + startIdx int + levels int + perLevel int + snapshotId string + prevHeads []string + isSnapshot func() bool +} + +// genResult is the result of genChanges +type genResult struct { + changes []*treechangeproto.RawTreeChangeWithId + heads []string + snapshotId string +} + +// genChanges generates several levels of tree changes where each level is connected only with previous one +func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { + src := rand.NewSource(time.Now().Unix()) + rnd := rand.New(src) + var ( + prevHeads []string + snapshotId = params.snapshotId + ) + prevHeads = append(prevHeads, params.prevHeads...) + + for i := 0; i < params.levels; i++ { + var ( + newHeads []string + usedIds = map[string]struct{}{} + ) + newChange := func(isSnapshot bool, idx int, prevIds []string) string { + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, idx) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, isSnapshot, prevIds...) + res.changes = append(res.changes, newCh) + return newId + } + if params.isSnapshot() { + newId := newChange(true, 0, prevHeads) + prevHeads = []string{newId} + snapshotId = newId + continue + } + perLevel := rnd.Intn(params.perLevel) + if perLevel == 0 { + perLevel = 1 + } + for j := 0; j < perLevel; j++ { + prevConns := rnd.Intn(len(prevHeads)) + if prevConns == 0 { + prevConns = 1 + } + rnd.Shuffle(len(prevHeads), func(i, j int) { + prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] + }) + // if we didn't connect with all prev ones + if j == perLevel-1 && len(usedIds) != len(prevHeads) { + var unusedIds []string + for _, id := range prevHeads { + if _, exists := usedIds[id]; !exists { + unusedIds = append(unusedIds, id) + } + } + prevHeads = unusedIds + prevConns = len(prevHeads) + } + var prevIds []string + for k := 0; k < prevConns; k++ { + prevIds = append(prevIds, prevHeads[k]) + usedIds[prevHeads[k]] = struct{}{} + } + newId := newChange(false, j, prevIds) + newHeads = append(newHeads, newId) + } + prevHeads = newHeads + } + res.heads = prevHeads + res.snapshotId = snapshotId + return +} diff --git a/commonspace/object/tree/treechangeproto/errors.go b/commonspace/object/tree/treechangeproto/errors.go new file mode 100644 index 00000000..375ebedb --- /dev/null +++ b/commonspace/object/tree/treechangeproto/errors.go @@ -0,0 +1,14 @@ +package treechangeproto + +import ( + "errors" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" +) + +var ( + errGroup = rpcerr.ErrGroup(ErrorCodes_ErrorOffset) + + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrorCodes_Unexpected)) + ErrGetTree = errGroup.Register(errors.New("tree not found"), uint64(ErrorCodes_GetTreeError)) + ErrFullSync = errGroup.Register(errors.New("full sync request error"), uint64(ErrorCodes_FullSyncRequestError)) +) diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index dfc74a42..eb331567 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -56,6 +56,13 @@ message RawTreeChangeWithId { string id = 2; } +enum ErrorCodes { + Unexpected = 0; + GetTreeError = 1; + FullSyncRequestError = 2; + ErrorOffset = 400; +} + message TreeSyncMessage { TreeSyncContentValue content = 1; RawTreeChangeWithId rootChange = 2; @@ -95,6 +102,7 @@ message TreeFullSyncResponse { // TreeErrorResponse is an error sent as a response for a full sync request message TreeErrorResponse { string error = 1; + uint64 errCode = 2; } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling diff --git a/commonspace/object/tree/treechangeproto/treechange.go b/commonspace/object/tree/treechangeproto/treechange.go index 3aba1fb0..92ef3f16 100644 --- a/commonspace/object/tree/treechangeproto/treechange.go +++ b/commonspace/object/tree/treechangeproto/treechange.go @@ -1,5 +1,7 @@ package treechangeproto +import "github.com/anytypeio/any-sync/net/rpc/rpcerr" + func WrapHeadUpdate(update *TreeHeadUpdate, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ @@ -30,7 +32,7 @@ func WrapFullResponse(response *TreeFullSyncResponse, rootChange *RawTreeChangeW func WrapError(err error, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ - Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{Error: err.Error()}}, + Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: rpcerr.Code(err)}}, }, RootChange: rootChange, } diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index 60eb619b..34d57487 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -22,6 +22,37 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +type ErrorCodes int32 + +const ( + ErrorCodes_Unexpected ErrorCodes = 0 + ErrorCodes_GetTreeError ErrorCodes = 1 + ErrorCodes_FullSyncRequestError ErrorCodes = 2 + ErrorCodes_ErrorOffset ErrorCodes = 400 +) + +var ErrorCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "GetTreeError", + 2: "FullSyncRequestError", + 400: "ErrorOffset", +} + +var ErrorCodes_value = map[string]int32{ + "Unexpected": 0, + "GetTreeError": 1, + "FullSyncRequestError": 2, + "ErrorOffset": 400, +} + +func (x ErrorCodes) String() string { + return proto.EnumName(ErrorCodes_name, int32(x)) +} + +func (ErrorCodes) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_5033f0301ef9b772, []int{0} +} + // RootChange is a root of a tree type RootChange struct { // AclHeadId is a cid of latest acl record at the time of tree creation @@ -691,7 +722,8 @@ func (m *TreeFullSyncResponse) GetSnapshotPath() []string { // TreeErrorResponse is an error sent as a response for a full sync request type TreeErrorResponse struct { - Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + ErrCode uint64 `protobuf:"varint,2,opt,name=errCode,proto3" json:"errCode,omitempty"` } func (m *TreeErrorResponse) Reset() { *m = TreeErrorResponse{} } @@ -734,6 +766,13 @@ func (m *TreeErrorResponse) GetError() string { return "" } +func (m *TreeErrorResponse) GetErrCode() uint64 { + if m != nil { + return m.ErrCode + } + return 0 +} + // TreeChangeInfo is used internally in Tree implementation for ease of marshalling type TreeChangeInfo struct { ChangeType string `protobuf:"bytes,1,opt,name=changeType,proto3" json:"changeType,omitempty"` @@ -788,6 +827,7 @@ func (m *TreeChangeInfo) GetChangePayload() []byte { } func init() { + proto.RegisterEnum("treechange.ErrorCodes", ErrorCodes_name, ErrorCodes_value) proto.RegisterType((*RootChange)(nil), "treechange.RootChange") proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange") proto.RegisterType((*RawTreeChange)(nil), "treechange.RawTreeChange") @@ -806,50 +846,54 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 677 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xcf, 0x4f, 0xd4, 0x40, - 0x14, 0xee, 0x74, 0x81, 0xb2, 0x8f, 0x05, 0x74, 0xe0, 0xd0, 0x10, 0xad, 0x4d, 0x63, 0x74, 0xbd, - 0x40, 0x82, 0x27, 0x8d, 0x09, 0x11, 0x04, 0x77, 0x43, 0x34, 0x64, 0x40, 0x4c, 0xbc, 0x0d, 0xed, - 0xc0, 0xd6, 0xec, 0x76, 0x6a, 0x67, 0x56, 0xb2, 0x7f, 0x80, 0x17, 0x4d, 0x88, 0xff, 0x92, 0x37, - 0x8f, 0x1c, 0x39, 0x1a, 0xf6, 0x1f, 0x31, 0x9d, 0x69, 0xb7, 0x3f, 0x76, 0x0f, 0xdc, 0xb8, 0x74, - 0xf7, 0x7d, 0x7d, 0xef, 0x7b, 0xdf, 0xfb, 0xe6, 0x47, 0x61, 0xc7, 0xe7, 0x83, 0x01, 0x8f, 0x44, - 0x4c, 0x7d, 0xb6, 0xc5, 0xcf, 0xbe, 0x32, 0x5f, 0x6e, 0xc9, 0x84, 0x31, 0xf5, 0xf0, 0x7b, 0x34, - 0xba, 0x60, 0x71, 0xc2, 0x25, 0xdf, 0x52, 0x4f, 0x51, 0x82, 0x37, 0x15, 0x82, 0xa1, 0x40, 0xbc, - 0x1b, 0x04, 0x40, 0x38, 0x97, 0x7b, 0x2a, 0xc4, 0x8f, 0xa0, 0x49, 0xfd, 0x7e, 0x87, 0xd1, 0xa0, - 0x1b, 0xd8, 0xc8, 0x45, 0xed, 0x26, 0x29, 0x00, 0x6c, 0x83, 0xa5, 0xba, 0x76, 0x03, 0xdb, 0x54, - 0xef, 0xf2, 0x10, 0x3b, 0x00, 0x9a, 0xf0, 0x64, 0x14, 0x33, 0xbb, 0xa1, 0x5e, 0x96, 0x90, 0x94, - 0x57, 0x86, 0x03, 0x26, 0x24, 0x1d, 0xc4, 0xf6, 0x9c, 0x8b, 0xda, 0x0d, 0x52, 0x00, 0x18, 0xc3, - 0x9c, 0x60, 0x2c, 0xb0, 0xe7, 0x5d, 0xd4, 0x6e, 0x11, 0xf5, 0x1f, 0x6f, 0xc0, 0x62, 0x18, 0xb0, - 0x48, 0x86, 0x72, 0x64, 0x2f, 0x28, 0x7c, 0x12, 0xe3, 0xa7, 0xb0, 0xac, 0xb9, 0x8f, 0xe8, 0xa8, - 0xcf, 0x69, 0x60, 0x5b, 0x2a, 0xa1, 0x0a, 0x7a, 0x57, 0x26, 0xc0, 0x49, 0xc2, 0x58, 0x36, 0x9a, - 0x0b, 0x4b, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd1, 0x6e, 0x92, 0x32, 0x54, 0x1d, 0xde, - 0xac, 0x0f, 0xff, 0x0c, 0x56, 0x44, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x97, 0x8a, 0xd4, 0x03, 0x3d, - 0x66, 0x0d, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x22, 0x65, 0x28, 0xed, - 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x24, 0x05, 0x50, 0xb5, 0x6a, 0xa1, 0x6e, - 0x55, 0xd9, 0x16, 0xab, 0x66, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x4c, 0x8d, 0xbd, 0xe8, 0xa2, 0xf6, - 0x22, 0x29, 0x21, 0xde, 0x7b, 0x58, 0x26, 0xf4, 0xb2, 0x64, 0x89, 0x0d, 0x56, 0x9c, 0x39, 0x88, - 0x14, 0x57, 0x1e, 0xa6, 0x22, 0x44, 0x78, 0x11, 0x51, 0x39, 0x4c, 0x98, 0xb2, 0xa2, 0x45, 0x0a, - 0xc0, 0xdb, 0x83, 0xb5, 0x0a, 0xd1, 0xe7, 0x50, 0xf6, 0xb4, 0xf2, 0x84, 0x5e, 0x6a, 0x28, 0x23, - 0x2c, 0x00, 0xbc, 0x02, 0x66, 0x98, 0xdb, 0x6a, 0x86, 0x81, 0x77, 0x85, 0x60, 0x35, 0xa5, 0x38, - 0x1e, 0x45, 0xfe, 0x07, 0x26, 0x04, 0xbd, 0x60, 0xf8, 0x35, 0x58, 0x3e, 0x8f, 0x24, 0x8b, 0xa4, - 0xaa, 0x5f, 0xda, 0x76, 0x37, 0x4b, 0xbb, 0x37, 0xcf, 0xde, 0xd3, 0x29, 0xa7, 0xb4, 0x3f, 0x64, - 0x24, 0x2f, 0xc0, 0x3b, 0x00, 0xc9, 0x64, 0x23, 0xab, 0x3e, 0x4b, 0xdb, 0x4f, 0xca, 0xe5, 0x33, - 0x24, 0x93, 0x52, 0x89, 0xf7, 0xc7, 0x84, 0xf5, 0x59, 0x2d, 0xf0, 0x1b, 0x80, 0x1e, 0xa3, 0xc1, - 0xa7, 0x38, 0xa0, 0x92, 0x65, 0xc2, 0x36, 0xea, 0xc2, 0x3a, 0x93, 0x8c, 0x8e, 0x41, 0x4a, 0xf9, - 0xf8, 0x10, 0x56, 0xcf, 0x87, 0xfd, 0x7e, 0xca, 0x4a, 0xd8, 0xb7, 0x21, 0x13, 0x72, 0x96, 0xb8, - 0x94, 0xe2, 0xa0, 0x9a, 0xd6, 0x31, 0x48, 0xbd, 0x12, 0x7f, 0x84, 0x07, 0x05, 0x24, 0x62, 0x1e, - 0x09, 0x7d, 0xda, 0x66, 0x38, 0x75, 0x50, 0xcb, 0xeb, 0x18, 0x64, 0xaa, 0x16, 0xef, 0xc3, 0x32, - 0x4b, 0x12, 0x9e, 0x4c, 0xc8, 0xe6, 0x14, 0xd9, 0xe3, 0x3a, 0xd9, 0x7e, 0x39, 0xa9, 0x63, 0x90, - 0x6a, 0xd5, 0xae, 0x05, 0xf3, 0xdf, 0x53, 0xab, 0xbc, 0x1f, 0x08, 0x56, 0xaa, 0x6e, 0xe0, 0x75, - 0x98, 0x4f, 0xdd, 0xc8, 0x4f, 0x9c, 0x0e, 0xf0, 0x2b, 0xb0, 0xb2, 0x23, 0x61, 0x9b, 0x6e, 0xe3, - 0x2e, 0x4b, 0x95, 0xe7, 0x63, 0x0f, 0x5a, 0xf9, 0x91, 0x3b, 0xa2, 0xb2, 0x67, 0x37, 0x14, 0x6f, - 0x05, 0xf3, 0x7e, 0x22, 0x58, 0x9b, 0x61, 0xe9, 0xfd, 0x88, 0xf9, 0x85, 0xf4, 0xc6, 0xaa, 0xaf, - 0xc8, 0xfd, 0xa8, 0x79, 0x01, 0x0f, 0xa7, 0x56, 0x34, 0x55, 0xa2, 0x56, 0x34, 0xbb, 0xf3, 0x75, - 0xe0, 0x9d, 0xea, 0xc5, 0xd4, 0xbd, 0xba, 0xd1, 0x39, 0xaf, 0xdd, 0xf3, 0x68, 0xea, 0x9e, 0x9f, - 0xba, 0x99, 0xcd, 0x19, 0x37, 0xf3, 0xee, 0xdb, 0xbf, 0xb7, 0x0e, 0xba, 0xbe, 0x75, 0xd0, 0xbf, - 0x5b, 0x07, 0xfd, 0x1e, 0x3b, 0xc6, 0xf5, 0xd8, 0x31, 0x6e, 0xc6, 0x8e, 0xf1, 0xe5, 0xf9, 0x1d, - 0xbf, 0x6d, 0x67, 0x0b, 0xea, 0xe7, 0xe5, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbc, 0xc0, 0xf7, - 0x30, 0x0d, 0x07, 0x00, 0x00, + // 741 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4f, 0x4f, 0xfb, 0x46, + 0x10, 0xf5, 0x3a, 0x81, 0x90, 0x49, 0x08, 0xe9, 0xc2, 0xc1, 0x42, 0xad, 0x6b, 0x59, 0x55, 0x1b, + 0xf5, 0x00, 0x12, 0x3d, 0xb5, 0xaa, 0x84, 0x4a, 0x0a, 0x24, 0x42, 0x6d, 0xd1, 0xf2, 0xa7, 0x12, + 0xb7, 0xc5, 0x9e, 0x10, 0x57, 0x89, 0xed, 0x7a, 0x37, 0xa5, 0xf9, 0x00, 0xbd, 0xb4, 0x12, 0xe2, + 0x2b, 0xf5, 0xf6, 0x3b, 0x72, 0xe4, 0xf8, 0x13, 0x7c, 0x91, 0x9f, 0xbc, 0x1b, 0x27, 0xb6, 0x93, + 0x03, 0x37, 0x2e, 0x89, 0xe7, 0x79, 0xe6, 0xed, 0x9b, 0x37, 0xde, 0x5d, 0x38, 0xf4, 0xa2, 0xf1, + 0x38, 0x0a, 0x45, 0xcc, 0x3d, 0xdc, 0x8f, 0x6e, 0xff, 0x40, 0x4f, 0xee, 0xcb, 0x04, 0x51, 0xfd, + 0x78, 0x43, 0x1e, 0xde, 0x61, 0x9c, 0x44, 0x32, 0xda, 0x57, 0xbf, 0x22, 0x07, 0xef, 0x29, 0x84, + 0xc2, 0x02, 0x71, 0x9f, 0x09, 0x00, 0x8b, 0x22, 0xd9, 0x55, 0x21, 0xfd, 0x1c, 0xea, 0xdc, 0x1b, + 0xf5, 0x90, 0xfb, 0x7d, 0xdf, 0x22, 0x0e, 0xe9, 0xd4, 0xd9, 0x02, 0xa0, 0x16, 0xd4, 0xd4, 0xaa, + 0x7d, 0xdf, 0x32, 0xd5, 0xbb, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, + 0xcb, 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, + 0x05, 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x99, 0xee, 0xc2, + 0x46, 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x57, 0xb0, 0xa9, 0xb9, + 0xcf, 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, + 0xe2, 0xac, 0x35, 0x07, 0x1a, 0x69, 0xdf, 0xba, 0x15, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, + 0x54, 0x6c, 0xde, 0x2c, 0x37, 0xff, 0x35, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, + 0xa9, 0x07, 0xba, 0xcd, 0x12, 0x9a, 0xae, 0xa3, 0x75, 0x88, 0x9f, 0xb9, 0xe4, 0xaa, 0xd9, 0x26, + 0xcb, 0x43, 0xe9, 0x3a, 0x09, 0x72, 0xff, 0x0c, 0xa7, 0x7d, 0xdd, 0x73, 0x9d, 0x2d, 0x80, 0xa2, + 0x55, 0xeb, 0x65, 0xab, 0xf2, 0xb6, 0xd4, 0x4a, 0xb6, 0xd8, 0x00, 0x81, 0xb8, 0x98, 0xa9, 0xb1, + 0x36, 0x1c, 0xd2, 0xd9, 0x60, 0x39, 0xc4, 0x3d, 0x85, 0x4d, 0xc6, 0xef, 0x73, 0x96, 0x58, 0x50, + 0x8b, 0x67, 0x0e, 0x12, 0xc5, 0x95, 0x85, 0xa9, 0x08, 0x11, 0xdc, 0x85, 0x5c, 0x4e, 0x12, 0x54, + 0x56, 0x34, 0xd9, 0x02, 0x70, 0xbb, 0xb0, 0x5d, 0x20, 0xfa, 0x3d, 0x90, 0x43, 0xad, 0x3c, 0xe1, + 0xf7, 0x1a, 0x9a, 0x11, 0x2e, 0x00, 0xda, 0x02, 0x33, 0xc8, 0x6c, 0x35, 0x03, 0xdf, 0x7d, 0x20, + 0xb0, 0x95, 0x52, 0x5c, 0x4c, 0x43, 0xef, 0x17, 0x14, 0x82, 0xdf, 0x21, 0xfd, 0x01, 0x6a, 0x5e, + 0x14, 0x4a, 0x0c, 0xa5, 0xaa, 0x6f, 0x1c, 0x38, 0x7b, 0xb9, 0xaf, 0x37, 0xcb, 0xee, 0xea, 0x94, + 0x6b, 0x3e, 0x9a, 0x20, 0xcb, 0x0a, 0xe8, 0x21, 0x40, 0x32, 0xff, 0x90, 0xd5, 0x3a, 0x8d, 0x83, + 0x2f, 0xf3, 0xe5, 0x2b, 0x24, 0xb3, 0x5c, 0x89, 0xfb, 0xbf, 0x09, 0x3b, 0xab, 0x96, 0xa0, 0x3f, + 0x02, 0x0c, 0x91, 0xfb, 0x57, 0xb1, 0xcf, 0x25, 0xce, 0x84, 0xed, 0x96, 0x85, 0xf5, 0xe6, 0x19, + 0x3d, 0x83, 0xe5, 0xf2, 0xe9, 0x19, 0x6c, 0x0d, 0x26, 0xa3, 0x51, 0xca, 0xca, 0xf0, 0xcf, 0x09, + 0x0a, 0xb9, 0x4a, 0x5c, 0x4a, 0x71, 0x52, 0x4c, 0xeb, 0x19, 0xac, 0x5c, 0x49, 0x7f, 0x85, 0xf6, + 0x02, 0x12, 0x71, 0x14, 0x0a, 0xbd, 0xdb, 0x56, 0x38, 0x75, 0x52, 0xca, 0xeb, 0x19, 0x6c, 0xa9, + 0x96, 0x1e, 0xc3, 0x26, 0x26, 0x49, 0x94, 0xcc, 0xc9, 0xaa, 0x8a, 0xec, 0x8b, 0x32, 0xd9, 0x71, + 0x3e, 0xa9, 0x67, 0xb0, 0x62, 0xd5, 0x51, 0x0d, 0xd6, 0xfe, 0x4a, 0xad, 0x72, 0xff, 0x21, 0xd0, + 0x2a, 0xba, 0x41, 0x77, 0x60, 0x2d, 0x75, 0x23, 0xdb, 0x71, 0x3a, 0xa0, 0xdf, 0x43, 0x6d, 0xb6, + 0x25, 0x2c, 0xd3, 0xa9, 0xbc, 0x65, 0x54, 0x59, 0x3e, 0x75, 0xa1, 0x99, 0x6d, 0xb9, 0x73, 0x2e, + 0x87, 0x56, 0x45, 0xf1, 0x16, 0x30, 0xf7, 0x5f, 0x02, 0xdb, 0x2b, 0x2c, 0x7d, 0x1f, 0x31, 0xff, + 0x11, 0xfd, 0x61, 0x95, 0x27, 0xf2, 0x3e, 0x6a, 0xba, 0xf0, 0xd9, 0xd2, 0x44, 0x53, 0x25, 0x6a, + 0xa2, 0xb3, 0x33, 0x5f, 0x07, 0xe9, 0xf9, 0x80, 0x49, 0xd2, 0x8d, 0x7c, 0xbd, 0x9f, 0xaa, 0x2c, + 0x0b, 0xdd, 0x6b, 0x3d, 0x66, 0xad, 0xa2, 0x1f, 0x0e, 0xa2, 0xd2, 0x0d, 0x40, 0x96, 0x6e, 0x80, + 0xa5, 0x33, 0xdb, 0x5c, 0x71, 0x66, 0x7f, 0x7b, 0x03, 0xa0, 0x84, 0xa5, 0x8b, 0x08, 0xda, 0x02, + 0xb8, 0x0a, 0xf1, 0xef, 0x18, 0x3d, 0x89, 0x7e, 0xdb, 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xab, + 0x6f, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0xc4, 0xfa, 0x8d, 0x49, 0xdb, 0xd0, 0x50, 0x8f, 0xbf, 0x0d, + 0x06, 0x02, 0x65, 0xfb, 0xb1, 0x72, 0xf4, 0xd3, 0x87, 0x17, 0x9b, 0x3c, 0xbd, 0xd8, 0xe4, 0xe3, + 0x8b, 0x4d, 0x1e, 0x5f, 0x6d, 0xe3, 0xe9, 0xd5, 0x36, 0x9e, 0x5f, 0x6d, 0xe3, 0xe6, 0x9b, 0x37, + 0xde, 0xa8, 0xb7, 0xeb, 0xea, 0xef, 0xbb, 0x4f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x35, 0x02, + 0x10, 0x83, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { @@ -1426,6 +1470,11 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ErrCode != 0 { + i = encodeVarintTreechange(dAtA, i, uint64(m.ErrCode)) + i-- + dAtA[i] = 0x10 + } if len(m.Error) > 0 { i -= len(m.Error) copy(dAtA[i:], m.Error) @@ -1763,6 +1812,9 @@ func (m *TreeErrorResponse) Size() (n int) { if l > 0 { n += 1 + l + sovTreechange(uint64(l)) } + if m.ErrCode != 0 { + n += 1 + sovTreechange(uint64(m.ErrCode)) + } return n } @@ -3392,6 +3444,25 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ErrCode", wireType) + } + m.ErrCode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTreechange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ErrCode |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTreechange(dAtA[iNdEx:]) diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index c5c5a5cc..f5b192ba 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -4,10 +4,11 @@ import ( "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/util/slice" "sync" ) -type inMemoryTreeStorage struct { +type InMemoryTreeStorage struct { id string root *treechangeproto.RawTreeChangeWithId heads []string @@ -16,7 +17,7 @@ type inMemoryTreeStorage struct { sync.RWMutex } -func (t *inMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { +func (t *InMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { t.RLock() defer t.RUnlock() @@ -37,46 +38,46 @@ func NewInMemoryTreeStorage( } allChanges[root.Id] = root - return &inMemoryTreeStorage{ + return &InMemoryTreeStorage{ id: root.Id, root: root, - heads: heads, + heads: append([]string(nil), heads...), changes: allChanges, RWMutex: sync.RWMutex{}, }, nil } -func (t *inMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { +func (t *InMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { _, exists := t.changes[id] return exists, nil } -func (t *inMemoryTreeStorage) Id() string { +func (t *InMemoryTreeStorage) Id() string { t.RLock() defer t.RUnlock() return t.id } -func (t *inMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() return t.root, nil } -func (t *inMemoryTreeStorage) Heads() ([]string, error) { +func (t *InMemoryTreeStorage) Heads() ([]string, error) { t.RLock() defer t.RUnlock() return t.heads, nil } -func (t *inMemoryTreeStorage) SetHeads(heads []string) error { +func (t *InMemoryTreeStorage) SetHeads(heads []string) error { t.Lock() defer t.Unlock() t.heads = append(t.heads[:0], heads...) return nil } -func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { +func (t *InMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { t.Lock() defer t.Unlock() // TODO: better to do deep copy @@ -84,7 +85,7 @@ func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChange return nil } -func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() if res, exists := t.changes[changeId]; exists { @@ -93,6 +94,33 @@ func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) return nil, fmt.Errorf("could not get change with id: %s", changeId) } -func (t *inMemoryTreeStorage) Delete() error { +func (t *InMemoryTreeStorage) Delete() error { return nil } + +func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { + var changes []*treechangeproto.RawTreeChangeWithId + for _, ch := range t.changes { + changes = append(changes, ch) + } + other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) + return other.(*InMemoryTreeStorage) +} + +func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool { + if !slice.UnsortedEquals(t.heads, other.heads) { + return false + } + if len(t.changes) != len(other.changes) { + return false + } + for k, v := range t.changes { + if otherV, exists := other.changes[k]; exists { + if otherV.Id == v.Id { + continue + } + } + return false + } + return true +} diff --git a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go b/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go deleted file mode 100644 index 6aee2208..00000000 --- a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/treegetter (interfaces: TreeGetter) - -// Package mock_treegetter is a generated GoMock package. -package mock_treegetter - -import ( - context "context" - reflect "reflect" - - app "github.com/anytypeio/any-sync/app" - objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - gomock "github.com/golang/mock/gomock" -) - -// MockTreeGetter is a mock of TreeGetter interface. -type MockTreeGetter struct { - ctrl *gomock.Controller - recorder *MockTreeGetterMockRecorder -} - -// MockTreeGetterMockRecorder is the mock recorder for MockTreeGetter. -type MockTreeGetterMockRecorder struct { - mock *MockTreeGetter -} - -// NewMockTreeGetter creates a new mock instance. -func NewMockTreeGetter(ctrl *gomock.Controller) *MockTreeGetter { - mock := &MockTreeGetter{ctrl: ctrl} - mock.recorder = &MockTreeGetterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTreeGetter) EXPECT() *MockTreeGetterMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockTreeGetter) Close(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockTreeGetterMockRecorder) Close(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeGetter)(nil).Close), arg0) -} - -// DeleteSpace mocks base method. -func (m *MockTreeGetter) DeleteSpace(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockTreeGetterMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeGetter)(nil).DeleteSpace), arg0, arg1) -} - -// DeleteTree mocks base method. -func (m *MockTreeGetter) DeleteTree(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteTree indicates an expected call of DeleteTree. -func (mr *MockTreeGetterMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeGetter)(nil).DeleteTree), arg0, arg1, arg2) -} - -// GetTree mocks base method. -func (m *MockTreeGetter) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) - ret0, _ := ret[0].(objecttree.ObjectTree) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTree indicates an expected call of GetTree. -func (mr *MockTreeGetterMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeGetter)(nil).GetTree), arg0, arg1, arg2) -} - -// Init mocks base method. -func (m *MockTreeGetter) Init(arg0 *app.App) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Init indicates an expected call of Init. -func (mr *MockTreeGetterMockRecorder) Init(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeGetter)(nil).Init), arg0) -} - -// Name mocks base method. -func (m *MockTreeGetter) Name() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Name") - ret0, _ := ret[0].(string) - return ret0 -} - -// Name indicates an expected call of Name. -func (mr *MockTreeGetterMockRecorder) Name() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeGetter)(nil).Name)) -} - -// Run mocks base method. -func (m *MockTreeGetter) Run(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Run indicates an expected call of Run. -func (mr *MockTreeGetterMockRecorder) Run(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeGetter)(nil).Run), arg0) -} diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go new file mode 100644 index 00000000..68f1d579 --- /dev/null +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -0,0 +1,122 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/object/treemanager (interfaces: TreeManager) + +// Package mock_treemanager is a generated GoMock package. +package mock_treemanager + +import ( + context "context" + reflect "reflect" + + app "github.com/anytypeio/any-sync/app" + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + gomock "github.com/golang/mock/gomock" +) + +// MockTreeManager is a mock of TreeManager interface. +type MockTreeManager struct { + ctrl *gomock.Controller + recorder *MockTreeManagerMockRecorder +} + +// MockTreeManagerMockRecorder is the mock recorder for MockTreeManager. +type MockTreeManagerMockRecorder struct { + mock *MockTreeManager +} + +// NewMockTreeManager creates a new mock instance. +func NewMockTreeManager(ctrl *gomock.Controller) *MockTreeManager { + mock := &MockTreeManager{ctrl: ctrl} + mock.recorder = &MockTreeManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTreeManager) EXPECT() *MockTreeManagerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockTreeManager) Close(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockTreeManagerMockRecorder) Close(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeManager)(nil).Close), arg0) +} + +// DeleteTree mocks base method. +func (m *MockTreeManager) DeleteTree(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteTree indicates an expected call of DeleteTree. +func (mr *MockTreeManagerMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeManager)(nil).DeleteTree), arg0, arg1, arg2) +} + +// GetTree mocks base method. +func (m *MockTreeManager) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) + ret0, _ := ret[0].(objecttree.ObjectTree) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTree indicates an expected call of GetTree. +func (mr *MockTreeManagerMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeManager)(nil).GetTree), arg0, arg1, arg2) +} + +// Init mocks base method. +func (m *MockTreeManager) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockTreeManagerMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeManager)(nil).Init), arg0) +} + +// Name mocks base method. +func (m *MockTreeManager) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockTreeManagerMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeManager)(nil).Name)) +} + +// Run mocks base method. +func (m *MockTreeManager) Run(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockTreeManagerMockRecorder) Run(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeManager)(nil).Run), arg0) +} diff --git a/commonspace/object/treegetter/treegetter.go b/commonspace/object/treemanager/treemanager.go similarity index 51% rename from commonspace/object/treegetter/treegetter.go rename to commonspace/object/treemanager/treemanager.go index 4e604d10..0573471e 100644 --- a/commonspace/object/treegetter/treegetter.go +++ b/commonspace/object/treemanager/treemanager.go @@ -1,5 +1,5 @@ -//go:generate mockgen -destination mock_treegetter/mock_treegetter.go github.com/anytypeio/any-sync/commonspace/object/treegetter TreeGetter -package treegetter +//go:generate mockgen -destination mock_treemanager/mock_treemanager.go github.com/anytypeio/any-sync/commonspace/object/treemanager TreeManager +package treemanager import ( "context" @@ -7,11 +7,10 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" ) -const CName = "common.object.treegetter" +const CName = "common.object.treemanager" -type TreeGetter interface { +type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) DeleteTree(ctx context.Context, spaceId, treeId string) error - DeleteSpace(ctx context.Context, spaceId string) error } diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go new file mode 100644 index 00000000..29ea2717 --- /dev/null +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -0,0 +1,154 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/objectsync (interfaces: SyncClient) + +// Package mock_objectsync is a generated GoMock package. +package mock_objectsync + +import ( + context "context" + reflect "reflect" + + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + objectsync "github.com/anytypeio/any-sync/commonspace/objectsync" + spacesyncproto "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + gomock "github.com/golang/mock/gomock" +) + +// MockSyncClient is a mock of SyncClient interface. +type MockSyncClient struct { + ctrl *gomock.Controller + recorder *MockSyncClientMockRecorder +} + +// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. +type MockSyncClientMockRecorder struct { + mock *MockSyncClient +} + +// NewMockSyncClient creates a new mock instance. +func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { + mock := &MockSyncClient{ctrl: ctrl} + mock.recorder = &MockSyncClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { + return m.recorder +} + +// Broadcast mocks base method. +func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Broadcast indicates an expected call of Broadcast. +func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) +} + +// CreateFullSyncRequest mocks base method. +func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. +func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) +} + +// CreateFullSyncResponse mocks base method. +func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. +func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) +} + +// CreateHeadUpdate mocks base method. +func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. +func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) +} + +// CreateNewTreeRequest mocks base method. +func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNewTreeRequest") + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. +func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) +} + +// MessagePool mocks base method. +func (m *MockSyncClient) MessagePool() objectsync.MessagePool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MessagePool") + ret0, _ := ret[0].(objectsync.MessagePool) + return ret0 +} + +// MessagePool indicates an expected call of MessagePool. +func (mr *MockSyncClientMockRecorder) MessagePool() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MessagePool", reflect.TypeOf((*MockSyncClient)(nil).MessagePool)) +} + +// SendSync mocks base method. +func (m *MockSyncClient) SendSync(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage) (*spacesyncproto.ObjectSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendSync", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*spacesyncproto.ObjectSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendSync indicates an expected call of SendSync. +func (mr *MockSyncClientMockRecorder) SendSync(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSync", reflect.TypeOf((*MockSyncClient)(nil).SendSync), arg0, arg1, arg2, arg3) +} + +// SendWithReply mocks base method. +func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage, arg4 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendWithReply indicates an expected call of SendWithReply. +func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3, arg4) +} diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 06127d0c..e22db4bd 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -1,7 +1,10 @@ +//go:generate mockgen -destination mock_objectsync/mock_objectsync.go github.com/anytypeio/any-sync/commonspace/objectsync SyncClient package objectsync import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/gogo/protobuf/proto" "sync/atomic" "time" @@ -21,9 +24,8 @@ var log = logger.NewNamed("common.commonspace.objectsync") type ObjectSync interface { LastUsage synchandler.SyncHandler - MessagePool() MessagePool + SyncClient() SyncClient - Init() Close() (err error) } @@ -31,6 +33,7 @@ type objectSync struct { spaceId string messagePool MessagePool + syncClient SyncClient objectGetter syncobjectgetter.SyncObjectGetter configuration nodeconf.NodeConf spaceStorage spacestorage.SpaceStorage @@ -48,40 +51,18 @@ func NewObjectSync( objectGetter syncobjectgetter.SyncObjectGetter, storage spacestorage.SpaceStorage) ObjectSync { syncCtx, cancel := context.WithCancel(context.Background()) - os := newObjectSync( - spaceId, - spaceIsDeleted, - configuration, - objectGetter, - storage, - syncCtx, - cancel) - msgPool := newMessagePool(peerManager, os.handleMessage) - os.messagePool = msgPool - return os -} - -func newObjectSync( - spaceId string, - spaceIsDeleted *atomic.Bool, - configuration nodeconf.NodeConf, - objectGetter syncobjectgetter.SyncObjectGetter, - spaceStorage spacestorage.SpaceStorage, - syncCtx context.Context, - cancel context.CancelFunc, -) *objectSync { - return &objectSync{ + os := &objectSync{ objectGetter: objectGetter, - spaceStorage: spaceStorage, + spaceStorage: storage, spaceId: spaceId, syncCtx: syncCtx, cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, } -} - -func (s *objectSync) Init() { + os.messagePool = newMessagePool(peerManager, os.handleMessage) + os.syncClient = NewSyncClient(spaceId, os.messagePool, NewRequestFactory()) + return os } func (s *objectSync) Close() (err error) { @@ -98,22 +79,60 @@ func (s *objectSync) HandleMessage(ctx context.Context, senderId string, message } func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)) + log := log.With( + zap.String("objectId", msg.ObjectId), + zap.String("requestId", msg.RequestId), + zap.String("replyId", msg.ReplyId)) if s.spaceIsDeleted.Load() { log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() { - return spacesyncproto.ErrSpaceIsDeleted + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId, msg.ObjectId) + } + } + log.DebugCtx(ctx, "handling message") + hasTree, err := s.spaceStorage.HasTree(msg.ObjectId) + if err != nil { + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId) + } + // in this case we will try to get it from remote, unless the sender also sent us the same request :-) + if !hasTree { + treeMsg := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, treeMsg) + if err != nil { + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.RequestId) + } + // this means that we don't have the tree locally and therefore can't return it + if s.isEmptyFullSyncRequest(treeMsg) { + return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.RequestId) } } - log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { - return + log.DebugCtx(ctx, "failed to get object") + return s.unmarshallSendError(ctx, msg, err, msg.ObjectId, senderId) } return obj.HandleMessage(ctx, senderId, msg) } -func (s *objectSync) MessagePool() MessagePool { - return s.messagePool +func (s *objectSync) SyncClient() SyncClient { + return s.syncClient +} + +func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId, objectId string) (err error) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, unmarshalled) + if err != nil { + return + } + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.RequestId) +} + +func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, objectId, replyId string) (err error) { + resp := treechangeproto.WrapError(respErr, root) + return s.syncClient.SendWithReply(ctx, senderId, objectId, resp, replyId) +} + +func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { + return msg.GetContent().GetFullSyncRequest() != nil && len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 } diff --git a/commonspace/object/tree/synctree/requestfactory.go b/commonspace/objectsync/requestfactory.go similarity index 95% rename from commonspace/object/tree/synctree/requestfactory.go rename to commonspace/objectsync/requestfactory.go index 6fc5202c..4d6a9124 100644 --- a/commonspace/object/tree/synctree/requestfactory.go +++ b/commonspace/objectsync/requestfactory.go @@ -1,4 +1,4 @@ -package synctree +package objectsync import ( "fmt" @@ -14,10 +14,8 @@ type RequestFactory interface { CreateFullSyncResponse(t objecttree.ObjectTree, theirHeads, theirSnapshotPath []string) (*treechangeproto.TreeSyncMessage, error) } -var sharedFactory = &requestFactory{} - -func GetRequestFactory() RequestFactory { - return sharedFactory +func NewRequestFactory() RequestFactory { + return &requestFactory{} } type requestFactory struct{} diff --git a/commonspace/objectsync/syncclient.go b/commonspace/objectsync/syncclient.go new file mode 100644 index 00000000..e6fd515e --- /dev/null +++ b/commonspace/objectsync/syncclient.go @@ -0,0 +1,74 @@ +package objectsync + +import ( + "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" +) + +type SyncClient interface { + RequestFactory + Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) + SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) + SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) + MessagePool() MessagePool +} + +type syncClient struct { + RequestFactory + spaceId string + messagePool MessagePool +} + +func NewSyncClient( + spaceId string, + messagePool MessagePool, + factory RequestFactory) SyncClient { + return &syncClient{ + messagePool: messagePool, + RequestFactory: factory, + spaceId: spaceId, + } +} + +func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") + if err != nil { + return + } + return s.messagePool.Broadcast(ctx, objMsg) +} + +func (s *syncClient) SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, "") + if err != nil { + return + } + return s.messagePool.SendSync(ctx, peerId, objMsg) +} + +func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, replyId) + if err != nil { + return + } + return s.messagePool.SendPeer(ctx, peerId, objMsg) +} + +func (s *syncClient) MessagePool() MessagePool { + return s.messagePool +} + +func MarshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { + payload, err := message.Marshal() + if err != nil { + return + } + objMsg = &spacesyncproto.ObjectSyncMessage{ + ReplyId: replyId, + Payload: payload, + ObjectId: objectId, + SpaceId: spaceId, + } + return +} diff --git a/commonspace/settings/deleter.go b/commonspace/settings/deleter.go index 55f026c1..c50da268 100644 --- a/commonspace/settings/deleter.go +++ b/commonspace/settings/deleter.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "go.uber.org/zap" @@ -15,10 +15,10 @@ type Deleter interface { type deleter struct { st spacestorage.SpaceStorage state settingsstate.ObjectDeletionState - getter treegetter.TreeGetter + getter treemanager.TreeManager } -func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treegetter.TreeGetter) Deleter { +func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treemanager.TreeManager) Deleter { return &deleter{st, state, getter} } diff --git a/commonspace/settings/deleter_test.go b/commonspace/settings/deleter_test.go index 3606225b..fe6f7c51 100644 --- a/commonspace/settings/deleter_test.go +++ b/commonspace/settings/deleter_test.go @@ -2,7 +2,7 @@ package settings import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -12,18 +12,18 @@ import ( func TestDeleter_Delete(t *testing.T) { ctrl := gomock.NewController(t) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - deleter := newDeleter(st, delState, treeGetter) + deleter := newDeleter(st, delState, treeManager) t.Run("deleter delete queued", func(t *testing.T) { id := "id" spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -34,7 +34,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -45,7 +45,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) deleter.Delete() }) diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 8bfc15e3..1e357cdf 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/util/slice" "go.uber.org/zap" @@ -20,12 +20,12 @@ func newDeletionManager( spaceId string, settingsId string, isResponsible bool, - treeGetter treegetter.TreeGetter, + treeManager treemanager.TreeManager, deletionState settingsstate.ObjectDeletionState, provider SpaceIdsProvider, onSpaceDelete func()) DeletionManager { return &deletionManager{ - treeGetter: treeGetter, + treeManager: treeManager, isResponsible: isResponsible, spaceId: spaceId, settingsId: settingsId, @@ -38,7 +38,7 @@ func newDeletionManager( type deletionManager struct { deletionState settingsstate.ObjectDeletionState provider SpaceIdsProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager spaceId string settingsId string isResponsible bool @@ -55,10 +55,6 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate. return nil } log.Debug("deleting space") - err = d.treeGetter.DeleteSpace(ctx, d.spaceId) - if err != nil { - log.Debug("failed to notify on space deletion", zap.Error(err)) - } if d.isResponsible { allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool { return id == d.settingsId diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index a4992903..31aff516 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -27,15 +27,14 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) delManager := newDeletionManager(spaceId, settingsId, false, - treeGetter, + treeManager, delState, nil, onDeleted) @@ -60,17 +59,16 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) provider := mock_settings.NewMockSpaceIdsProvider(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) delManager := newDeletionManager(spaceId, settingsId, true, - treeGetter, + treeManager, delState, provider, onDeleted) diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index 8ad55eac..ab327cdf 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -13,7 +13,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" @@ -47,7 +47,7 @@ type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener. type Deps struct { BuildFunc BuildTreeFunc Account accountservice.Service - TreeGetter treegetter.TreeGetter + TreeManager treemanager.TreeManager Store spacestorage.SpaceStorage Configuration nodeconf.NodeConf DeletionState settingsstate.ObjectDeletionState @@ -62,13 +62,13 @@ type Deps struct { type settingsObject struct { synctree.SyncTree - account accountservice.Service - spaceId string - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - builder settingsstate.StateBuilder - buildFunc BuildTreeFunc - loop *deleteLoop + account accountservice.Service + spaceId string + treeManager treemanager.TreeManager + store spacestorage.SpaceStorage + builder settingsstate.StateBuilder + buildFunc BuildTreeFunc + loop *deleteLoop state *settingsstate.State deletionState settingsstate.ObjectDeletionState @@ -84,7 +84,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { changeFactory settingsstate.ChangeFactory ) if deps.del == nil { - deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) + deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeManager) } else { deleter = deps.del } @@ -93,7 +93,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId, deps.Store.SpaceSettingsId(), deps.Configuration.IsResponsible(spaceId), - deps.TreeGetter, + deps.TreeManager, deps.DeletionState, deps.Provider, deps.OnSpaceDelete) @@ -123,7 +123,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId: spaceId, account: deps.Account, deletionState: deps.DeletionState, - treeGetter: deps.TreeGetter, + treeManager: deps.TreeManager, store: deps.Store, buildFunc: deps.BuildFunc, builder: builder, diff --git a/commonspace/settings/settings_test.go b/commonspace/settings/settings_test.go index 166cf284..b4879f53 100644 --- a/commonspace/settings/settings_test.go +++ b/commonspace/settings/settings_test.go @@ -9,7 +9,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -45,7 +45,7 @@ type settingsFixture struct { docId string doc *settingsObject ctrl *gomock.Controller - treeGetter *mock_treegetter.MockTreeGetter + treeManager *mock_treemanager.MockTreeManager spaceStorage *mock_spacestorage.MockSpaceStorage stateBuilder *mock_settingsstate.MockStateBuilder deletionManager *mock_settings.MockDeletionManager @@ -62,7 +62,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { ctrl := gomock.NewController(t) acc := mock_accountservice.NewMockService(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) delManager := mock_settings.NewMockDeletionManager(ctrl) @@ -81,7 +81,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { deps := Deps{ BuildFunc: buildFunc, Account: acc, - TreeGetter: treeGetter, + TreeManager: treeManager, Store: st, DeletionState: delState, delManager: delManager, @@ -95,7 +95,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { docId: objectId, doc: doc, ctrl: ctrl, - treeGetter: treeGetter, + treeManager: treeManager, spaceStorage: st, stateBuilder: stateBuilder, changeFactory: changeFactory, diff --git a/commonspace/space.go b/commonspace/space.go index cdb00e18..10090b95 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -118,12 +118,13 @@ type space struct { headSync headsync.HeadSync syncStatus syncstatus.StatusUpdater storage spacestorage.SpaceStorage - cache *commonGetter + treeManager *commonGetter account accountservice.Service aclList *syncacl.SyncAcl configuration nodeconf.NodeConf settingsObject settings.SettingsObject peerManager peermanager.PeerManager + treeBuilder objecttree.BuildObjectTreeFunc handleQueue multiqueue.MultiQueue[HandleMessage] @@ -178,8 +179,8 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.MessagePool()) - s.cache.AddObject(s.aclList) + s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.SyncClient().MessagePool()) + s.treeManager.AddObject(s.aclList) deletionState := settingsstate.NewObjectDeletionState(s.storage) deps := settings.Deps{ @@ -196,7 +197,7 @@ func (s *space) Init(ctx context.Context) (err error) { return }, Account: s.account, - TreeGetter: s.cache, + TreeManager: s.treeManager, Store: s.storage, DeletionState: deletionState, Provider: s.headSync, @@ -204,13 +205,12 @@ func (s *space) Init(ctx context.Context) (err error) { OnSpaceDelete: s.onSpaceDelete, } s.settingsObject = settings.NewSettingsObject(deps, s.id) - s.objectSync.Init() s.headSync.Init(initialIds, deletionState) err = s.settingsObject.Init(ctx) if err != nil { return } - s.cache.AddObject(s.settingsObject) + s.treeManager.AddObject(s.settingsObject) s.syncStatus.Run() s.handleQueue = multiqueue.New[HandleMessage](s.handleMessage, 100) return nil @@ -283,16 +283,17 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea return } deps := synctree.BuildDeps{ - SpaceId: s.id, - ObjectSync: s.objectSync, - Configuration: s.configuration, - HeadNotifiable: s.headSync, - Listener: listener, - AclList: s.aclList, - SpaceStorage: s.storage, - OnClose: s.onObjectClose, - SyncStatus: s.syncStatus, - PeerGetter: s.peerManager, + SpaceId: s.id, + SyncClient: s.objectSync.SyncClient(), + Configuration: s.configuration, + HeadNotifiable: s.headSync, + Listener: listener, + AclList: s.aclList, + SpaceStorage: s.storage, + OnClose: s.onObjectClose, + SyncStatus: s.syncStatus, + PeerGetter: s.peerManager, + BuildObjectTree: s.treeBuilder, } t, err = synctree.PutSyncTree(ctx, payload, deps) if err != nil { @@ -321,7 +322,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t deps := synctree.BuildDeps{ SpaceId: s.id, - ObjectSync: s.objectSync, + SyncClient: s.objectSync.SyncClient(), Configuration: s.configuration, HeadNotifiable: s.headSync, Listener: opts.Listener, @@ -331,6 +332,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t SyncStatus: s.syncStatus, WaitTreeRemoteSync: opts.WaitTreeRemoteSync, PeerGetter: s.peerManager, + BuildObjectTree: s.treeBuilder, } if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { return nil, err diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index 0d2168c7..addf1f7f 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -8,8 +8,9 @@ import ( "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -49,7 +50,7 @@ type spaceService struct { storageProvider spacestorage.SpaceStorageProvider peermanagerProvider peermanager.PeerManagerProvider credentialProvider credentialprovider.CredentialProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager pool pool.Pool } @@ -58,7 +59,7 @@ func (s *spaceService) Init(a *app.App) (err error) { s.account = a.MustComponent(accountservice.CName).(accountservice.Service) s.storageProvider = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorageProvider) s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service) - s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter) + s.treeManager = a.MustComponent(treemanager.CName).(treemanager.TreeManager) s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider) credProvider := a.Component(credentialprovider.CName) if credProvider != nil { @@ -145,13 +146,19 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { return nil, err } spaceIsDeleted.Swap(isDeleted) - getter := newCommonGetter(st.Id(), s.treeGetter, spaceIsClosed) + getter := newCommonGetter(st.Id(), s.treeManager, spaceIsClosed) syncStatus := syncstatus.NewNoOpSyncStatus() // this will work only for clients, not the best solution, but... if !lastConfiguration.IsResponsible(st.Id()) { // TODO: move it to the client package and add possibility to inject StatusProvider from the client syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } + var builder objecttree.BuildObjectTreeFunc + if s.config.KeepTreeDataInMemory { + builder = objecttree.BuildObjectTree + } else { + builder = objecttree.BuildEmptyDataObjectTree + } peerManager, err := s.peermanagerProvider.NewPeerManager(ctx, id) if err != nil { @@ -165,12 +172,13 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { objectSync: objectSync, headSync: headSync, syncStatus: syncStatus, - cache: getter, + treeManager: getter, account: s.account, configuration: lastConfiguration, peerManager: peerManager, storage: st, treesUsed: &atomic.Int32{}, + treeBuilder: builder, isClosed: spaceIsClosed, isDeleted: spaceIsDeleted, } diff --git a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go index 82a14be2..93baa877 100644 --- a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go +++ b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go @@ -81,6 +81,21 @@ func (mr *MockSpaceStorageMockRecorder) CreateTreeStorage(arg0 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTreeStorage", reflect.TypeOf((*MockSpaceStorage)(nil).CreateTreeStorage), arg0) } +// HasTree mocks base method. +func (m *MockSpaceStorage) HasTree(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasTree", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HasTree indicates an expected call of HasTree. +func (mr *MockSpaceStorageMockRecorder) HasTree(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasTree", reflect.TypeOf((*MockSpaceStorage)(nil).HasTree), arg0) +} + // Id mocks base method. func (m *MockSpaceStorage) Id() string { m.ctrl.T.Helper() diff --git a/commonspace/spacestorage/spacestorage.go b/commonspace/spacestorage/spacestorage.go index ebd4b68d..3f21c22e 100644 --- a/commonspace/spacestorage/spacestorage.go +++ b/commonspace/spacestorage/spacestorage.go @@ -47,6 +47,7 @@ type SpaceStorage interface { StoredIds() ([]string, error) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error) TreeStorage(id string) (treestorage.TreeStorage, error) + HasTree(id string) (bool, error) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) WriteSpaceHash(hash string) error ReadSpaceHash() (hash string, err error) diff --git a/net/rpc/rpcerr/registry.go b/net/rpc/rpcerr/registry.go index 5160fec3..e8a749e6 100644 --- a/net/rpc/rpcerr/registry.go +++ b/net/rpc/rpcerr/registry.go @@ -24,6 +24,10 @@ func RegisterErr(err error, code uint64) error { return errWithCode } +func Code(err error) uint64 { + return drpcerr.Code(err) +} + func Err(code uint64) error { err, ok := errsMap[code] if !ok {