diff --git a/commonspace/deletion_test.go b/commonspace/deletion_test.go new file mode 100644 index 00000000..27ff2b0f --- /dev/null +++ b/commonspace/deletion_test.go @@ -0,0 +1,271 @@ +package commonspace + +import ( + "context" + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/settings" + "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" + "github.com/anytypeio/any-sync/commonspace/spacestorage" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/util/crypto" + "github.com/stretchr/testify/require" + "math/rand" + "testing" + "time" +) + +func addIncorrectSnapshot(settingsObject settings.SettingsObject, acc *accountdata.AccountKeys, partialIds map[string]struct{}, newId string) (err error) { + factory := settingsstate.NewChangeFactory() + bytes, err := factory.CreateObjectDeleteChange(newId, &settingsstate.State{DeletedIds: partialIds}, true) + if err != nil { + return + } + ch, err := settingsObject.PrepareChange(objecttree.SignableChangeContent{ + Data: bytes, + Key: acc.SignKey, + IsSnapshot: true, + IsEncrypted: false, + Timestamp: time.Now().Unix(), + }) + if err != nil { + return + } + res, err := settingsObject.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: []string{ch.Id}, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ch}, + }) + if err != nil { + return + } + if res.Mode != objecttree.Rebuild { + return fmt.Errorf("incorrect mode: %d", res.Mode) + } + return +} + +func TestSpaceDeleteIds(t *testing.T) { + fx := newFixture(t) + acc := fx.account.Account() + rk := crypto.NewAES() + ctx := context.Background() + totalObjs := 1500 + + // creating space + sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{ + SigningKey: acc.SignKey, + SpaceType: "type", + ReadKey: rk.Bytes(), + ReplicationKey: 10, + MasterKey: acc.PeerKey, + }) + require.NoError(t, err) + require.NotNil(t, sp) + + // initializing space + spc, err := fx.spaceService.NewSpace(ctx, sp) + require.NoError(t, err) + require.NotNil(t, spc) + // adding space to tree manager + fx.treeManager.space = spc + err = spc.Init(ctx) + require.NoError(t, err) + + var ids []string + for i := 0; i < totalObjs; i++ { + // creating a tree + bytes := make([]byte, 32) + rand.Read(bytes) + doc, err := spc.CreateTree(ctx, objecttree.ObjectTreeCreatePayload{ + PrivKey: acc.SignKey, + ChangeType: "some", + SpaceId: spc.Id(), + IsEncrypted: false, + Seed: bytes, + Timestamp: time.Now().Unix(), + }) + require.NoError(t, err) + tr, err := spc.PutTree(ctx, doc, nil) + require.NoError(t, err) + ids = append(ids, tr.Id()) + tr.Close() + } + // deleting trees + for _, id := range ids { + err = spc.DeleteTree(ctx, id) + require.NoError(t, err) + } + time.Sleep(3 * time.Second) + spc.Close() + require.Equal(t, len(ids), len(fx.treeManager.deletedIds)) +} + +func createTree(t *testing.T, ctx context.Context, spc Space, acc *accountdata.AccountKeys) string { + bytes := make([]byte, 32) + rand.Read(bytes) + doc, err := spc.CreateTree(ctx, objecttree.ObjectTreeCreatePayload{ + PrivKey: acc.SignKey, + ChangeType: "some", + SpaceId: spc.Id(), + IsEncrypted: false, + Seed: bytes, + Timestamp: time.Now().Unix(), + }) + require.NoError(t, err) + tr, err := spc.PutTree(ctx, doc, nil) + require.NoError(t, err) + tr.Close() + return tr.Id() +} + +func TestSpaceDeleteIdsIncorrectSnapshot(t *testing.T) { + fx := newFixture(t) + acc := fx.account.Account() + rk := crypto.NewAES() + ctx := context.Background() + totalObjs := 1500 + partialObjs := 300 + + // creating space + sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{ + SigningKey: acc.SignKey, + SpaceType: "type", + ReadKey: rk.Bytes(), + ReplicationKey: 10, + MasterKey: acc.PeerKey, + }) + require.NoError(t, err) + require.NotNil(t, sp) + + // initializing space + spc, err := fx.spaceService.NewSpace(ctx, sp) + require.NoError(t, err) + require.NotNil(t, spc) + // adding space to tree manager + fx.treeManager.space = spc + err = spc.Init(ctx) + require.NoError(t, err) + + settingsObject := spc.(*space).settingsObject + var ids []string + for i := 0; i < totalObjs; i++ { + id := createTree(t, ctx, spc, acc) + ids = append(ids, id) + } + // copying storage, so we will have all the trees locally + inmemory := spc.Storage().(*commonStorage).SpaceStorage.(*spacestorage.InMemorySpaceStorage) + storageCopy := inmemory.CopyStorage() + treesCopy := inmemory.AllTrees() + + // deleting trees + for _, id := range ids { + err = spc.DeleteTree(ctx, id) + require.NoError(t, err) + } + mapIds := map[string]struct{}{} + for _, id := range ids[:partialObjs] { + mapIds[id] = struct{}{} + } + // adding snapshot that breaks the state + err = addIncorrectSnapshot(settingsObject, acc, mapIds, ids[partialObjs]) + require.NoError(t, err) + // copying the contents of the settings tree + treesCopy[settingsObject.Id()] = settingsObject.Storage() + storageCopy.SetTrees(treesCopy) + spc.Close() + time.Sleep(100 * time.Millisecond) + // now we replace the storage, so the trees are back, but the settings object says that they are deleted + fx.storageProvider.(*spacestorage.InMemorySpaceStorageProvider).SetStorage(storageCopy) + + spc, err = fx.spaceService.NewSpace(ctx, sp) + require.NoError(t, err) + require.NotNil(t, spc) + fx.treeManager.space = spc + fx.treeManager.deletedIds = nil + err = spc.Init(ctx) + require.NoError(t, err) + + // waiting until everything is deleted + time.Sleep(3 * time.Second) + require.Equal(t, len(ids), len(fx.treeManager.deletedIds)) + + // checking that new snapshot will contain all the changes + settingsObject = spc.(*space).settingsObject + settings.DoSnapshot = func(treeLen int) bool { + return true + } + id := createTree(t, ctx, spc, acc) + err = spc.DeleteTree(ctx, id) + require.NoError(t, err) + delIds := settingsObject.Root().Model.(*spacesyncproto.SettingsData).Snapshot.DeletedIds + require.Equal(t, totalObjs+1, len(delIds)) +} + +func TestSpaceDeleteIdsMarkDeleted(t *testing.T) { + fx := newFixture(t) + acc := fx.account.Account() + rk := crypto.NewAES() + ctx := context.Background() + totalObjs := 1500 + + // creating space + sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{ + SigningKey: acc.SignKey, + SpaceType: "type", + ReadKey: rk.Bytes(), + ReplicationKey: 10, + MasterKey: acc.PeerKey, + }) + require.NoError(t, err) + require.NotNil(t, sp) + + // initializing space + spc, err := fx.spaceService.NewSpace(ctx, sp) + require.NoError(t, err) + require.NotNil(t, spc) + // adding space to tree manager + fx.treeManager.space = spc + err = spc.Init(ctx) + require.NoError(t, err) + + settingsObject := spc.(*space).settingsObject + var ids []string + for i := 0; i < totalObjs; i++ { + id := createTree(t, ctx, spc, acc) + ids = append(ids, id) + } + // copying storage, so we will have the same contents, except for empty trees + inmemory := spc.Storage().(*commonStorage).SpaceStorage.(*spacestorage.InMemorySpaceStorage) + storageCopy := inmemory.CopyStorage() + + // deleting trees, this will prepare the document to have all the deletion changes + for _, id := range ids { + err = spc.DeleteTree(ctx, id) + require.NoError(t, err) + } + treesMap := map[string]treestorage.TreeStorage{} + // copying the contents of the settings tree + treesMap[settingsObject.Id()] = settingsObject.Storage() + storageCopy.SetTrees(treesMap) + spc.Close() + time.Sleep(100 * time.Millisecond) + // now we replace the storage, so the trees are back, but the settings object says that they are deleted + fx.storageProvider.(*spacestorage.InMemorySpaceStorageProvider).SetStorage(storageCopy) + + spc, err = fx.spaceService.NewSpace(ctx, sp) + require.NoError(t, err) + require.NotNil(t, spc) + fx.treeManager.space = spc + fx.treeManager.deletedIds = nil + fx.treeManager.markedIds = nil + err = spc.Init(ctx) + require.NoError(t, err) + + // waiting until everything is deleted + time.Sleep(3 * time.Second) + require.Equal(t, len(ids), len(fx.treeManager.markedIds)) + require.Zero(t, len(fx.treeManager.deletedIds)) +} diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go index 68f1d579..8c0f12a9 100644 --- a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -93,6 +93,20 @@ func (mr *MockTreeManagerMockRecorder) Init(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeManager)(nil).Init), arg0) } +// MarkTreeDeleted mocks base method. +func (m *MockTreeManager) MarkTreeDeleted(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkTreeDeleted", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkTreeDeleted indicates an expected call of MarkTreeDeleted. +func (mr *MockTreeManagerMockRecorder) MarkTreeDeleted(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkTreeDeleted", reflect.TypeOf((*MockTreeManager)(nil).MarkTreeDeleted), arg0, arg1, arg2) +} + // Name mocks base method. func (m *MockTreeManager) Name() string { m.ctrl.T.Helper() diff --git a/commonspace/object/treemanager/treemanager.go b/commonspace/object/treemanager/treemanager.go index 0573471e..0cc5513c 100644 --- a/commonspace/object/treemanager/treemanager.go +++ b/commonspace/object/treemanager/treemanager.go @@ -12,5 +12,6 @@ const CName = "common.object.treemanager" type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) + MarkTreeDeleted(ctx context.Context, spaceId, treeId string) error DeleteTree(ctx context.Context, spaceId, treeId string) error } diff --git a/commonspace/settings/deleter.go b/commonspace/settings/deleter.go index c50da268..6cc3c0b5 100644 --- a/commonspace/settings/deleter.go +++ b/commonspace/settings/deleter.go @@ -2,6 +2,7 @@ package settings import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -23,17 +24,40 @@ func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletion } func (d *deleter) Delete() { - allQueued := d.state.GetQueued() + var ( + allQueued = d.state.GetQueued() + spaceId = d.st.Id() + ) for _, id := range allQueued { - err := d.getter.DeleteTree(context.Background(), d.st.Id(), id) - if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted { - log.With(zap.String("id", id), zap.Error(err)).Error("failed to delete object") - continue + log := log.With(zap.String("treeId", id), zap.String("spaceId", spaceId)) + shouldDelete, err := d.tryMarkDeleted(spaceId, id) + if !shouldDelete { + if err != nil { + log.Error("failed to mark object as deleted", zap.Error(err)) + continue + } + } else { + err = d.getter.DeleteTree(context.Background(), spaceId, id) + if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted { + log.Error("failed to delete object", zap.Error(err)) + continue + } } err = d.state.Delete(id) if err != nil { - log.With(zap.String("id", id), zap.Error(err)).Error("failed to mark object as deleted") + log.Error("failed to mark object as deleted", zap.Error(err)) } - log.With(zap.String("id", id), zap.Error(err)).Debug("object successfully deleted") + log.Debug("object successfully deleted", zap.Error(err)) } } + +func (d *deleter) tryMarkDeleted(spaceId, treeId string) (bool, error) { + _, err := d.st.TreeStorage(treeId) + if err == nil { + return true, nil + } + if err != treestorage.ErrUnknownTreeId { + return false, err + } + return false, d.getter.MarkTreeDeleted(context.Background(), spaceId, treeId) +} diff --git a/commonspace/settings/deleter_test.go b/commonspace/settings/deleter_test.go index fe6f7c51..21719f93 100644 --- a/commonspace/settings/deleter_test.go +++ b/commonspace/settings/deleter_test.go @@ -2,9 +2,9 @@ package settings import ( "fmt" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" - "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" "github.com/golang/mock/gomock" "testing" @@ -18,23 +18,46 @@ func TestDeleter_Delete(t *testing.T) { deleter := newDeleter(st, delState, treeManager) - t.Run("deleter delete queued", func(t *testing.T) { + t.Run("deleter delete mark deleted success", func(t *testing.T) { id := "id" spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) + st.EXPECT().TreeStorage(id).Return(nil, treestorage.ErrUnknownTreeId) + treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(nil) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() }) - t.Run("deleter delete already deleted", func(t *testing.T) { + t.Run("deleter delete mark deleted other error", func(t *testing.T) { id := "id" spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) + st.EXPECT().TreeStorage(id).Return(nil, fmt.Errorf("unknown error")) + + deleter.Delete() + }) + + t.Run("deleter delete mark deleted fail", func(t *testing.T) { + id := "id" + spaceId := "spaceId" + delState.EXPECT().GetQueued().Return([]string{id}) + st.EXPECT().Id().Return(spaceId) + st.EXPECT().TreeStorage(id).Return(nil, treestorage.ErrUnknownTreeId) + treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(fmt.Errorf("mark error")) + + deleter.Delete() + }) + //treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) + t.Run("deleter delete success", func(t *testing.T) { + id := "id" + spaceId := "spaceId" + delState.EXPECT().GetQueued().Return([]string{id}) + st.EXPECT().Id().Return(spaceId) + st.EXPECT().TreeStorage(id).Return(nil, nil) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -45,6 +68,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) + st.EXPECT().TreeStorage(id).Return(nil, nil) treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) deleter.Delete() diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 1e357cdf..eb66873d 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -4,7 +4,6 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" - "github.com/anytypeio/any-sync/util/slice" "go.uber.org/zap" ) @@ -47,22 +46,20 @@ type deletionManager struct { func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate.State) error { log := log.With(zap.String("spaceId", d.spaceId)) - err := d.deletionState.Add(state.DeletedIds) - if err != nil { - log.Debug("failed to add deleted ids to deletion state") - } + d.deletionState.Add(state.DeletedIds) if state.DeleterId == "" { return nil } + // we should delete space log.Debug("deleting space") if d.isResponsible { - allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool { - return id == d.settingsId - }) - err := d.deletionState.Add(allIds) - if err != nil { - log.Debug("failed to add all ids to deletion state") + mapIds := map[string]struct{}{} + for _, id := range d.provider.AllIds() { + if id != d.settingsId { + mapIds[id] = struct{}{} + } } + d.deletionState.Add(mapIds) } d.onSpaceDelete() return nil diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index 31aff516..1f3afeae 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -19,7 +19,7 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { spaceId := "spaceId" settingsId := "settingsId" state := &settingsstate.State{ - DeletedIds: []string{"id"}, + DeletedIds: map[string]struct{}{"id": {}}, DeleterId: "deleterId", } deleted := false @@ -29,7 +29,7 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) treeManager := mock_treemanager.NewMockTreeManager(ctrl) - delState.EXPECT().Add(state.DeletedIds).Return(nil) + delState.EXPECT().Add(state.DeletedIds) delManager := newDeletionManager(spaceId, settingsId, @@ -51,7 +51,7 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { spaceId := "spaceId" settingsId := "settingsId" state := &settingsstate.State{ - DeletedIds: []string{"id"}, + DeletedIds: map[string]struct{}{"id": struct{}{}}, DeleterId: "deleterId", } deleted := false @@ -62,9 +62,9 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { treeManager := mock_treemanager.NewMockTreeManager(ctrl) provider := mock_settings.NewMockSpaceIdsProvider(ctrl) - delState.EXPECT().Add(state.DeletedIds).Return(nil) + delState.EXPECT().Add(state.DeletedIds) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) - delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) + delState.EXPECT().Add(map[string]struct{}{"id": {}, "otherId": {}}) delManager := newDeletionManager(spaceId, settingsId, true, diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index ab327cdf..c7539d4e 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -40,7 +40,16 @@ var ( ErrCantDeleteSpace = errors.New("not able to delete space") ) -var doSnapshot = objecttree.DoSnapshot +var ( + DoSnapshot = objecttree.DoSnapshot + buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) { + return objecttree.BuildHistoryTree(objecttree.HistoryTreeParams{ + TreeStorage: objTree.Storage(), + AclList: objTree.AclList(), + BuildFullTree: true, + }) + } +) type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) @@ -166,11 +175,36 @@ func (s *settingsObject) Init(ctx context.Context) (err error) { if err != nil { return } - + // TODO: remove this check when everybody updates + if err = s.checkHistoryState(ctx); err != nil { + return + } s.loop.Run() return } +func (s *settingsObject) checkHistoryState(ctx context.Context) (err error) { + historyTree, err := buildHistoryTree(s.SyncTree) + if err != nil { + return + } + fullState, err := s.builder.Build(historyTree, nil) + if err != nil { + return + } + if len(fullState.DeletedIds) != len(s.state.DeletedIds) { + log.WarnCtx(ctx, "state does not have all deleted ids", + zap.Int("fullstate ids", len(fullState.DeletedIds)), + zap.Int("state ids", len(fullState.DeletedIds))) + s.state = fullState + err = s.deletionManager.UpdateState(context.Background(), s.state) + if err != nil { + return + } + } + return +} + func (s *settingsObject) Close() error { s.loop.Close() return s.SyncTree.Close() @@ -221,7 +255,7 @@ func (s *settingsObject) DeleteObject(id string) (err error) { err = ErrDeleteSelf return } - if s.deletionState.Exists(id) { + if s.state.Exists(id) { err = ErrAlreadyDeleted return nil } @@ -230,7 +264,7 @@ func (s *settingsObject) DeleteObject(id string) (err error) { err = ErrObjDoesNotExist return } - isSnapshot := doSnapshot(s.Len()) + isSnapshot := DoSnapshot(s.Len()) res, err := s.changeFactory.CreateObjectDeleteChange(id, s.state, isSnapshot) if err != nil { return @@ -249,7 +283,7 @@ func (s *settingsObject) verifyDeleteSpace(raw *treechangeproto.RawTreeChangeWit func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) { accountData := s.account.Account() - _, err = s.AddContent(context.Background(), objecttree.SignableChangeContent{ + res, err := s.AddContent(context.Background(), objecttree.SignableChangeContent{ Data: data, Key: accountData.SignKey, IsSnapshot: isSnapshot, @@ -258,8 +292,11 @@ func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) { if err != nil { return } - - s.Update(s) + if res.Mode == objecttree.Rebuild { + s.Rebuild(s) + } else { + s.Update(s) + } return } diff --git a/commonspace/settings/settings_test.go b/commonspace/settings/settings_test.go index b4879f53..264e6a11 100644 --- a/commonspace/settings/settings_test.go +++ b/commonspace/settings/settings_test.go @@ -5,6 +5,7 @@ import ( "github.com/anytypeio/any-sync/accountservice/mock_accountservice" "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" @@ -52,6 +53,7 @@ type settingsFixture struct { changeFactory *mock_settingsstate.MockChangeFactory deleter *mock_settings.MockDeleter syncTree *mock_synctree.MockSyncTree + historyTree *mock_objecttree.MockObjectTree delState *mock_settingsstate.MockObjectDeletionState account *mock_accountservice.MockService } @@ -69,6 +71,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { stateBuilder := mock_settingsstate.NewMockStateBuilder(ctrl) changeFactory := mock_settingsstate.NewMockChangeFactory(ctrl) syncTree := mock_synctree.NewMockSyncTree(ctrl) + historyTree := mock_objecttree.NewMockObjectTree(ctrl) del := mock_settings.NewMockDeleter(ctrl) delState.EXPECT().AddObserver(gomock.Any()) @@ -77,6 +80,9 @@ func newSettingsFixture(t *testing.T) *settingsFixture { require.Equal(t, objectId, id) return newTestObjMock(syncTree), nil }) + buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) { + return historyTree, nil + } deps := Deps{ BuildFunc: buildFunc, @@ -104,45 +110,48 @@ func newSettingsFixture(t *testing.T) *settingsFixture { syncTree: syncTree, account: acc, delState: delState, + historyTree: historyTree, } } -func (fx *settingsFixture) stop() { +func (fx *settingsFixture) init(t *testing.T) { + fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) + fx.deleter.EXPECT().Delete() + fx.stateBuilder.EXPECT().Build(fx.historyTree, nil).Return(&settingsstate.State{}, nil) + fx.doc.state = &settingsstate.State{} + + err := fx.doc.Init(context.Background()) + require.NoError(t, err) +} + +func (fx *settingsFixture) stop(t *testing.T) { + fx.syncTree.EXPECT().Close().Return(nil) + + err := fx.doc.Close() + require.NoError(t, err) fx.ctrl.Finish() } func TestSettingsObject_Init(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - fx.syncTree.EXPECT().Close().Return(nil) - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) - err = fx.doc.Close() - require.NoError(t, err) + fx.init(t) } func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + fx.init(t) delId := "delId" - doSnapshot = func(len int) bool { + DoSnapshot = func(len int) bool { return false } fx.syncTree.EXPECT().Id().Return("syncId") fx.syncTree.EXPECT().Len().Return(10) - fx.delState.EXPECT().Exists(delId).Return(false) fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil) res := []byte("settingsData") fx.doc.state = &settingsstate.State{LastIteratedId: "someId"} @@ -162,30 +171,20 @@ func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) { fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil) err = fx.doc.DeleteObject(delId) require.NoError(t, err) - - fx.syncTree.EXPECT().Close().Return(nil) - err = fx.doc.Close() - require.NoError(t, err) } func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() - - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + defer fx.stop(t) + fx.init(t) delId := "delId" - doSnapshot = func(len int) bool { + DoSnapshot = func(len int) bool { return true } fx.syncTree.EXPECT().Id().Return("syncId") fx.syncTree.EXPECT().Len().Return(10) - fx.delState.EXPECT().Exists(delId).Return(false) fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil) res := []byte("settingsData") fx.doc.state = &settingsstate.State{LastIteratedId: "someId"} @@ -199,27 +198,19 @@ func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) { Key: accountData.SignKey, IsSnapshot: true, IsEncrypted: false, - }).Return(objecttree.AddResult{}, nil) + }).Return(objecttree.AddResult{Mode: objecttree.Rebuild}, nil) - fx.stateBuilder.EXPECT().Build(fx.doc, fx.doc.state).Return(fx.doc.state, nil) + fx.stateBuilder.EXPECT().Build(fx.doc, nil).Return(fx.doc.state, nil) fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil) err = fx.doc.DeleteObject(delId) require.NoError(t, err) - - fx.syncTree.EXPECT().Close().Return(nil) - err = fx.doc.Close() - require.NoError(t, err) } func TestSettingsObject_Rebuild(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + fx.init(t) time.Sleep(100 * time.Millisecond) newSt := &settingsstate.State{} @@ -232,13 +223,9 @@ func TestSettingsObject_Rebuild(t *testing.T) { func TestSettingsObject_Update(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + fx.init(t) time.Sleep(100 * time.Millisecond) fx.doc.state = &settingsstate.State{} @@ -250,13 +237,9 @@ func TestSettingsObject_Update(t *testing.T) { func TestSettingsObject_DeleteSpace(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + fx.init(t) time.Sleep(100 * time.Millisecond) deleterId := "delId" @@ -275,19 +258,15 @@ func TestSettingsObject_DeleteSpace(t *testing.T) { Heads: []string{rawCh.Id}, }, nil) - err = fx.doc.DeleteSpace(context.Background(), rawCh) + err := fx.doc.DeleteSpace(context.Background(), rawCh) require.NoError(t, err) } func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) { fx := newSettingsFixture(t) - defer fx.stop() + defer fx.stop(t) - fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) - fx.deleter.EXPECT().Delete() - - err := fx.doc.Init(context.Background()) - require.NoError(t, err) + fx.init(t) time.Sleep(100 * time.Millisecond) t.Run("incorrect change type", func(t *testing.T) { @@ -299,7 +278,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) { delChange, _ := changeFactory.CreateObjectDeleteChange("otherId", &settingsstate.State{}, false) fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil) - err = fx.doc.DeleteSpace(context.Background(), rawCh) + err := fx.doc.DeleteSpace(context.Background(), rawCh) require.NotNil(t, err) }) @@ -312,7 +291,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) { delChange, _ := changeFactory.CreateSpaceDeleteChange("", &settingsstate.State{}, false) fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil) - err = fx.doc.DeleteSpace(context.Background(), rawCh) + err := fx.doc.DeleteSpace(context.Background(), rawCh) require.NotNil(t, err) }) } diff --git a/commonspace/settings/settingsstate/changefactory.go b/commonspace/settings/settingsstate/changefactory.go index a5a70dc5..03a8de2f 100644 --- a/commonspace/settings/settingsstate/changefactory.go +++ b/commonspace/settings/settingsstate/changefactory.go @@ -50,7 +50,7 @@ func (c *changeFactory) CreateSpaceDeleteChange(peerId string, state *State, isS func (c *changeFactory) makeSnapshot(state *State, objectId, deleterPeer string) *spacesyncproto.SpaceSettingsSnapshot { var ( - deletedIds = state.DeletedIds + deletedIds = make([]string, 0, len(state.DeletedIds)+1) deleterId = state.DeleterId ) if objectId != "" { @@ -59,6 +59,9 @@ func (c *changeFactory) makeSnapshot(state *State, objectId, deleterPeer string) if deleterPeer != "" { deleterId = deleterPeer } + for id := range state.DeletedIds { + deletedIds = append(deletedIds, id) + } return &spacesyncproto.SpaceSettingsSnapshot{ DeletedIds: deletedIds, DeleterPeerId: deleterId, diff --git a/commonspace/settings/settingsstate/changefactory_test.go b/commonspace/settings/settingsstate/changefactory_test.go index 41135b36..35cfbf55 100644 --- a/commonspace/settings/settingsstate/changefactory_test.go +++ b/commonspace/settings/settingsstate/changefactory_test.go @@ -4,13 +4,14 @@ import ( "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "testing" ) func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) { factory := NewChangeFactory() state := &State{ - DeletedIds: []string{"1", "2"}, + DeletedIds: map[string]struct{}{"1": {}, "2": {}}, DeleterId: "del", } marshalled, err := factory.CreateObjectDeleteChange("3", state, false) @@ -26,6 +27,7 @@ func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) { data = &spacesyncproto.SettingsData{} err = proto.Unmarshal(marshalled, data) require.NoError(t, err) + slices.Sort(data.Snapshot.DeletedIds) require.Equal(t, &spacesyncproto.SpaceSettingsSnapshot{ DeletedIds: []string{"1", "2", "3"}, DeleterPeerId: "del", @@ -36,7 +38,7 @@ func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) { func TestChangeFactory_CreateSpaceDeleteChange(t *testing.T) { factory := NewChangeFactory() state := &State{ - DeletedIds: []string{"1", "2"}, + DeletedIds: map[string]struct{}{"1": {}, "2": {}}, } marshalled, err := factory.CreateSpaceDeleteChange("del", state, false) require.NoError(t, err) @@ -51,6 +53,7 @@ func TestChangeFactory_CreateSpaceDeleteChange(t *testing.T) { data = &spacesyncproto.SettingsData{} err = proto.Unmarshal(marshalled, data) require.NoError(t, err) + slices.Sort(data.Snapshot.DeletedIds) require.Equal(t, &spacesyncproto.SpaceSettingsSnapshot{ DeletedIds: []string{"1", "2"}, DeleterPeerId: "del", diff --git a/commonspace/settings/settingsstate/deletionstate.go b/commonspace/settings/settingsstate/deletionstate.go index a65152de..4641832e 100644 --- a/commonspace/settings/settingsstate/deletionstate.go +++ b/commonspace/settings/settingsstate/deletionstate.go @@ -2,7 +2,9 @@ package settingsstate import ( + "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/spacestorage" + "go.uber.org/zap" "sync" ) @@ -10,7 +12,7 @@ type StateUpdateObserver func(ids []string) type ObjectDeletionState interface { AddObserver(observer StateUpdateObserver) - Add(ids []string) (err error) + Add(ids map[string]struct{}) GetQueued() (ids []string) Delete(id string) (err error) Exists(id string) bool @@ -19,14 +21,16 @@ type ObjectDeletionState interface { type objectDeletionState struct { sync.RWMutex + log logger.CtxLogger queued map[string]struct{} deleted map[string]struct{} stateUpdateObservers []StateUpdateObserver storage spacestorage.SpaceStorage } -func NewObjectDeletionState(storage spacestorage.SpaceStorage) ObjectDeletionState { +func NewObjectDeletionState(log logger.CtxLogger, storage spacestorage.SpaceStorage) ObjectDeletionState { return &objectDeletionState{ + log: log, queued: map[string]struct{}{}, deleted: map[string]struct{}{}, storage: storage, @@ -39,19 +43,17 @@ func (st *objectDeletionState) AddObserver(observer StateUpdateObserver) { st.stateUpdateObservers = append(st.stateUpdateObservers, observer) } -func (st *objectDeletionState) Add(ids []string) (err error) { +func (st *objectDeletionState) Add(ids map[string]struct{}) { + var added []string st.Lock() defer func() { st.Unlock() - if err != nil { - return - } for _, ob := range st.stateUpdateObservers { - ob(ids) + ob(added) } }() - for _, id := range ids { + for id := range ids { if _, exists := st.deleted[id]; exists { continue } @@ -60,9 +62,10 @@ func (st *objectDeletionState) Add(ids []string) (err error) { } var status string - status, err = st.storage.TreeDeletedStatus(id) + status, err := st.storage.TreeDeletedStatus(id) if err != nil { - return + st.log.Warn("failed to get deleted status", zap.String("treeId", id), zap.Error(err)) + continue } switch status { @@ -71,14 +74,15 @@ func (st *objectDeletionState) Add(ids []string) (err error) { case spacestorage.TreeDeletedStatusDeleted: st.deleted[id] = struct{}{} default: - st.queued[id] = struct{}{} - err = st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued) + err := st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued) if err != nil { - return + st.log.Warn("failed to set deleted status", zap.String("treeId", id), zap.Error(err)) + continue } + st.queued[id] = struct{}{} } + added = append(added, id) } - return } func (st *objectDeletionState) GetQueued() (ids []string) { diff --git a/commonspace/settings/settingsstate/deletionstate_test.go b/commonspace/settings/settingsstate/deletionstate_test.go index 31b14a0d..2320bf98 100644 --- a/commonspace/settings/settingsstate/deletionstate_test.go +++ b/commonspace/settings/settingsstate/deletionstate_test.go @@ -1,6 +1,7 @@ package settingsstate import ( + "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" "github.com/golang/mock/gomock" @@ -18,7 +19,7 @@ type fixture struct { func newFixture(t *testing.T) *fixture { ctrl := gomock.NewController(t) spaceStorage := mock_spacestorage.NewMockSpaceStorage(ctrl) - delState := NewObjectDeletionState(spaceStorage).(*objectDeletionState) + delState := NewObjectDeletionState(logger.NewNamed("test"), spaceStorage).(*objectDeletionState) return &fixture{ ctrl: ctrl, delState: delState, @@ -37,8 +38,7 @@ func TestDeletionState_Add(t *testing.T) { id := "newId" fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil) fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil) - err := fx.delState.Add([]string{id}) - require.NoError(t, err) + fx.delState.Add(map[string]struct{}{id: {}}) require.Contains(t, fx.delState.queued, id) }) @@ -47,8 +47,7 @@ func TestDeletionState_Add(t *testing.T) { defer fx.stop() id := "newId" fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusQueued, nil) - err := fx.delState.Add([]string{id}) - require.NoError(t, err) + fx.delState.Add(map[string]struct{}{id: {}}) require.Contains(t, fx.delState.queued, id) }) @@ -57,8 +56,7 @@ func TestDeletionState_Add(t *testing.T) { defer fx.stop() id := "newId" fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusDeleted, nil) - err := fx.delState.Add([]string{id}) - require.NoError(t, err) + fx.delState.Add(map[string]struct{}{id: {}}) require.Contains(t, fx.delState.deleted, id) }) } @@ -98,8 +96,7 @@ func TestDeletionState_AddObserver(t *testing.T) { id := "newId" fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil) fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil) - err := fx.delState.Add([]string{id}) - require.NoError(t, err) + fx.delState.Add(map[string]struct{}{id: {}}) require.Contains(t, fx.delState.queued, id) require.Equal(t, []string{id}, queued) } diff --git a/commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go b/commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go index 948f72c4..b19a26ee 100644 --- a/commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go +++ b/commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go @@ -36,11 +36,9 @@ func (m *MockObjectDeletionState) EXPECT() *MockObjectDeletionStateMockRecorder } // Add mocks base method. -func (m *MockObjectDeletionState) Add(arg0 []string) error { +func (m *MockObjectDeletionState) Add(arg0 map[string]struct{}) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Add", arg0) - ret0, _ := ret[0].(error) - return ret0 + m.ctrl.Call(m, "Add", arg0) } // Add indicates an expected call of Add. @@ -145,7 +143,7 @@ func (m *MockStateBuilder) EXPECT() *MockStateBuilderMockRecorder { } // Build mocks base method. -func (m *MockStateBuilder) Build(arg0 objecttree.ObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) { +func (m *MockStateBuilder) Build(arg0 objecttree.ReadableObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Build", arg0, arg1) ret0, _ := ret[0].(*settingsstate.State) diff --git a/commonspace/settings/settingsstate/settingsstate.go b/commonspace/settings/settingsstate/settingsstate.go index 2f4f0afd..bd010714 100644 --- a/commonspace/settings/settingsstate/settingsstate.go +++ b/commonspace/settings/settingsstate/settingsstate.go @@ -1,7 +1,28 @@ package settingsstate +import "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + type State struct { - DeletedIds []string + DeletedIds map[string]struct{} DeleterId string LastIteratedId string } + +func NewState() *State { + return &State{DeletedIds: map[string]struct{}{}} +} + +func NewStateFromSnapshot(snapshot *spacesyncproto.SpaceSettingsSnapshot, lastIteratedId string) *State { + st := NewState() + for _, id := range snapshot.DeletedIds { + st.DeletedIds[id] = struct{}{} + } + st.DeleterId = snapshot.DeleterPeerId + st.LastIteratedId = lastIteratedId + return st +} + +func (s *State) Exists(id string) bool { + _, exists := s.DeletedIds[id] + return exists +} diff --git a/commonspace/settings/settingsstate/statebuilder.go b/commonspace/settings/settingsstate/statebuilder.go index a6cf2c68..7f171ec3 100644 --- a/commonspace/settings/settingsstate/statebuilder.go +++ b/commonspace/settings/settingsstate/statebuilder.go @@ -7,7 +7,7 @@ import ( ) type StateBuilder interface { - Build(tree objecttree.ObjectTree, state *State) (*State, error) + Build(tree objecttree.ReadableObjectTree, state *State) (*State, error) } func NewStateBuilder() StateBuilder { @@ -17,14 +17,14 @@ func NewStateBuilder() StateBuilder { type stateBuilder struct { } -func (s *stateBuilder) Build(tr objecttree.ObjectTree, oldState *State) (state *State, err error) { +func (s *stateBuilder) Build(tr objecttree.ReadableObjectTree, oldState *State) (state *State, err error) { var ( rootId = tr.Root().Id startId = rootId ) state = oldState if state == nil { - state = &State{} + state = NewState() } else if state.LastIteratedId != "" { startId = state.LastIteratedId } @@ -55,11 +55,7 @@ func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, s deleteChange := change.Model.(*spacesyncproto.SettingsData) // getting data from snapshot if we start from it if change.Id == rootId { - state = &State{ - DeletedIds: deleteChange.Snapshot.DeletedIds, - DeleterId: deleteChange.Snapshot.DeleterPeerId, - LastIteratedId: rootId, - } + state = NewStateFromSnapshot(deleteChange.Snapshot, rootId) return state } @@ -67,7 +63,7 @@ func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, s for _, cnt := range deleteChange.Content { switch { case cnt.GetObjectDelete() != nil: - state.DeletedIds = append(state.DeletedIds, cnt.GetObjectDelete().GetId()) + state.DeletedIds[cnt.GetObjectDelete().GetId()] = struct{}{} case cnt.GetSpaceDelete() != nil: state.DeleterId = cnt.GetSpaceDelete().GetDeleterPeerId() } diff --git a/commonspace/settings/settingsstate/statebuilder_test.go b/commonspace/settings/settingsstate/statebuilder_test.go index e2ac4448..3119814b 100644 --- a/commonspace/settings/settingsstate/statebuilder_test.go +++ b/commonspace/settings/settingsstate/statebuilder_test.go @@ -17,9 +17,9 @@ func TestStateBuilder_ProcessChange(t *testing.T) { t.Run("empty model", func(t *testing.T) { ch := &objecttree.Change{} newSt := sb.processChange(ch, rootId, &State{ - DeletedIds: []string{deletedId}, + DeletedIds: map[string]struct{}{deletedId: struct{}{}}, }) - require.Equal(t, []string{deletedId}, newSt.DeletedIds) + require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds) }) t.Run("changeId is equal to startId, LastIteratedId is equal to startId", func(t *testing.T) { @@ -34,10 +34,10 @@ func TestStateBuilder_ProcessChange(t *testing.T) { ch.Id = "startId" startId := "startId" newSt := sb.processChange(ch, rootId, &State{ - DeletedIds: []string{deletedId}, + DeletedIds: map[string]struct{}{deletedId: struct{}{}}, LastIteratedId: startId, }) - require.Equal(t, []string{deletedId}, newSt.DeletedIds) + require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds) }) t.Run("changeId is equal to rootId", func(t *testing.T) { @@ -50,8 +50,8 @@ func TestStateBuilder_ProcessChange(t *testing.T) { }, } ch.Id = "rootId" - newSt := sb.processChange(ch, rootId, &State{}) - require.Equal(t, []string{"id1", "id2"}, newSt.DeletedIds) + newSt := sb.processChange(ch, rootId, NewState()) + require.Equal(t, map[string]struct{}{"id1": struct{}{}, "id2": struct{}{}}, newSt.DeletedIds) require.Equal(t, "peerId", newSt.DeleterId) }) @@ -66,8 +66,8 @@ func TestStateBuilder_ProcessChange(t *testing.T) { }, } ch.Id = "someId" - newSt := sb.processChange(ch, rootId, &State{}) - require.Equal(t, []string{deletedId}, newSt.DeletedIds) + newSt := sb.processChange(ch, rootId, NewState()) + require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds) }) } diff --git a/commonspace/space.go b/commonspace/space.go index acd58a22..6ad9c22e 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -195,7 +195,7 @@ func (s *space) Init(ctx context.Context) (err error) { s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.SyncClient().MessagePool()) s.treeManager.AddObject(s.aclList) - deletionState := settingsstate.NewObjectDeletionState(s.storage) + deletionState := settingsstate.NewObjectDeletionState(log, s.storage) deps := settings.Deps{ BuildFunc: func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) { res, err := s.BuildTree(ctx, id, BuildTreeOpts{ diff --git a/commonspace/spacestorage/inmemoryprovider.go b/commonspace/spacestorage/inmemoryprovider.go new file mode 100644 index 00000000..02dd65ac --- /dev/null +++ b/commonspace/spacestorage/inmemoryprovider.go @@ -0,0 +1,60 @@ +package spacestorage + +import ( + "context" + "github.com/anytypeio/any-sync/app" + "sync" +) + +func NewInMemorySpaceStorageProvider() SpaceStorageProvider { + return &InMemorySpaceStorageProvider{ + storages: map[string]SpaceStorage{}, + } +} + +type InMemorySpaceStorageProvider struct { + storages map[string]SpaceStorage + sync.Mutex +} + +func (i *InMemorySpaceStorageProvider) Init(a *app.App) (err error) { + return nil +} + +func (i *InMemorySpaceStorageProvider) Name() (name string) { + return CName +} + +func (i *InMemorySpaceStorageProvider) WaitSpaceStorage(ctx context.Context, id string) (SpaceStorage, error) { + i.Lock() + defer i.Unlock() + storage, exists := i.storages[id] + if !exists { + return nil, ErrSpaceStorageMissing + } + return storage, nil +} + +func (i *InMemorySpaceStorageProvider) SpaceExists(id string) bool { + i.Lock() + defer i.Unlock() + _, exists := i.storages[id] + return exists +} + +func (i *InMemorySpaceStorageProvider) CreateSpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) { + i.Lock() + defer i.Unlock() + spaceStorage, err := NewInMemorySpaceStorage(payload) + if err != nil { + return nil, err + } + i.storages[payload.SpaceHeaderWithId.Id] = spaceStorage + return spaceStorage, nil +} + +func (i *InMemorySpaceStorageProvider) SetStorage(storage SpaceStorage) { + i.Lock() + defer i.Unlock() + i.storages[storage.Id()] = storage +} diff --git a/commonspace/spacestorage/inmemorystorage.go b/commonspace/spacestorage/inmemorystorage.go new file mode 100644 index 00000000..f6071afb --- /dev/null +++ b/commonspace/spacestorage/inmemorystorage.go @@ -0,0 +1,193 @@ +package spacestorage + +import ( + "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "sync" +) + +type InMemorySpaceStorage struct { + id string + isDeleted bool + spaceSettingsId string + treeDeleted map[string]string + trees map[string]treestorage.TreeStorage + aclStorage liststorage.ListStorage + spaceHeader *spacesyncproto.RawSpaceHeaderWithId + spaceHash string + sync.Mutex +} + +func NewInMemorySpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) { + aclStorage, err := liststorage.NewInMemoryAclListStorage(payload.AclWithId.Id, []*aclrecordproto.RawAclRecordWithId{payload.AclWithId}) + if err != nil { + return nil, err + } + inMemory := &InMemorySpaceStorage{ + id: payload.SpaceHeaderWithId.Id, + spaceSettingsId: payload.SpaceSettingsWithId.Id, + treeDeleted: map[string]string{}, + trees: map[string]treestorage.TreeStorage{}, + aclStorage: aclStorage, + spaceHeader: payload.SpaceHeaderWithId, + } + _, err = inMemory.CreateTreeStorage(treestorage.TreeStorageCreatePayload{ + RootRawChange: payload.SpaceSettingsWithId, + Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId}, + Heads: []string{payload.SpaceSettingsWithId.Id}, + }) + if err != nil { + return nil, err + } + return inMemory, nil +} + +func (i *InMemorySpaceStorage) Id() string { + return i.id +} + +func (i *InMemorySpaceStorage) SetSpaceDeleted() error { + i.Lock() + defer i.Unlock() + i.isDeleted = true + return nil +} + +func (i *InMemorySpaceStorage) IsSpaceDeleted() (bool, error) { + i.Lock() + defer i.Unlock() + return i.isDeleted, nil +} + +func (i *InMemorySpaceStorage) SetTreeDeletedStatus(id, state string) error { + i.Lock() + defer i.Unlock() + i.treeDeleted[id] = state + return nil +} + +func (i *InMemorySpaceStorage) TreeDeletedStatus(id string) (string, error) { + i.Lock() + defer i.Unlock() + return i.treeDeleted[id], nil +} + +func (i *InMemorySpaceStorage) SpaceSettingsId() string { + return i.spaceSettingsId +} + +func (i *InMemorySpaceStorage) AclStorage() (liststorage.ListStorage, error) { + return i.aclStorage, nil +} + +func (i *InMemorySpaceStorage) SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error) { + return i.spaceHeader, nil +} + +func (i *InMemorySpaceStorage) StoredIds() ([]string, error) { + i.Lock() + defer i.Unlock() + var allIds []string + for id := range i.trees { + allIds = append(allIds, id) + } + return allIds, nil +} + +func (i *InMemorySpaceStorage) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error) { + i.Lock() + defer i.Unlock() + treeStorage, exists := i.trees[id] + if !exists { + return nil, treestorage.ErrUnknownTreeId + } + return treeStorage.Root() +} + +func (i *InMemorySpaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) { + i.Lock() + defer i.Unlock() + treeStorage, exists := i.trees[id] + if !exists { + return nil, treestorage.ErrUnknownTreeId + } + return treeStorage, nil +} + +func (i *InMemorySpaceStorage) HasTree(id string) (bool, error) { + i.Lock() + defer i.Unlock() + _, exists := i.trees[id] + return exists, nil +} + +func (i *InMemorySpaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) { + i.Lock() + defer i.Unlock() + storage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes) + if err != nil { + return nil, err + } + i.trees[payload.RootRawChange.Id] = storage + return storage, nil +} + +func (i *InMemorySpaceStorage) WriteSpaceHash(hash string) error { + i.Lock() + defer i.Unlock() + i.spaceHash = hash + return nil +} + +func (i *InMemorySpaceStorage) ReadSpaceHash() (hash string, err error) { + i.Lock() + defer i.Unlock() + return i.spaceHash, nil +} + +func (i *InMemorySpaceStorage) Close() error { + return nil +} + +func (i *InMemorySpaceStorage) AllTrees() map[string]treestorage.TreeStorage { + i.Lock() + defer i.Unlock() + cp := map[string]treestorage.TreeStorage{} + for id, store := range i.trees { + cp[id] = store + } + return cp +} + +func (i *InMemorySpaceStorage) SetTrees(trees map[string]treestorage.TreeStorage) { + i.Lock() + defer i.Unlock() + i.trees = trees +} + +func (i *InMemorySpaceStorage) CopyStorage() *InMemorySpaceStorage { + i.Lock() + defer i.Unlock() + copyTreeDeleted := map[string]string{} + for id, status := range i.treeDeleted { + copyTreeDeleted[id] = status + } + copyTrees := map[string]treestorage.TreeStorage{} + for id, store := range i.trees { + copyTrees[id] = store + } + return &InMemorySpaceStorage{ + id: i.id, + isDeleted: i.isDeleted, + spaceSettingsId: i.spaceSettingsId, + treeDeleted: copyTreeDeleted, + trees: copyTrees, + aclStorage: i.aclStorage, + spaceHeader: i.spaceHeader, + spaceHash: i.spaceHash, + Mutex: sync.Mutex{}, + } +} diff --git a/commonspace/spaceutils_test.go b/commonspace/spaceutils_test.go new file mode 100644 index 00000000..62371bbd --- /dev/null +++ b/commonspace/spaceutils_test.go @@ -0,0 +1,321 @@ +package commonspace + +import ( + "context" + "fmt" + accountService "github.com/anytypeio/any-sync/accountservice" + "github.com/anytypeio/any-sync/app" + "github.com/anytypeio/any-sync/app/ocache" + "github.com/anytypeio/any-sync/commonspace/credentialprovider" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" + "github.com/anytypeio/any-sync/commonspace/peermanager" + "github.com/anytypeio/any-sync/commonspace/spacestorage" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/net/pool" + "github.com/anytypeio/any-sync/nodeconf" + "github.com/anytypeio/any-sync/testutil/accounttest" + "github.com/anytypeio/go-chash" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +// +// Mock NodeConf implementation +// + +type mockConf struct { + id string + networkId string + configuration nodeconf.Configuration +} + +func (m *mockConf) NetworkCompatibilityStatus() nodeconf.NetworkCompatibilityStatus { + return nodeconf.NetworkCompatibilityStatusOk +} + +func (m *mockConf) Init(a *app.App) (err error) { + accountKeys := a.MustComponent(accountService.CName).(accountService.Service).Account() + networkId := accountKeys.SignKey.GetPublic().Network() + node := nodeconf.Node{ + PeerId: accountKeys.PeerId, + Addresses: []string{"127.0.0.1:4430"}, + Types: []nodeconf.NodeType{nodeconf.NodeTypeTree}, + } + m.id = networkId + m.networkId = networkId + m.configuration = nodeconf.Configuration{ + Id: networkId, + NetworkId: networkId, + Nodes: []nodeconf.Node{node}, + CreationTime: time.Now(), + } + return nil +} + +func (m *mockConf) Name() (name string) { + return nodeconf.CName +} + +func (m *mockConf) Run(ctx context.Context) (err error) { + return nil +} + +func (m *mockConf) Close(ctx context.Context) (err error) { + return nil +} + +func (m *mockConf) Id() string { + return m.id +} + +func (m *mockConf) Configuration() nodeconf.Configuration { + return m.configuration +} + +func (m *mockConf) NodeIds(spaceId string) []string { + var nodeIds []string + for _, node := range m.configuration.Nodes { + nodeIds = append(nodeIds, node.PeerId) + } + return nodeIds +} + +func (m *mockConf) IsResponsible(spaceId string) bool { + return true +} + +func (m *mockConf) FilePeers() []string { + return nil +} + +func (m *mockConf) ConsensusPeers() []string { + return nil +} + +func (m *mockConf) CoordinatorPeers() []string { + return nil +} + +func (m *mockConf) PeerAddresses(peerId string) (addrs []string, ok bool) { + if peerId == m.configuration.Nodes[0].PeerId { + return m.configuration.Nodes[0].Addresses, true + } + return nil, false +} + +func (m *mockConf) CHash() chash.CHash { + return nil +} + +func (m *mockConf) Partition(spaceId string) (part int) { + return 0 +} + +func (m *mockConf) NodeTypes(nodeId string) []nodeconf.NodeType { + if nodeId == m.configuration.Nodes[0].PeerId { + return m.configuration.Nodes[0].Types + } + return nil +} + +// +// Mock PeerManager +// + +type mockPeerManager struct { +} + +func (p *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + return nil +} + +func (p *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + return nil +} + +func (p *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + return nil, nil +} + +// +// Mock PeerManagerProvider +// + +type mockPeerManagerProvider struct { +} + +func (m *mockPeerManagerProvider) Init(a *app.App) (err error) { + return nil +} + +func (m *mockPeerManagerProvider) Name() (name string) { + return peermanager.CName +} + +func (m *mockPeerManagerProvider) NewPeerManager(ctx context.Context, spaceId string) (sm peermanager.PeerManager, err error) { + return &mockPeerManager{}, nil +} + +// +// Mock Pool +// + +type mockPool struct { +} + +func (m *mockPool) Init(a *app.App) (err error) { + return nil +} + +func (m *mockPool) Name() (name string) { + return pool.CName +} + +func (m *mockPool) Get(ctx context.Context, id string) (peer.Peer, error) { + return nil, fmt.Errorf("no such peer") +} + +func (m *mockPool) Dial(ctx context.Context, id string) (peer.Peer, error) { + return nil, fmt.Errorf("can't dial peer") +} + +func (m *mockPool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) { + return nil, fmt.Errorf("can't dial peer") +} + +func (m *mockPool) DialOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) { + return nil, fmt.Errorf("can't dial peer") +} + +// +// Mock Config +// + +type mockConfig struct { +} + +func (m *mockConfig) Init(a *app.App) (err error) { + return nil +} + +func (m *mockConfig) Name() (name string) { + return "config" +} + +func (m *mockConfig) GetSpace() Config { + return Config{ + GCTTL: 60, + SyncPeriod: 20, + KeepTreeDataInMemory: true, + } +} + +// +// Mock TreeManager +// + +type mockTreeManager struct { + space Space + cache ocache.OCache + deletedIds []string + markedIds []string +} + +func (t *mockTreeManager) MarkTreeDeleted(ctx context.Context, spaceId, treeId string) error { + t.markedIds = append(t.markedIds, treeId) + return nil +} + +func (t *mockTreeManager) Init(a *app.App) (err error) { + t.cache = ocache.New(func(ctx context.Context, id string) (value ocache.Object, err error) { + return t.space.BuildTree(ctx, id, BuildTreeOpts{}) + }, + ocache.WithGCPeriod(time.Minute), + ocache.WithTTL(time.Duration(60)*time.Second)) + return nil +} + +func (t *mockTreeManager) Name() (name string) { + return treemanager.CName +} + +func (t *mockTreeManager) Run(ctx context.Context) (err error) { + return nil +} + +func (t *mockTreeManager) Close(ctx context.Context) (err error) { + return t.cache.Close() +} + +func (t *mockTreeManager) GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) { + val, err := t.cache.Get(ctx, treeId) + if err != nil { + return nil, err + } + return val.(objecttree.ObjectTree), nil +} + +func (t *mockTreeManager) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) { + tr, err := t.GetTree(ctx, spaceId, treeId) + if err != nil { + return + } + err = tr.Delete() + if err != nil { + return + } + t.deletedIds = append(t.deletedIds, treeId) + _, err = t.cache.Remove(ctx, treeId) + return nil +} + +// +// Space fixture +// + +type spaceFixture struct { + app *app.App + config *mockConfig + account accountService.Service + configurationService nodeconf.Service + storageProvider spacestorage.SpaceStorageProvider + peermanagerProvider peermanager.PeerManagerProvider + credentialProvider credentialprovider.CredentialProvider + treeManager *mockTreeManager + pool *mockPool + spaceService SpaceService + cancelFunc context.CancelFunc +} + +func newFixture(t *testing.T) *spaceFixture { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + fx := &spaceFixture{ + cancelFunc: cancel, + config: &mockConfig{}, + app: &app.App{}, + account: &accounttest.AccountTestService{}, + configurationService: &mockConf{}, + storageProvider: spacestorage.NewInMemorySpaceStorageProvider(), + peermanagerProvider: &mockPeerManagerProvider{}, + treeManager: &mockTreeManager{}, + pool: &mockPool{}, + spaceService: New(), + } + fx.app.Register(fx.account). + Register(fx.config). + Register(fx.configurationService). + Register(fx.storageProvider). + Register(fx.peermanagerProvider). + Register(fx.treeManager). + Register(fx.pool). + Register(fx.spaceService) + err := fx.app.Start(ctx) + if err != nil { + fx.cancelFunc() + } + require.NoError(t, err) + return fx +}