benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #103 from anytypeio/fix-settings-state

Browse Source
This commit is contained in:
Mikhail Rakhmanov 2023-05-23 12:34:02 +02:00 committed by GitHub
commit 333402490c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1092 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
commonspace/deletion_test.go Normal file
View File

@ -0,0 +1,271 @@
package commonspace
import (
"context"
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/accountdata"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/settings"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/util/crypto"
"github.com/stretchr/testify/require"
"math/rand"
"testing"
"time"
)
func addIncorrectSnapshot(settingsObject settings.SettingsObject, acc *accountdata.AccountKeys, partialIds map[string]struct{}, newId string) (err error) {
factory := settingsstate.NewChangeFactory()
bytes, err := factory.CreateObjectDeleteChange(newId, &settingsstate.State{DeletedIds: partialIds}, true)
if err != nil {
return
}
ch, err := settingsObject.PrepareChange(objecttree.SignableChangeContent{
Data: bytes,
Key: acc.SignKey,
IsSnapshot: true,
IsEncrypted: false,
Timestamp: time.Now().Unix(),
})
if err != nil {
return
}
res, err := settingsObject.AddRawChanges(context.Background(), objecttree.RawChangesPayload{
NewHeads: []string{ch.Id},
RawChanges: []*treechangeproto.RawTreeChangeWithId{ch},
})
if err != nil {
return
}
if res.Mode != objecttree.Rebuild {
return fmt.Errorf("incorrect mode: %d", res.Mode)
}
return
}
func TestSpaceDeleteIds(t *testing.T) {
fx := newFixture(t)
acc := fx.account.Account()
rk := crypto.NewAES()
ctx := context.Background()
totalObjs := 1500
// creating space
sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{
SigningKey: acc.SignKey,
SpaceType: "type",
ReadKey: rk.Bytes(),
ReplicationKey: 10,
MasterKey: acc.PeerKey,
})
require.NoError(t, err)
require.NotNil(t, sp)
// initializing space
spc, err := fx.spaceService.NewSpace(ctx, sp)
require.NoError(t, err)
require.NotNil(t, spc)
// adding space to tree manager
fx.treeManager.space = spc
err = spc.Init(ctx)
require.NoError(t, err)
var ids []string
for i := 0; i < totalObjs; i++ {
// creating a tree
bytes := make([]byte, 32)
rand.Read(bytes)
doc, err := spc.CreateTree(ctx, objecttree.ObjectTreeCreatePayload{
PrivKey: acc.SignKey,
ChangeType: "some",
SpaceId: spc.Id(),
IsEncrypted: false,
Seed: bytes,
Timestamp: time.Now().Unix(),
})
require.NoError(t, err)
tr, err := spc.PutTree(ctx, doc, nil)
require.NoError(t, err)
ids = append(ids, tr.Id())
tr.Close()
}
// deleting trees
for _, id := range ids {
err = spc.DeleteTree(ctx, id)
require.NoError(t, err)
}
time.Sleep(3 * time.Second)
spc.Close()
require.Equal(t, len(ids), len(fx.treeManager.deletedIds))
}
func createTree(t *testing.T, ctx context.Context, spc Space, acc *accountdata.AccountKeys) string {
bytes := make([]byte, 32)
rand.Read(bytes)
doc, err := spc.CreateTree(ctx, objecttree.ObjectTreeCreatePayload{
PrivKey: acc.SignKey,
ChangeType: "some",
SpaceId: spc.Id(),
IsEncrypted: false,
Seed: bytes,
Timestamp: time.Now().Unix(),
})
require.NoError(t, err)
tr, err := spc.PutTree(ctx, doc, nil)
require.NoError(t, err)
tr.Close()
return tr.Id()
}
func TestSpaceDeleteIdsIncorrectSnapshot(t *testing.T) {
fx := newFixture(t)
acc := fx.account.Account()
rk := crypto.NewAES()
ctx := context.Background()
totalObjs := 1500
partialObjs := 300
// creating space
sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{
SigningKey: acc.SignKey,
SpaceType: "type",
ReadKey: rk.Bytes(),
ReplicationKey: 10,
MasterKey: acc.PeerKey,
})
require.NoError(t, err)
require.NotNil(t, sp)
// initializing space
spc, err := fx.spaceService.NewSpace(ctx, sp)
require.NoError(t, err)
require.NotNil(t, spc)
// adding space to tree manager
fx.treeManager.space = spc
err = spc.Init(ctx)
require.NoError(t, err)
settingsObject := spc.(*space).settingsObject
var ids []string
for i := 0; i < totalObjs; i++ {
id := createTree(t, ctx, spc, acc)
ids = append(ids, id)
}
// copying storage, so we will have all the trees locally
inmemory := spc.Storage().(*commonStorage).SpaceStorage.(*spacestorage.InMemorySpaceStorage)
storageCopy := inmemory.CopyStorage()
treesCopy := inmemory.AllTrees()
// deleting trees
for _, id := range ids {
err = spc.DeleteTree(ctx, id)
require.NoError(t, err)
}
mapIds := map[string]struct{}{}
for _, id := range ids[:partialObjs] {
mapIds[id] = struct{}{}
}
// adding snapshot that breaks the state
err = addIncorrectSnapshot(settingsObject, acc, mapIds, ids[partialObjs])
require.NoError(t, err)
// copying the contents of the settings tree
treesCopy[settingsObject.Id()] = settingsObject.Storage()
storageCopy.SetTrees(treesCopy)
spc.Close()
time.Sleep(100 * time.Millisecond)
// now we replace the storage, so the trees are back, but the settings object says that they are deleted
fx.storageProvider.(*spacestorage.InMemorySpaceStorageProvider).SetStorage(storageCopy)
spc, err = fx.spaceService.NewSpace(ctx, sp)
require.NoError(t, err)
require.NotNil(t, spc)
fx.treeManager.space = spc
fx.treeManager.deletedIds = nil
err = spc.Init(ctx)
require.NoError(t, err)
// waiting until everything is deleted
time.Sleep(3 * time.Second)
require.Equal(t, len(ids), len(fx.treeManager.deletedIds))
// checking that new snapshot will contain all the changes
settingsObject = spc.(*space).settingsObject
settings.DoSnapshot = func(treeLen int) bool {
return true
}
id := createTree(t, ctx, spc, acc)
err = spc.DeleteTree(ctx, id)
require.NoError(t, err)
delIds := settingsObject.Root().Model.(*spacesyncproto.SettingsData).Snapshot.DeletedIds
require.Equal(t, totalObjs+1, len(delIds))
}
func TestSpaceDeleteIdsMarkDeleted(t *testing.T) {
fx := newFixture(t)
acc := fx.account.Account()
rk := crypto.NewAES()
ctx := context.Background()
totalObjs := 1500
// creating space
sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{
SigningKey: acc.SignKey,
SpaceType: "type",
ReadKey: rk.Bytes(),
ReplicationKey: 10,
MasterKey: acc.PeerKey,
})
require.NoError(t, err)
require.NotNil(t, sp)
// initializing space
spc, err := fx.spaceService.NewSpace(ctx, sp)
require.NoError(t, err)
require.NotNil(t, spc)
// adding space to tree manager
fx.treeManager.space = spc
err = spc.Init(ctx)
require.NoError(t, err)
settingsObject := spc.(*space).settingsObject
var ids []string
for i := 0; i < totalObjs; i++ {
id := createTree(t, ctx, spc, acc)
ids = append(ids, id)
}
// copying storage, so we will have the same contents, except for empty trees
inmemory := spc.Storage().(*commonStorage).SpaceStorage.(*spacestorage.InMemorySpaceStorage)
storageCopy := inmemory.CopyStorage()
// deleting trees, this will prepare the document to have all the deletion changes
for _, id := range ids {
err = spc.DeleteTree(ctx, id)
require.NoError(t, err)
}
treesMap := map[string]treestorage.TreeStorage{}
// copying the contents of the settings tree
treesMap[settingsObject.Id()] = settingsObject.Storage()
storageCopy.SetTrees(treesMap)
spc.Close()
time.Sleep(100 * time.Millisecond)
// now we replace the storage, so the trees are back, but the settings object says that they are deleted
fx.storageProvider.(*spacestorage.InMemorySpaceStorageProvider).SetStorage(storageCopy)
spc, err = fx.spaceService.NewSpace(ctx, sp)
require.NoError(t, err)
require.NotNil(t, spc)
fx.treeManager.space = spc
fx.treeManager.deletedIds = nil
fx.treeManager.markedIds = nil
err = spc.Init(ctx)
require.NoError(t, err)
// waiting until everything is deleted
time.Sleep(3 * time.Second)
require.Equal(t, len(ids), len(fx.treeManager.markedIds))
require.Zero(t, len(fx.treeManager.deletedIds))
}

14
commonspace/object/treemanager/mock_treemanager/mock_treemanager.go
View File

@ -93,6 +93,20 @@ func (mr *MockTreeManagerMockRecorder) Init(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeManager)(nil).Init), arg0)
}
// MarkTreeDeleted mocks base method.
func (m *MockTreeManager) MarkTreeDeleted(arg0 context.Context, arg1, arg2 string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "MarkTreeDeleted", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// MarkTreeDeleted indicates an expected call of MarkTreeDeleted.
func (mr *MockTreeManagerMockRecorder) MarkTreeDeleted(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkTreeDeleted", reflect.TypeOf((*MockTreeManager)(nil).MarkTreeDeleted), arg0, arg1, arg2)
}
// Name mocks base method.
func (m *MockTreeManager) Name() string {
m.ctrl.T.Helper()

1
commonspace/object/treemanager/treemanager.go
View File

@ -12,5 +12,6 @@ const CName = "common.object.treemanager"
type TreeManager interface {
app.ComponentRunnable
GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error)
MarkTreeDeleted(ctx context.Context, spaceId, treeId string) error
DeleteTree(ctx context.Context, spaceId, treeId string) error
}

36
commonspace/settings/deleter.go
View File

@ -2,6 +2,7 @@ package settings
import (
"context"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/object/treemanager"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
@ -23,17 +24,40 @@ func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletion
}
func (d *deleter) Delete() {
allQueued := d.state.GetQueued()
var (
allQueued = d.state.GetQueued()
spaceId = d.st.Id()
)
for _, id := range allQueued {
err := d.getter.DeleteTree(context.Background(), d.st.Id(), id)
if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted {
log.With(zap.String("id", id), zap.Error(err)).Error("failed to delete object")
log := log.With(zap.String("treeId", id), zap.String("spaceId", spaceId))
shouldDelete, err := d.tryMarkDeleted(spaceId, id)
if !shouldDelete {
if err != nil {
log.Error("failed to mark object as deleted", zap.Error(err))
continue
}
} else {
err = d.getter.DeleteTree(context.Background(), spaceId, id)
if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted {
log.Error("failed to delete object", zap.Error(err))
continue
}
}
err = d.state.Delete(id)
if err != nil {
log.With(zap.String("id", id), zap.Error(err)).Error("failed to mark object as deleted")
log.Error("failed to mark object as deleted", zap.Error(err))
}
log.With(zap.String("id", id), zap.Error(err)).Debug("object successfully deleted")
log.Debug("object successfully deleted", zap.Error(err))
}
}
func (d *deleter) tryMarkDeleted(spaceId, treeId string) (bool, error) {
_, err := d.st.TreeStorage(treeId)
if err == nil {
return true, nil
}
if err != treestorage.ErrUnknownTreeId {
return false, err
}
return false, d.getter.MarkTreeDeleted(context.Background(), spaceId, treeId)
}

34
commonspace/settings/deleter_test.go
View File

@ -2,9 +2,9 @@ package settings
import (
"fmt"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage"
"github.com/golang/mock/gomock"
"testing"
@ -18,23 +18,46 @@ func TestDeleter_Delete(t *testing.T) {
deleter := newDeleter(st, delState, treeManager)
t.Run("deleter delete queued", func(t *testing.T) {
t.Run("deleter delete mark deleted success", func(t *testing.T) {
id := "id"
spaceId := "spaceId"
delState.EXPECT().GetQueued().Return([]string{id})
st.EXPECT().Id().Return(spaceId)
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil)
st.EXPECT().TreeStorage(id).Return(nil, treestorage.ErrUnknownTreeId)
treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(nil)
delState.EXPECT().Delete(id).Return(nil)
deleter.Delete()
})
t.Run("deleter delete already deleted", func(t *testing.T) {
t.Run("deleter delete mark deleted other error", func(t *testing.T) {
id := "id"
spaceId := "spaceId"
delState.EXPECT().GetQueued().Return([]string{id})
st.EXPECT().Id().Return(spaceId)
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted)
st.EXPECT().TreeStorage(id).Return(nil, fmt.Errorf("unknown error"))
deleter.Delete()
})
t.Run("deleter delete mark deleted fail", func(t *testing.T) {
id := "id"
spaceId := "spaceId"
delState.EXPECT().GetQueued().Return([]string{id})
st.EXPECT().Id().Return(spaceId)
st.EXPECT().TreeStorage(id).Return(nil, treestorage.ErrUnknownTreeId)
treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(fmt.Errorf("mark error"))
deleter.Delete()
})
//treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted)
t.Run("deleter delete success", func(t *testing.T) {
id := "id"
spaceId := "spaceId"
delState.EXPECT().GetQueued().Return([]string{id})
st.EXPECT().Id().Return(spaceId)
st.EXPECT().TreeStorage(id).Return(nil, nil)
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil)
delState.EXPECT().Delete(id).Return(nil)
deleter.Delete()
@ -45,6 +68,7 @@ func TestDeleter_Delete(t *testing.T) {
spaceId := "spaceId"
delState.EXPECT().GetQueued().Return([]string{id})
st.EXPECT().Id().Return(spaceId)
st.EXPECT().TreeStorage(id).Return(nil, nil)
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error"))
deleter.Delete()

19
commonspace/settings/deletionmanager.go
View File

@ -4,7 +4,6 @@ import (
"context"
"github.com/anytypeio/any-sync/commonspace/object/treemanager"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
"github.com/anytypeio/any-sync/util/slice"
"go.uber.org/zap"
)
@ -47,23 +46,21 @@ type deletionManager struct {
func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate.State) error {
log := log.With(zap.String("spaceId", d.spaceId))
err := d.deletionState.Add(state.DeletedIds)
if err != nil {
log.Debug("failed to add deleted ids to deletion state")
}
d.deletionState.Add(state.DeletedIds)
if state.DeleterId == "" {
return nil
}
// we should delete space
log.Debug("deleting space")
if d.isResponsible {
allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool {
return id == d.settingsId
})
err := d.deletionState.Add(allIds)
if err != nil {
log.Debug("failed to add all ids to deletion state")
mapIds := map[string]struct{}{}
for _, id := range d.provider.AllIds() {
if id != d.settingsId {
mapIds[id] = struct{}{}
}
}
d.deletionState.Add(mapIds)
}
d.onSpaceDelete()
return nil
}

10
commonspace/settings/deletionmanager_test.go
View File

@ -19,7 +19,7 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) {
spaceId := "spaceId"
settingsId := "settingsId"
state := &settingsstate.State{
DeletedIds: []string{"id"},
DeletedIds: map[string]struct{}{"id": {}},
DeleterId: "deleterId",
}
deleted := false
@ -29,7 +29,7 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) {
delState := mock_settingsstate.NewMockObjectDeletionState(ctrl)
treeManager := mock_treemanager.NewMockTreeManager(ctrl)
delState.EXPECT().Add(state.DeletedIds).Return(nil)
delState.EXPECT().Add(state.DeletedIds)
delManager := newDeletionManager(spaceId,
settingsId,
@ -51,7 +51,7 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) {
spaceId := "spaceId"
settingsId := "settingsId"
state := &settingsstate.State{
DeletedIds: []string{"id"},
DeletedIds: map[string]struct{}{"id": struct{}{}},
DeleterId: "deleterId",
}
deleted := false
@ -62,9 +62,9 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) {
treeManager := mock_treemanager.NewMockTreeManager(ctrl)
provider := mock_settings.NewMockSpaceIdsProvider(ctrl)
delState.EXPECT().Add(state.DeletedIds).Return(nil)
delState.EXPECT().Add(state.DeletedIds)
provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId})
delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil)
delState.EXPECT().Add(map[string]struct{}{"id": {}, "otherId": {}})
delManager := newDeletionManager(spaceId,
settingsId,
true,

49
commonspace/settings/settings.go
View File

@ -40,7 +40,16 @@ var (
ErrCantDeleteSpace = errors.New("not able to delete space")
)
var doSnapshot = objecttree.DoSnapshot
var (
DoSnapshot = objecttree.DoSnapshot
buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) {
return objecttree.BuildHistoryTree(objecttree.HistoryTreeParams{
TreeStorage: objTree.Storage(),
AclList: objTree.AclList(),
BuildFullTree: true,
})
}
)
type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error)
@ -166,11 +175,36 @@ func (s *settingsObject) Init(ctx context.Context) (err error) {
if err != nil {
return
}
// TODO: remove this check when everybody updates
if err = s.checkHistoryState(ctx); err != nil {
return
}
s.loop.Run()
return
}
func (s *settingsObject) checkHistoryState(ctx context.Context) (err error) {
historyTree, err := buildHistoryTree(s.SyncTree)
if err != nil {
return
}
fullState, err := s.builder.Build(historyTree, nil)
if err != nil {
return
}
if len(fullState.DeletedIds) != len(s.state.DeletedIds) {
log.WarnCtx(ctx, "state does not have all deleted ids",
zap.Int("fullstate ids", len(fullState.DeletedIds)),
zap.Int("state ids", len(fullState.DeletedIds)))
s.state = fullState
err = s.deletionManager.UpdateState(context.Background(), s.state)
if err != nil {
return
}
}
return
}
func (s *settingsObject) Close() error {
s.loop.Close()
return s.SyncTree.Close()
@ -221,7 +255,7 @@ func (s *settingsObject) DeleteObject(id string) (err error) {
err = ErrDeleteSelf
return
}
if s.deletionState.Exists(id) {
if s.state.Exists(id) {
err = ErrAlreadyDeleted
return nil
}
@ -230,7 +264,7 @@ func (s *settingsObject) DeleteObject(id string) (err error) {
err = ErrObjDoesNotExist
return
}
isSnapshot := doSnapshot(s.Len())
isSnapshot := DoSnapshot(s.Len())
res, err := s.changeFactory.CreateObjectDeleteChange(id, s.state, isSnapshot)
if err != nil {
return
@ -249,7 +283,7 @@ func (s *settingsObject) verifyDeleteSpace(raw *treechangeproto.RawTreeChangeWit
func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) {
accountData := s.account.Account()
_, err = s.AddContent(context.Background(), objecttree.SignableChangeContent{
res, err := s.AddContent(context.Background(), objecttree.SignableChangeContent{
Data: data,
Key: accountData.SignKey,
IsSnapshot: isSnapshot,
@ -258,8 +292,11 @@ func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) {
if err != nil {
return
}
if res.Mode == objecttree.Rebuild {
s.Rebuild(s)
} else {
s.Update(s)
}
return
}

107
commonspace/settings/settings_test.go
View File

@ -5,6 +5,7 @@ import (
"github.com/anytypeio/any-sync/accountservice/mock_accountservice"
"github.com/anytypeio/any-sync/commonspace/object/accountdata"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener"
@ -52,6 +53,7 @@ type settingsFixture struct {
changeFactory *mock_settingsstate.MockChangeFactory
deleter *mock_settings.MockDeleter
syncTree *mock_synctree.MockSyncTree
historyTree *mock_objecttree.MockObjectTree
delState *mock_settingsstate.MockObjectDeletionState
account *mock_accountservice.MockService
}
@ -69,6 +71,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
stateBuilder := mock_settingsstate.NewMockStateBuilder(ctrl)
changeFactory := mock_settingsstate.NewMockChangeFactory(ctrl)
syncTree := mock_synctree.NewMockSyncTree(ctrl)
historyTree := mock_objecttree.NewMockObjectTree(ctrl)
del := mock_settings.NewMockDeleter(ctrl)
delState.EXPECT().AddObserver(gomock.Any())
@ -77,6 +80,9 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
require.Equal(t, objectId, id)
return newTestObjMock(syncTree), nil
})
buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) {
return historyTree, nil
}
deps := Deps{
BuildFunc: buildFunc,
@ -104,45 +110,48 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
syncTree: syncTree,
account: acc,
delState: delState,
historyTree: historyTree,
}
}
func (fx *settingsFixture) stop() {
func (fx *settingsFixture) init(t *testing.T) {
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
fx.stateBuilder.EXPECT().Build(fx.historyTree, nil).Return(&settingsstate.State{}, nil)
fx.doc.state = &settingsstate.State{}
err := fx.doc.Init(context.Background())
require.NoError(t, err)
}
func (fx *settingsFixture) stop(t *testing.T) {
fx.syncTree.EXPECT().Close().Return(nil)
err := fx.doc.Close()
require.NoError(t, err)
fx.ctrl.Finish()
}
func TestSettingsObject_Init(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
fx.syncTree.EXPECT().Close().Return(nil)
err := fx.doc.Init(context.Background())
require.NoError(t, err)
err = fx.doc.Close()
require.NoError(t, err)
fx.init(t)
}
func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
delId := "delId"
doSnapshot = func(len int) bool {
DoSnapshot = func(len int) bool {
return false
}
fx.syncTree.EXPECT().Id().Return("syncId")
fx.syncTree.EXPECT().Len().Return(10)
fx.delState.EXPECT().Exists(delId).Return(false)
fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil)
res := []byte("settingsData")
fx.doc.state = &settingsstate.State{LastIteratedId: "someId"}
@ -162,30 +171,20 @@ func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) {
fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil)
err = fx.doc.DeleteObject(delId)
require.NoError(t, err)
fx.syncTree.EXPECT().Close().Return(nil)
err = fx.doc.Close()
require.NoError(t, err)
}
func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
defer fx.stop(t)
fx.init(t)
delId := "delId"
doSnapshot = func(len int) bool {
DoSnapshot = func(len int) bool {
return true
}
fx.syncTree.EXPECT().Id().Return("syncId")
fx.syncTree.EXPECT().Len().Return(10)
fx.delState.EXPECT().Exists(delId).Return(false)
fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil)
res := []byte("settingsData")
fx.doc.state = &settingsstate.State{LastIteratedId: "someId"}
@ -199,27 +198,19 @@ func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) {
Key: accountData.SignKey,
IsSnapshot: true,
IsEncrypted: false,
}).Return(objecttree.AddResult{}, nil)
}).Return(objecttree.AddResult{Mode: objecttree.Rebuild}, nil)
fx.stateBuilder.EXPECT().Build(fx.doc, fx.doc.state).Return(fx.doc.state, nil)
fx.stateBuilder.EXPECT().Build(fx.doc, nil).Return(fx.doc.state, nil)
fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil)
err = fx.doc.DeleteObject(delId)
require.NoError(t, err)
fx.syncTree.EXPECT().Close().Return(nil)
err = fx.doc.Close()
require.NoError(t, err)
}
func TestSettingsObject_Rebuild(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
newSt := &settingsstate.State{}
@ -232,13 +223,9 @@ func TestSettingsObject_Rebuild(t *testing.T) {
func TestSettingsObject_Update(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
fx.doc.state = &settingsstate.State{}
@ -250,13 +237,9 @@ func TestSettingsObject_Update(t *testing.T) {
func TestSettingsObject_DeleteSpace(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
deleterId := "delId"
@ -275,19 +258,15 @@ func TestSettingsObject_DeleteSpace(t *testing.T) {
Heads: []string{rawCh.Id},
}, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NoError(t, err)
}
func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
t.Run("incorrect change type", func(t *testing.T) {
@ -299,7 +278,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
delChange, _ := changeFactory.CreateObjectDeleteChange("otherId", &settingsstate.State{}, false)
fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NotNil(t, err)
})
@ -312,7 +291,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
delChange, _ := changeFactory.CreateSpaceDeleteChange("", &settingsstate.State{}, false)
fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NotNil(t, err)
})
}

5
commonspace/settings/settingsstate/changefactory.go
View File

@ -50,7 +50,7 @@ func (c *changeFactory) CreateSpaceDeleteChange(peerId string, state *State, isS
func (c *changeFactory) makeSnapshot(state *State, objectId, deleterPeer string) *spacesyncproto.SpaceSettingsSnapshot {
var (
deletedIds = state.DeletedIds
deletedIds = make([]string, 0, len(state.DeletedIds)+1)
deleterId = state.DeleterId
)
if objectId != "" {
@ -59,6 +59,9 @@ func (c *changeFactory) makeSnapshot(state *State, objectId, deleterPeer string)
if deleterPeer != "" {
deleterId = deleterPeer
}
for id := range state.DeletedIds {
deletedIds = append(deletedIds, id)
}
return &spacesyncproto.SpaceSettingsSnapshot{
DeletedIds: deletedIds,
DeleterPeerId: deleterId,

7
commonspace/settings/settingsstate/changefactory_test.go
View File

@ -4,13 +4,14 @@ import (
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
"testing"
)
func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) {
factory := NewChangeFactory()
state := &State{
DeletedIds: []string{"1", "2"},
DeletedIds: map[string]struct{}{"1": {}, "2": {}},
DeleterId: "del",
}
marshalled, err := factory.CreateObjectDeleteChange("3", state, false)
@ -26,6 +27,7 @@ func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) {
data = &spacesyncproto.SettingsData{}
err = proto.Unmarshal(marshalled, data)
require.NoError(t, err)
slices.Sort(data.Snapshot.DeletedIds)
require.Equal(t, &spacesyncproto.SpaceSettingsSnapshot{
DeletedIds: []string{"1", "2", "3"},
DeleterPeerId: "del",
@ -36,7 +38,7 @@ func TestChangeFactory_CreateObjectDeleteChange(t *testing.T) {
func TestChangeFactory_CreateSpaceDeleteChange(t *testing.T) {
factory := NewChangeFactory()
state := &State{
DeletedIds: []string{"1", "2"},
DeletedIds: map[string]struct{}{"1": {}, "2": {}},
}
marshalled, err := factory.CreateSpaceDeleteChange("del", state, false)
require.NoError(t, err)
@ -51,6 +53,7 @@ func TestChangeFactory_CreateSpaceDeleteChange(t *testing.T) {
data = &spacesyncproto.SettingsData{}
err = proto.Unmarshal(marshalled, data)
require.NoError(t, err)
slices.Sort(data.Snapshot.DeletedIds)
require.Equal(t, &spacesyncproto.SpaceSettingsSnapshot{
DeletedIds: []string{"1", "2"},
DeleterPeerId: "del",

32
commonspace/settings/settingsstate/deletionstate.go
View File

@ -2,7 +2,9 @@
package settingsstate
import (
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"go.uber.org/zap"
"sync"
)
@ -10,7 +12,7 @@ type StateUpdateObserver func(ids []string)
type ObjectDeletionState interface {
AddObserver(observer StateUpdateObserver)
Add(ids []string) (err error)
Add(ids map[string]struct{})
GetQueued() (ids []string)
Delete(id string) (err error)
Exists(id string) bool
@ -19,14 +21,16 @@ type ObjectDeletionState interface {
type objectDeletionState struct {
sync.RWMutex
log logger.CtxLogger
queued map[string]struct{}
deleted map[string]struct{}
stateUpdateObservers []StateUpdateObserver
storage spacestorage.SpaceStorage
}
func NewObjectDeletionState(storage spacestorage.SpaceStorage) ObjectDeletionState {
func NewObjectDeletionState(log logger.CtxLogger, storage spacestorage.SpaceStorage) ObjectDeletionState {
return &objectDeletionState{
log: log,
queued: map[string]struct{}{},
deleted: map[string]struct{}{},
storage: storage,
@ -39,19 +43,17 @@ func (st *objectDeletionState) AddObserver(observer StateUpdateObserver) {
st.stateUpdateObservers = append(st.stateUpdateObservers, observer)
}
func (st *objectDeletionState) Add(ids []string) (err error) {
func (st *objectDeletionState) Add(ids map[string]struct{}) {
var added []string
st.Lock()
defer func() {
st.Unlock()
if err != nil {
return
}
for _, ob := range st.stateUpdateObservers {
ob(ids)
ob(added)
}
}()
for _, id := range ids {
for id := range ids {
if _, exists := st.deleted[id]; exists {
continue
}
@ -60,9 +62,10 @@ func (st *objectDeletionState) Add(ids []string) (err error) {
}
var status string
status, err = st.storage.TreeDeletedStatus(id)
status, err := st.storage.TreeDeletedStatus(id)
if err != nil {
return
st.log.Warn("failed to get deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
switch status {
@ -71,14 +74,15 @@ func (st *objectDeletionState) Add(ids []string) (err error) {
case spacestorage.TreeDeletedStatusDeleted:
st.deleted[id] = struct{}{}
default:
st.queued[id] = struct{}{}
err = st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued)
err := st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued)
if err != nil {
return
st.log.Warn("failed to set deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
st.queued[id] = struct{}{}
}
added = append(added, id)
}
return
}
func (st *objectDeletionState) GetQueued() (ids []string) {

15
commonspace/settings/settingsstate/deletionstate_test.go
View File

@ -1,6 +1,7 @@
package settingsstate
import (
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage"
"github.com/golang/mock/gomock"
@ -18,7 +19,7 @@ type fixture struct {
func newFixture(t *testing.T) *fixture {
ctrl := gomock.NewController(t)
spaceStorage := mock_spacestorage.NewMockSpaceStorage(ctrl)
delState := NewObjectDeletionState(spaceStorage).(*objectDeletionState)
delState := NewObjectDeletionState(logger.NewNamed("test"), spaceStorage).(*objectDeletionState)
return &fixture{
ctrl: ctrl,
delState: delState,
@ -37,8 +38,7 @@ func TestDeletionState_Add(t *testing.T) {
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil)
fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add(map[string]struct{}{id: {}})
require.Contains(t, fx.delState.queued, id)
})
@ -47,8 +47,7 @@ func TestDeletionState_Add(t *testing.T) {
defer fx.stop()
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusQueued, nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add(map[string]struct{}{id: {}})
require.Contains(t, fx.delState.queued, id)
})
@ -57,8 +56,7 @@ func TestDeletionState_Add(t *testing.T) {
defer fx.stop()
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusDeleted, nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add(map[string]struct{}{id: {}})
require.Contains(t, fx.delState.deleted, id)
})
}
@ -98,8 +96,7 @@ func TestDeletionState_AddObserver(t *testing.T) {
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil)
fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add(map[string]struct{}{id: {}})
require.Contains(t, fx.delState.queued, id)
require.Equal(t, []string{id}, queued)
}

8
commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go
View File

@ -36,11 +36,9 @@ func (m *MockObjectDeletionState) EXPECT() *MockObjectDeletionStateMockRecorder
}
// Add mocks base method.
func (m *MockObjectDeletionState) Add(arg0 []string) error {
func (m *MockObjectDeletionState) Add(arg0 map[string]struct{}) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Add", arg0)
ret0, _ := ret[0].(error)
return ret0
m.ctrl.Call(m, "Add", arg0)
}
// Add indicates an expected call of Add.
@ -145,7 +143,7 @@ func (m *MockStateBuilder) EXPECT() *MockStateBuilderMockRecorder {
}
// Build mocks base method.
func (m *MockStateBuilder) Build(arg0 objecttree.ObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) {
func (m *MockStateBuilder) Build(arg0 objecttree.ReadableObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Build", arg0, arg1)
ret0, _ := ret[0].(*settingsstate.State)

23
commonspace/settings/settingsstate/settingsstate.go
View File

@ -1,7 +1,28 @@
package settingsstate
import "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
type State struct {
DeletedIds []string
DeletedIds map[string]struct{}
DeleterId string
LastIteratedId string
}
func NewState() *State {
return &State{DeletedIds: map[string]struct{}{}}
}
func NewStateFromSnapshot(snapshot *spacesyncproto.SpaceSettingsSnapshot, lastIteratedId string) *State {
st := NewState()
for _, id := range snapshot.DeletedIds {
st.DeletedIds[id] = struct{}{}
}
st.DeleterId = snapshot.DeleterPeerId
st.LastIteratedId = lastIteratedId
return st
}
func (s *State) Exists(id string) bool {
_, exists := s.DeletedIds[id]
return exists
}

14
commonspace/settings/settingsstate/statebuilder.go
View File

@ -7,7 +7,7 @@ import (
)
type StateBuilder interface {
Build(tree objecttree.ObjectTree, state *State) (*State, error)
Build(tree objecttree.ReadableObjectTree, state *State) (*State, error)
}
func NewStateBuilder() StateBuilder {
@ -17,14 +17,14 @@ func NewStateBuilder() StateBuilder {
type stateBuilder struct {
}
func (s *stateBuilder) Build(tr objecttree.ObjectTree, oldState *State) (state *State, err error) {
func (s *stateBuilder) Build(tr objecttree.ReadableObjectTree, oldState *State) (state *State, err error) {
var (
rootId = tr.Root().Id
startId = rootId
)
state = oldState
if state == nil {
state = &State{}
state = NewState()
} else if state.LastIteratedId != "" {
startId = state.LastIteratedId
}
@ -55,11 +55,7 @@ func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, s
deleteChange := change.Model.(*spacesyncproto.SettingsData)
// getting data from snapshot if we start from it
if change.Id == rootId {
state = &State{
DeletedIds: deleteChange.Snapshot.DeletedIds,
DeleterId: deleteChange.Snapshot.DeleterPeerId,
LastIteratedId: rootId,
}
state = NewStateFromSnapshot(deleteChange.Snapshot, rootId)
return state
}
@ -67,7 +63,7 @@ func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, s
for _, cnt := range deleteChange.Content {
switch {
case cnt.GetObjectDelete() != nil:
state.DeletedIds = append(state.DeletedIds, cnt.GetObjectDelete().GetId())
state.DeletedIds[cnt.GetObjectDelete().GetId()] = struct{}{}
case cnt.GetSpaceDelete() != nil:
state.DeleterId = cnt.GetSpaceDelete().GetDeleterPeerId()
}

16
commonspace/settings/settingsstate/statebuilder_test.go
View File

@ -17,9 +17,9 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
t.Run("empty model", func(t *testing.T) {
ch := &objecttree.Change{}
newSt := sb.processChange(ch, rootId, &State{
DeletedIds: []string{deletedId},
DeletedIds: map[string]struct{}{deletedId: struct{}{}},
})
require.Equal(t, []string{deletedId}, newSt.DeletedIds)
require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds)
})
t.Run("changeId is equal to startId, LastIteratedId is equal to startId", func(t *testing.T) {
@ -34,10 +34,10 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
ch.Id = "startId"
startId := "startId"
newSt := sb.processChange(ch, rootId, &State{
DeletedIds: []string{deletedId},
DeletedIds: map[string]struct{}{deletedId: struct{}{}},
LastIteratedId: startId,
})
require.Equal(t, []string{deletedId}, newSt.DeletedIds)
require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds)
})
t.Run("changeId is equal to rootId", func(t *testing.T) {
@ -50,8 +50,8 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
},
}
ch.Id = "rootId"
newSt := sb.processChange(ch, rootId, &State{})
require.Equal(t, []string{"id1", "id2"}, newSt.DeletedIds)
newSt := sb.processChange(ch, rootId, NewState())
require.Equal(t, map[string]struct{}{"id1": struct{}{}, "id2": struct{}{}}, newSt.DeletedIds)
require.Equal(t, "peerId", newSt.DeleterId)
})
@ -66,8 +66,8 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
},
}
ch.Id = "someId"
newSt := sb.processChange(ch, rootId, &State{})
require.Equal(t, []string{deletedId}, newSt.DeletedIds)
newSt := sb.processChange(ch, rootId, NewState())
require.Equal(t, map[string]struct{}{deletedId: struct{}{}}, newSt.DeletedIds)
})
}

2
commonspace/space.go
View File

@ -195,7 +195,7 @@ func (s *space) Init(ctx context.Context) (err error) {
s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.SyncClient().MessagePool())
s.treeManager.AddObject(s.aclList)
deletionState := settingsstate.NewObjectDeletionState(s.storage)
deletionState := settingsstate.NewObjectDeletionState(log, s.storage)
deps := settings.Deps{
BuildFunc: func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) {
res, err := s.BuildTree(ctx, id, BuildTreeOpts{

60
commonspace/spacestorage/inmemoryprovider.go Normal file
View File

@ -0,0 +1,60 @@
package spacestorage
import (
"context"
"github.com/anytypeio/any-sync/app"
"sync"
)
func NewInMemorySpaceStorageProvider() SpaceStorageProvider {
return &InMemorySpaceStorageProvider{
storages: map[string]SpaceStorage{},
}
}
type InMemorySpaceStorageProvider struct {
storages map[string]SpaceStorage
sync.Mutex
}
func (i *InMemorySpaceStorageProvider) Init(a *app.App) (err error) {
return nil
}
func (i *InMemorySpaceStorageProvider) Name() (name string) {
return CName
}
func (i *InMemorySpaceStorageProvider) WaitSpaceStorage(ctx context.Context, id string) (SpaceStorage, error) {
i.Lock()
defer i.Unlock()
storage, exists := i.storages[id]
if !exists {
return nil, ErrSpaceStorageMissing
}
return storage, nil
}
func (i *InMemorySpaceStorageProvider) SpaceExists(id string) bool {
i.Lock()
defer i.Unlock()
_, exists := i.storages[id]
return exists
}
func (i *InMemorySpaceStorageProvider) CreateSpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) {
i.Lock()
defer i.Unlock()
spaceStorage, err := NewInMemorySpaceStorage(payload)
if err != nil {
return nil, err
}
i.storages[payload.SpaceHeaderWithId.Id] = spaceStorage
return spaceStorage, nil
}
func (i *InMemorySpaceStorageProvider) SetStorage(storage SpaceStorage) {
i.Lock()
defer i.Unlock()
i.storages[storage.Id()] = storage
}

193
commonspace/spacestorage/inmemorystorage.go Normal file
View File

@ -0,0 +1,193 @@
package spacestorage
import (
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/liststorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"sync"
)
type InMemorySpaceStorage struct {
id string
isDeleted bool
spaceSettingsId string
treeDeleted map[string]string
trees map[string]treestorage.TreeStorage
aclStorage liststorage.ListStorage
spaceHeader *spacesyncproto.RawSpaceHeaderWithId
spaceHash string
sync.Mutex
}
func NewInMemorySpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) {
aclStorage, err := liststorage.NewInMemoryAclListStorage(payload.AclWithId.Id, []*aclrecordproto.RawAclRecordWithId{payload.AclWithId})
if err != nil {
return nil, err
}
inMemory := &InMemorySpaceStorage{
id: payload.SpaceHeaderWithId.Id,
spaceSettingsId: payload.SpaceSettingsWithId.Id,
treeDeleted: map[string]string{},
trees: map[string]treestorage.TreeStorage{},
aclStorage: aclStorage,
spaceHeader: payload.SpaceHeaderWithId,
}
_, err = inMemory.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
RootRawChange: payload.SpaceSettingsWithId,
Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId},
Heads: []string{payload.SpaceSettingsWithId.Id},
})
if err != nil {
return nil, err
}
return inMemory, nil
}
func (i *InMemorySpaceStorage) Id() string {
return i.id
}
func (i *InMemorySpaceStorage) SetSpaceDeleted() error {
i.Lock()
defer i.Unlock()
i.isDeleted = true
return nil
}
func (i *InMemorySpaceStorage) IsSpaceDeleted() (bool, error) {
i.Lock()
defer i.Unlock()
return i.isDeleted, nil
}
func (i *InMemorySpaceStorage) SetTreeDeletedStatus(id, state string) error {
i.Lock()
defer i.Unlock()
i.treeDeleted[id] = state
return nil
}
func (i *InMemorySpaceStorage) TreeDeletedStatus(id string) (string, error) {
i.Lock()
defer i.Unlock()
return i.treeDeleted[id], nil
}
func (i *InMemorySpaceStorage) SpaceSettingsId() string {
return i.spaceSettingsId
}
func (i *InMemorySpaceStorage) AclStorage() (liststorage.ListStorage, error) {
return i.aclStorage, nil
}
func (i *InMemorySpaceStorage) SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error) {
return i.spaceHeader, nil
}
func (i *InMemorySpaceStorage) StoredIds() ([]string, error) {
i.Lock()
defer i.Unlock()
var allIds []string
for id := range i.trees {
allIds = append(allIds, id)
}
return allIds, nil
}
func (i *InMemorySpaceStorage) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error) {
i.Lock()
defer i.Unlock()
treeStorage, exists := i.trees[id]
if !exists {
return nil, treestorage.ErrUnknownTreeId
}
return treeStorage.Root()
}
func (i *InMemorySpaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) {
i.Lock()
defer i.Unlock()
treeStorage, exists := i.trees[id]
if !exists {
return nil, treestorage.ErrUnknownTreeId
}
return treeStorage, nil
}
func (i *InMemorySpaceStorage) HasTree(id string) (bool, error) {
i.Lock()
defer i.Unlock()
_, exists := i.trees[id]
return exists, nil
}
func (i *InMemorySpaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) {
i.Lock()
defer i.Unlock()
storage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes)
if err != nil {
return nil, err
}
i.trees[payload.RootRawChange.Id] = storage
return storage, nil
}
func (i *InMemorySpaceStorage) WriteSpaceHash(hash string) error {
i.Lock()
defer i.Unlock()
i.spaceHash = hash
return nil
}
func (i *InMemorySpaceStorage) ReadSpaceHash() (hash string, err error) {
i.Lock()
defer i.Unlock()
return i.spaceHash, nil
}
func (i *InMemorySpaceStorage) Close() error {
return nil
}
func (i *InMemorySpaceStorage) AllTrees() map[string]treestorage.TreeStorage {
i.Lock()
defer i.Unlock()
cp := map[string]treestorage.TreeStorage{}
for id, store := range i.trees {
cp[id] = store
}
return cp
}
func (i *InMemorySpaceStorage) SetTrees(trees map[string]treestorage.TreeStorage) {
i.Lock()
defer i.Unlock()
i.trees = trees
}
func (i *InMemorySpaceStorage) CopyStorage() *InMemorySpaceStorage {
i.Lock()
defer i.Unlock()
copyTreeDeleted := map[string]string{}
for id, status := range i.treeDeleted {
copyTreeDeleted[id] = status
}
copyTrees := map[string]treestorage.TreeStorage{}
for id, store := range i.trees {
copyTrees[id] = store
}
return &InMemorySpaceStorage{
id: i.id,
isDeleted: i.isDeleted,
spaceSettingsId: i.spaceSettingsId,
treeDeleted: copyTreeDeleted,
trees: copyTrees,
aclStorage: i.aclStorage,
spaceHeader: i.spaceHeader,
spaceHash: i.spaceHash,
Mutex: sync.Mutex{},
}
}

321
commonspace/spaceutils_test.go Normal file
View File

@ -0,0 +1,321 @@
package commonspace
import (
"context"
"fmt"
accountService "github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/treemanager"
"github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/net/peer"
"github.com/anytypeio/any-sync/net/pool"
"github.com/anytypeio/any-sync/nodeconf"
"github.com/anytypeio/any-sync/testutil/accounttest"
"github.com/anytypeio/go-chash"
"github.com/stretchr/testify/require"
"testing"
"time"
)
//
// Mock NodeConf implementation
//
type mockConf struct {
id string
networkId string
configuration nodeconf.Configuration
}
func (m *mockConf) NetworkCompatibilityStatus() nodeconf.NetworkCompatibilityStatus {
return nodeconf.NetworkCompatibilityStatusOk
}
func (m *mockConf) Init(a *app.App) (err error) {
accountKeys := a.MustComponent(accountService.CName).(accountService.Service).Account()
networkId := accountKeys.SignKey.GetPublic().Network()
node := nodeconf.Node{
PeerId: accountKeys.PeerId,
Addresses: []string{"127.0.0.1:4430"},
Types: []nodeconf.NodeType{nodeconf.NodeTypeTree},
}
m.id = networkId
m.networkId = networkId
m.configuration = nodeconf.Configuration{
Id: networkId,
NetworkId: networkId,
Nodes: []nodeconf.Node{node},
CreationTime: time.Now(),
}
return nil
}
func (m *mockConf) Name() (name string) {
return nodeconf.CName
}
func (m *mockConf) Run(ctx context.Context) (err error) {
return nil
}
func (m *mockConf) Close(ctx context.Context) (err error) {
return nil
}
func (m *mockConf) Id() string {
return m.id
}
func (m *mockConf) Configuration() nodeconf.Configuration {
return m.configuration
}
func (m *mockConf) NodeIds(spaceId string) []string {
var nodeIds []string
for _, node := range m.configuration.Nodes {
nodeIds = append(nodeIds, node.PeerId)
}
return nodeIds
}
func (m *mockConf) IsResponsible(spaceId string) bool {
return true
}
func (m *mockConf) FilePeers() []string {
return nil
}
func (m *mockConf) ConsensusPeers() []string {
return nil
}
func (m *mockConf) CoordinatorPeers() []string {
return nil
}
func (m *mockConf) PeerAddresses(peerId string) (addrs []string, ok bool) {
if peerId == m.configuration.Nodes[0].PeerId {
return m.configuration.Nodes[0].Addresses, true
}
return nil, false
}
func (m *mockConf) CHash() chash.CHash {
return nil
}
func (m *mockConf) Partition(spaceId string) (part int) {
return 0
}
func (m *mockConf) NodeTypes(nodeId string) []nodeconf.NodeType {
if nodeId == m.configuration.Nodes[0].PeerId {
return m.configuration.Nodes[0].Types
}
return nil
}
//
// Mock PeerManager
//
type mockPeerManager struct {
}
func (p *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) {
return nil
}
func (p *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) {
return nil
}
func (p *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) {
return nil, nil
}
//
// Mock PeerManagerProvider
//
type mockPeerManagerProvider struct {
}
func (m *mockPeerManagerProvider) Init(a *app.App) (err error) {
return nil
}
func (m *mockPeerManagerProvider) Name() (name string) {
return peermanager.CName
}
func (m *mockPeerManagerProvider) NewPeerManager(ctx context.Context, spaceId string) (sm peermanager.PeerManager, err error) {
return &mockPeerManager{}, nil
}
//
// Mock Pool
//
type mockPool struct {
}
func (m *mockPool) Init(a *app.App) (err error) {
return nil
}
func (m *mockPool) Name() (name string) {
return pool.CName
}
func (m *mockPool) Get(ctx context.Context, id string) (peer.Peer, error) {
return nil, fmt.Errorf("no such peer")
}
func (m *mockPool) Dial(ctx context.Context, id string) (peer.Peer, error) {
return nil, fmt.Errorf("can't dial peer")
}
func (m *mockPool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
return nil, fmt.Errorf("can't dial peer")
}
func (m *mockPool) DialOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
return nil, fmt.Errorf("can't dial peer")
}
//
// Mock Config
//
type mockConfig struct {
}
func (m *mockConfig) Init(a *app.App) (err error) {
return nil
}
func (m *mockConfig) Name() (name string) {
return "config"
}
func (m *mockConfig) GetSpace() Config {
return Config{
GCTTL: 60,
SyncPeriod: 20,
KeepTreeDataInMemory: true,
}
}
//
// Mock TreeManager
//
type mockTreeManager struct {
space Space
cache ocache.OCache
deletedIds []string
markedIds []string
}
func (t *mockTreeManager) MarkTreeDeleted(ctx context.Context, spaceId, treeId string) error {
t.markedIds = append(t.markedIds, treeId)
return nil
}
func (t *mockTreeManager) Init(a *app.App) (err error) {
t.cache = ocache.New(func(ctx context.Context, id string) (value ocache.Object, err error) {
return t.space.BuildTree(ctx, id, BuildTreeOpts{})
},
ocache.WithGCPeriod(time.Minute),
ocache.WithTTL(time.Duration(60)*time.Second))
return nil
}
func (t *mockTreeManager) Name() (name string) {
return treemanager.CName
}
func (t *mockTreeManager) Run(ctx context.Context) (err error) {
return nil
}
func (t *mockTreeManager) Close(ctx context.Context) (err error) {
return t.cache.Close()
}
func (t *mockTreeManager) GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) {
val, err := t.cache.Get(ctx, treeId)
if err != nil {
return nil, err
}
return val.(objecttree.ObjectTree), nil
}
func (t *mockTreeManager) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) {
tr, err := t.GetTree(ctx, spaceId, treeId)
if err != nil {
return
}
err = tr.Delete()
if err != nil {
return
}
t.deletedIds = append(t.deletedIds, treeId)
_, err = t.cache.Remove(ctx, treeId)
return nil
}
//
// Space fixture
//
type spaceFixture struct {
app *app.App
config *mockConfig
account accountService.Service
configurationService nodeconf.Service
storageProvider spacestorage.SpaceStorageProvider
peermanagerProvider peermanager.PeerManagerProvider
credentialProvider credentialprovider.CredentialProvider
treeManager *mockTreeManager
pool *mockPool
spaceService SpaceService
cancelFunc context.CancelFunc
}
func newFixture(t *testing.T) *spaceFixture {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
fx := &spaceFixture{
cancelFunc: cancel,
config: &mockConfig{},
app: &app.App{},
account: &accounttest.AccountTestService{},
configurationService: &mockConf{},
storageProvider: spacestorage.NewInMemorySpaceStorageProvider(),
peermanagerProvider: &mockPeerManagerProvider{},
treeManager: &mockTreeManager{},
pool: &mockPool{},
spaceService: New(),
}
fx.app.Register(fx.account).
Register(fx.config).
Register(fx.configurationService).
Register(fx.storageProvider).
Register(fx.peermanagerProvider).
Register(fx.treeManager).
Register(fx.pool).
Register(fx.spaceService)
err := fx.app.Start(ctx)
if err != nil {
fx.cancelFunc()
}
require.NoError(t, err)
return fx
}