From d3acc71d957534881904c8b89d854239b7b3845b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 10 Oct 2022 20:10:47 +0200 Subject: [PATCH] Add some raw prototypes for storage --- common/commonspace/storage/storage.go | 1 + go.mod | 4 +- go.sum | 4 +- node/storage/keys.go | 37 +++++ node/storage/spacestorage.go | 136 +++++++++++++++ node/storage/storageservice.go | 27 +++ node/storage/treestorage.go | 157 ++++++++++++++++++ .../tree/mock_objecttree/mock_objecttree.go | 4 +- 8 files changed, 363 insertions(+), 7 deletions(-) create mode 100644 node/storage/keys.go create mode 100644 node/storage/spacestorage.go create mode 100644 node/storage/storageservice.go create mode 100644 node/storage/treestorage.go diff --git a/common/commonspace/storage/storage.go b/common/commonspace/storage/storage.go index fc1cfa8d..3a98da37 100644 --- a/common/commonspace/storage/storage.go +++ b/common/commonspace/storage/storage.go @@ -12,6 +12,7 @@ import ( const CName = "commonspace.storage" var ErrSpaceStorageExists = errors.New("space storage exists") +var ErrSpaceStorageMissing = errors.New("space storage missing") type SpaceStorage interface { storage.Provider diff --git a/go.mod b/go.mod index c25e1b35..8023dfa2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/anytypeio/go-anytype-infrastructure-experiments go 1.18 require ( + github.com/akrylysov/pogreb v0.10.1 github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab github.com/cespare/xxhash v1.1.0 @@ -61,12 +62,9 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/mod v0.4.2 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.5 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/blake3 v1.1.6 // indirect diff --git a/go.sum b/go.sum index 6805b56c..08295176 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= +github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 h1:kMPPZYmJgbs4AJfodbg2OCXg5cp+9LPAJcLZJqmcghk= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232/go.mod h1:+PeHBAWp7gUh/yw6uAauKc5ku0w4cFNg6DUddGxoGq0= github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab h1:+cdNqtOJWjvepyhxy23G7z7vmpYCoC65AP0nqi1f53s= @@ -167,7 +169,6 @@ golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -208,7 +209,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/node/storage/keys.go b/node/storage/keys.go new file mode 100644 index 00000000..dbfdf385 --- /dev/null +++ b/node/storage/keys.go @@ -0,0 +1,37 @@ +package storage + +import ( + "fmt" + "strings" +) + +type treeKeys struct { + id string +} + +func (t treeKeys) HeadsKey() string { + return fmt.Sprintf("%s/heads", t.id) +} + +func (t treeKeys) RootKey() string { + return fmt.Sprintf("t/%s", t.id) +} + +func (t treeKeys) RawChangeKey(id string) string { + return fmt.Sprintf("%s/%s", t.id, id) +} + +type spaceKeys struct { +} + +func (s spaceKeys) HeaderKey() string { + return "header" +} + +func (s spaceKeys) ACLKey() string { + return "acl" +} + +func isTreeKey(path string) bool { + return strings.HasPrefix(path, "t/") +} diff --git a/node/storage/spacestorage.go b/node/storage/spacestorage.go new file mode 100644 index 00000000..66279f72 --- /dev/null +++ b/node/storage/spacestorage.go @@ -0,0 +1,136 @@ +package storage + +import ( + "github.com/akrylysov/pogreb" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" + spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage" + "github.com/gogo/protobuf/proto" + "path" + "sync" +) + +type spaceStorage struct { + objDb *pogreb.DB + keys spaceKeys + mx sync.Mutex +} + +func newSpaceStorage(rootPath string, spaceId string) (store spacestorage.SpaceStorage, err error) { + dbPath := path.Join(rootPath, spaceId) + objDb, err := pogreb.Open(dbPath, nil) + if err != nil { + return + } + keys := spaceKeys{} + has, err := objDb.Has([]byte(keys.HeaderKey())) + if err != nil { + return + } + if !has { + err = spacestorage.ErrSpaceStorageMissing + return + } + + has, err = objDb.Has([]byte(keys.ACLKey())) + if err != nil { + return + } + if !has { + err = spacestorage.ErrSpaceStorageMissing + return + } + + store = &spaceStorage{ + objDb: objDb, + keys: keys, + } + return +} + +func createSpaceStorage(rootPath string, payload spacestorage.SpaceStorageCreatePayload) (store spacestorage.SpaceStorage, err error) { + dbPath := path.Join(rootPath, payload.Id) + db, err := pogreb.Open(dbPath, nil) + if err != nil { + return + } + + keys := spaceKeys{} + has, err := db.Has([]byte(keys.HeaderKey())) + if err != nil { + return + } + if has { + err = spacestorage.ErrSpaceStorageExists + return + } + + err = db.Put([]byte(payload.RecWithId.Id), payload.RecWithId.Payload) + if err != nil { + return + } + + marshalled, err := payload.SpaceHeader.Marshal() + if err != nil { + return + } + err = db.Put([]byte(payload.Id), marshalled) + if err != nil { + return + } + store = &spaceStorage{ + objDb: db, + keys: keys, + } + return +} + +func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) { + return newTreeStorage(s.objDb, id) +} + +func (s *spaceStorage) CreateTreeStorage(payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) { + s.mx.Lock() + defer s.mx.Unlock() + has, err := s.objDb.Has([]byte(payload.TreeId)) + if err != nil { + return + } + if has { + err = spacestorage.ErrSpaceStorageExists + return + } + return createTreeStorage(s.objDb, payload) +} + +func (s *spaceStorage) ACLStorage() (storage.ListStorage, error) { + return nil, nil +} + +func (s *spaceStorage) SpaceHeader() (header *spacesyncproto.SpaceHeader, err error) { + res, err := s.objDb.Get([]byte(s.keys.HeaderKey())) + if err != nil { + return + } + + header = &spacesyncproto.SpaceHeader{} + err = proto.Unmarshal(res, header) + return +} + +func (s *spaceStorage) StoredIds() (ids []string, err error) { + index := s.objDb.Items() + _, value, err := index.Next() + for err == nil { + strVal := string(value) + if isTreeKey(strVal) { + ids = append(ids, string(value)) + } + _, value, err = index.Next() + } + if err != pogreb.ErrIterationDone { + return + } + err = nil + return +} diff --git a/node/storage/storageservice.go b/node/storage/storageservice.go new file mode 100644 index 00000000..802a4f47 --- /dev/null +++ b/node/storage/storageservice.go @@ -0,0 +1,27 @@ +package storage + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/app" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" +) + +type storageService struct { + rootPath string +} + +func (s *storageService) Init(a *app.App) (err error) { + //TODO implement me + panic("implement me") +} + +func (s *storageService) Name() (name string) { + return storage.CName +} + +func (s *storageService) SpaceStorage(id string) (storage.SpaceStorage, error) { + return newSpaceStorage(s.rootPath, id) +} + +func (s *storageService) CreateSpaceStorage(payload storage.SpaceStorageCreatePayload) (storage.SpaceStorage, error) { + return createSpaceStorage(s.rootPath, payload) +} diff --git a/node/storage/treestorage.go b/node/storage/treestorage.go new file mode 100644 index 00000000..2512f565 --- /dev/null +++ b/node/storage/treestorage.go @@ -0,0 +1,157 @@ +package storage + +import ( + "bytes" + "context" + "github.com/akrylysov/pogreb" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treechangeproto" + "github.com/gogo/protobuf/proto" + "strings" + "sync" +) + +type treeStorage struct { + db *pogreb.DB + path treeKeys + id string + rootPath []byte + headsPath []byte + heads []string + root *treechangeproto.RawTreeChangeWithId + headsMx sync.Mutex +} + +func newTreeStorage(db *pogreb.DB, treeId string) (ts storage.TreeStorage, err error) { + path := treeKeys{treeId} + heads, err := db.Get([]byte(path.HeadsKey())) + if err != nil { + return + } + + res, err := db.Get([]byte(path.RootKey())) + if err != nil { + return + } + + root := &treechangeproto.RawTreeChangeWithId{} + err = proto.Unmarshal(res, root) + if err != nil { + return + } + + ts = &treeStorage{ + db: db, + path: path, + rootPath: []byte(path.RootKey()), + headsPath: []byte(path.HeadsKey()), + id: treeId, + heads: parseHeads(heads), + root: root, + } + return +} + +func createTreeStorage(db *pogreb.DB, payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) { + path := treeKeys{payload.TreeId} + heads := createHeadsPayload(payload.Heads) + + for _, ch := range payload.Changes { + err = db.Put([]byte(path.RawChangeKey(ch.Id)), ch.GetRawChange()) + if err != nil { + return + } + } + + err = db.Put([]byte(path.HeadsKey()), heads) + if err != nil { + return + } + + err = db.Put([]byte(path.RootKey()), payload.RootRawChange.GetRawChange()) + if err != nil { + return + } + + ts = &treeStorage{ + db: db, + path: path, + rootPath: []byte(path.RootKey()), + headsPath: []byte(path.HeadsKey()), + id: payload.TreeId, + heads: payload.Heads, + root: payload.RootRawChange, + } + return +} + +func (t *treeStorage) ID() (string, error) { + return t.id, nil +} + +func (t *treeStorage) Root() (raw *treechangeproto.RawTreeChangeWithId, err error) { + return t.root, nil +} + +func (t *treeStorage) Heads() ([]string, error) { + t.headsMx.Lock() + defer t.headsMx.Unlock() + return t.heads, nil +} + +func (t *treeStorage) SetHeads(heads []string) (err error) { + defer func() { + if err == nil { + t.headsMx.Lock() + t.heads = heads + t.headsMx.Unlock() + } + }() + payload := createHeadsPayload(heads) + return t.db.Put(t.headsPath, payload) +} + +func (t *treeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) { + return t.db.Put([]byte(change.Id), change.RawChange) +} + +func (t *treeStorage) GetRawChange(ctx context.Context, id string) (raw *treechangeproto.RawTreeChangeWithId, err error) { + res, err := t.db.Get([]byte(t.path.RawChangeKey(id))) + if err != nil { + return + } + + raw = &treechangeproto.RawTreeChangeWithId{ + RawChange: res, + Id: id, + } + return +} + +func (t *treeStorage) HasChange(ctx context.Context, id string) (bool, error) { + return t.db.Has([]byte(id)) +} + +func parseHeads(headsPayload []byte) []string { + return strings.Split(string(headsPayload), "/") +} + +func createHeadsPayload(heads []string) []byte { + var ( + b bytes.Buffer + totalLen int + ) + for _, s := range heads { + totalLen += len(s) + } + // adding separators + totalLen += len(heads) - 1 + b.Grow(totalLen) + for idx, s := range heads { + if idx > 0 { + b.WriteString("/") + } + b.WriteString(s) + } + return b.Bytes() +} diff --git a/pkg/acl/tree/mock_objecttree/mock_objecttree.go b/pkg/acl/tree/mock_objecttree/mock_objecttree.go index 70afa9d4..b739900c 100644 --- a/pkg/acl/tree/mock_objecttree/mock_objecttree.go +++ b/pkg/acl/tree/mock_objecttree/mock_objecttree.go @@ -137,7 +137,7 @@ func (mr *MockObjectTreeMockRecorder) HasChanges(arg0 ...interface{}) *gomock.Ca // Header mocks base method. func (m *MockObjectTree) Header() *treechangeproto.RawTreeChangeWithId { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Header") + ret := m.ctrl.Call(m, "HeaderKey") ret0, _ := ret[0].(*treechangeproto.RawTreeChangeWithId) return ret0 } @@ -145,7 +145,7 @@ func (m *MockObjectTree) Header() *treechangeproto.RawTreeChangeWithId { // Header indicates an expected call of Header. func (mr *MockObjectTreeMockRecorder) Header() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockObjectTree)(nil).Header)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeaderKey", reflect.TypeOf((*MockObjectTree)(nil).Header)) } // Heads mocks base method.