Add some raw prototypes for storage

common/commonspace/storage/storage.go

@@ -12,6 +12,7 @@ import (
 const CName = "commonspace.storage"
 
 var ErrSpaceStorageExists = errors.New("space storage exists")
+var ErrSpaceStorageMissing = errors.New("space storage missing")
 
 type SpaceStorage interface {
 	storage.Provider

go.mod

@@ -3,6 +3,7 @@ module github.com/anytypeio/go-anytype-infrastructure-experiments
 go 1.18
 
 require (
+	github.com/akrylysov/pogreb v0.10.1
 	github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232
 	github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab
 	github.com/cespare/xxhash v1.1.0
@@ -61,12 +62,9 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
-	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.5 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/blake3 v1.1.6 // indirect

go.sum

@@ -1,6 +1,8 @@
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
+github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 h1:kMPPZYmJgbs4AJfodbg2OCXg5cp+9LPAJcLZJqmcghk=
 github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232/go.mod h1:+PeHBAWp7gUh/yw6uAauKc5ku0w4cFNg6DUddGxoGq0=
 github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab h1:+cdNqtOJWjvepyhxy23G7z7vmpYCoC65AP0nqi1f53s=
@@ -167,7 +169,6 @@ golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+o
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -208,7 +209,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

node/storage/keys.go

@@ -0,0 +1,37 @@
+package storage
+
+import (
+	"fmt"
+	"strings"
+)
+
+type treeKeys struct {
+	id string
+}
+
+func (t treeKeys) HeadsKey() string {
+	return fmt.Sprintf("%s/heads", t.id)
+}
+
+func (t treeKeys) RootKey() string {
+	return fmt.Sprintf("t/%s", t.id)
+}
+
+func (t treeKeys) RawChangeKey(id string) string {
+	return fmt.Sprintf("%s/%s", t.id, id)
+}
+
+type spaceKeys struct {
+}
+
+func (s spaceKeys) HeaderKey() string {
+	return "header"
+}
+
+func (s spaceKeys) ACLKey() string {
+	return "acl"
+}
+
+func isTreeKey(path string) bool {
+	return strings.HasPrefix(path, "t/")
+}

node/storage/spacestorage.go

@@ -0,0 +1,136 @@
+package storage
+
+import (
+	"github.com/akrylysov/pogreb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto"
+	spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
+	"github.com/gogo/protobuf/proto"
+	"path"
+	"sync"
+)
+
+type spaceStorage struct {
+	objDb *pogreb.DB
+	keys  spaceKeys
+	mx    sync.Mutex
+}
+
+func newSpaceStorage(rootPath string, spaceId string) (store spacestorage.SpaceStorage, err error) {
+	dbPath := path.Join(rootPath, spaceId)
+	objDb, err := pogreb.Open(dbPath, nil)
+	if err != nil {
+		return
+	}
+	keys := spaceKeys{}
+	has, err := objDb.Has([]byte(keys.HeaderKey()))
+	if err != nil {
+		return
+	}
+	if !has {
+		err = spacestorage.ErrSpaceStorageMissing
+		return
+	}
+
+	has, err = objDb.Has([]byte(keys.ACLKey()))
+	if err != nil {
+		return
+	}
+	if !has {
+		err = spacestorage.ErrSpaceStorageMissing
+		return
+	}
+
+	store = &spaceStorage{
+		objDb: objDb,
+		keys:  keys,
+	}
+	return
+}
+
+func createSpaceStorage(rootPath string, payload spacestorage.SpaceStorageCreatePayload) (store spacestorage.SpaceStorage, err error) {
+	dbPath := path.Join(rootPath, payload.Id)
+	db, err := pogreb.Open(dbPath, nil)
+	if err != nil {
+		return
+	}
+
+	keys := spaceKeys{}
+	has, err := db.Has([]byte(keys.HeaderKey()))
+	if err != nil {
+		return
+	}
+	if has {
+		err = spacestorage.ErrSpaceStorageExists
+		return
+	}
+
+	err = db.Put([]byte(payload.RecWithId.Id), payload.RecWithId.Payload)
+	if err != nil {
+		return
+	}
+
+	marshalled, err := payload.SpaceHeader.Marshal()
+	if err != nil {
+		return
+	}
+	err = db.Put([]byte(payload.Id), marshalled)
+	if err != nil {
+		return
+	}
+	store = &spaceStorage{
+		objDb: db,
+		keys:  keys,
+	}
+	return
+}
+
+func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) {
+	return newTreeStorage(s.objDb, id)
+}
+
+func (s *spaceStorage) CreateTreeStorage(payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+	has, err := s.objDb.Has([]byte(payload.TreeId))
+	if err != nil {
+		return
+	}
+	if has {
+		err = spacestorage.ErrSpaceStorageExists
+		return
+	}
+	return createTreeStorage(s.objDb, payload)
+}
+
+func (s *spaceStorage) ACLStorage() (storage.ListStorage, error) {
+	return nil, nil
+}
+
+func (s *spaceStorage) SpaceHeader() (header *spacesyncproto.SpaceHeader, err error) {
+	res, err := s.objDb.Get([]byte(s.keys.HeaderKey()))
+	if err != nil {
+		return
+	}
+
+	header = &spacesyncproto.SpaceHeader{}
+	err = proto.Unmarshal(res, header)
+	return
+}
+
+func (s *spaceStorage) StoredIds() (ids []string, err error) {
+	index := s.objDb.Items()
+	_, value, err := index.Next()
+	for err == nil {
+		strVal := string(value)
+		if isTreeKey(strVal) {
+			ids = append(ids, string(value))
+		}
+		_, value, err = index.Next()
+	}
+	if err != pogreb.ErrIterationDone {
+		return
+	}
+	err = nil
+	return
+}

node/storage/storageservice.go

@@ -0,0 +1,27 @@
+package storage
+
+import (
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage"
+)
+
+type storageService struct {
+	rootPath string
+}
+
+func (s *storageService) Init(a *app.App) (err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (s *storageService) Name() (name string) {
+	return storage.CName
+}
+
+func (s *storageService) SpaceStorage(id string) (storage.SpaceStorage, error) {
+	return newSpaceStorage(s.rootPath, id)
+}
+
+func (s *storageService) CreateSpaceStorage(payload storage.SpaceStorageCreatePayload) (storage.SpaceStorage, error) {
+	return createSpaceStorage(s.rootPath, payload)
+}

node/storage/treestorage.go

@@ -0,0 +1,157 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"github.com/akrylysov/pogreb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treechangeproto"
+	"github.com/gogo/protobuf/proto"
+	"strings"
+	"sync"
+)
+
+type treeStorage struct {
+	db        *pogreb.DB
+	path      treeKeys
+	id        string
+	rootPath  []byte
+	headsPath []byte
+	heads     []string
+	root      *treechangeproto.RawTreeChangeWithId
+	headsMx   sync.Mutex
+}
+
+func newTreeStorage(db *pogreb.DB, treeId string) (ts storage.TreeStorage, err error) {
+	path := treeKeys{treeId}
+	heads, err := db.Get([]byte(path.HeadsKey()))
+	if err != nil {
+		return
+	}
+
+	res, err := db.Get([]byte(path.RootKey()))
+	if err != nil {
+		return
+	}
+
+	root := &treechangeproto.RawTreeChangeWithId{}
+	err = proto.Unmarshal(res, root)
+	if err != nil {
+		return
+	}
+
+	ts = &treeStorage{
+		db:        db,
+		path:      path,
+		rootPath:  []byte(path.RootKey()),
+		headsPath: []byte(path.HeadsKey()),
+		id:        treeId,
+		heads:     parseHeads(heads),
+		root:      root,
+	}
+	return
+}
+
+func createTreeStorage(db *pogreb.DB, payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) {
+	path := treeKeys{payload.TreeId}
+	heads := createHeadsPayload(payload.Heads)
+
+	for _, ch := range payload.Changes {
+		err = db.Put([]byte(path.RawChangeKey(ch.Id)), ch.GetRawChange())
+		if err != nil {
+			return
+		}
+	}
+
+	err = db.Put([]byte(path.HeadsKey()), heads)
+	if err != nil {
+		return
+	}
+
+	err = db.Put([]byte(path.RootKey()), payload.RootRawChange.GetRawChange())
+	if err != nil {
+		return
+	}
+
+	ts = &treeStorage{
+		db:        db,
+		path:      path,
+		rootPath:  []byte(path.RootKey()),
+		headsPath: []byte(path.HeadsKey()),
+		id:        payload.TreeId,
+		heads:     payload.Heads,
+		root:      payload.RootRawChange,
+	}
+	return
+}
+
+func (t *treeStorage) ID() (string, error) {
+	return t.id, nil
+}
+
+func (t *treeStorage) Root() (raw *treechangeproto.RawTreeChangeWithId, err error) {
+	return t.root, nil
+}
+
+func (t *treeStorage) Heads() ([]string, error) {
+	t.headsMx.Lock()
+	defer t.headsMx.Unlock()
+	return t.heads, nil
+}
+
+func (t *treeStorage) SetHeads(heads []string) (err error) {
+	defer func() {
+		if err == nil {
+			t.headsMx.Lock()
+			t.heads = heads
+			t.headsMx.Unlock()
+		}
+	}()
+	payload := createHeadsPayload(heads)
+	return t.db.Put(t.headsPath, payload)
+}
+
+func (t *treeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) {
+	return t.db.Put([]byte(change.Id), change.RawChange)
+}
+
+func (t *treeStorage) GetRawChange(ctx context.Context, id string) (raw *treechangeproto.RawTreeChangeWithId, err error) {
+	res, err := t.db.Get([]byte(t.path.RawChangeKey(id)))
+	if err != nil {
+		return
+	}
+
+	raw = &treechangeproto.RawTreeChangeWithId{
+		RawChange: res,
+		Id:        id,
+	}
+	return
+}
+
+func (t *treeStorage) HasChange(ctx context.Context, id string) (bool, error) {
+	return t.db.Has([]byte(id))
+}
+
+func parseHeads(headsPayload []byte) []string {
+	return strings.Split(string(headsPayload), "/")
+}
+
+func createHeadsPayload(heads []string) []byte {
+	var (
+		b        bytes.Buffer
+		totalLen int
+	)
+	for _, s := range heads {
+		totalLen += len(s)
+	}
+	// adding separators
+	totalLen += len(heads) - 1
+	b.Grow(totalLen)
+	for idx, s := range heads {
+		if idx > 0 {
+			b.WriteString("/")
+		}
+		b.WriteString(s)
+	}
+	return b.Bytes()
+}

pkg/acl/tree/mock_objecttree/mock_objecttree.go

@@ -137,7 +137,7 @@ func (mr *MockObjectTreeMockRecorder) HasChanges(arg0 ...interface{}) *gomock.Ca
 // Header mocks base method.
 func (m *MockObjectTree) Header() *treechangeproto.RawTreeChangeWithId {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Header")
+	ret := m.ctrl.Call(m, "HeaderKey")
 	ret0, _ := ret[0].(*treechangeproto.RawTreeChangeWithId)
 	return ret0
 }
@@ -145,7 +145,7 @@ func (m *MockObjectTree) Header() *treechangeproto.RawTreeChangeWithId {
 // Header indicates an expected call of Header.
 func (mr *MockObjectTreeMockRecorder) Header() *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockObjectTree)(nil).Header))
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeaderKey", reflect.TypeOf((*MockObjectTree)(nil).Header))
 }
 
 // Heads mocks base method.