benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create ACL list and update tree package

Browse Source
This commit is contained in:
mcrakhman 2022-08-18 23:03:27 +02:00 committed by Mikhail Iudin
parent 38209fc1fb
commit 70184e1554
No known key found for this signature in database
GPG Key ID: FAAAA8BAABDFF1C0
14 changed files with 984 additions and 866 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

808
pkg/acl/aclchanges/aclpb/aclchanges.pb.go
View File

File diff suppressed because it is too large Load Diff

16
pkg/acl/aclchanges/aclpb/protos/aclchanges.proto
View File

@ -8,6 +8,12 @@ message RawChange {
string id = 3;
}
message RawRecord {
bytes payload = 1;
bytes signature = 2;
string id = 3;
}
// the element of change tree used to store and internal apply smartBlock history
message ACLChange {
repeated string treeHeadIds = 1;
@ -112,7 +118,7 @@ message ACLChange {
message Change {
repeated string treeHeadIds = 1;
repeated string aclHeadIds = 2;
string aclHeadId = 2;
string snapshotBaseId = 3; // we will only have one base snapshot for both
bytes changesData = 4;
uint64 currentReadKeyHash = 5;
@ -120,3 +126,11 @@ message Change {
string identity = 7;
bool isSnapshot = 8;
}
message Record {
string prevId = 1;
string identity = 2;
bytes data = 3;
uint64 currentReadKeyHash = 4;
int64 timestamp = 5;
}

47
pkg/acl/tree/aclstate.go → pkg/acl/list/aclstate.go
View File

@ -1,17 +1,22 @@
package tree
package list
import (
"bytes"
"errors"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
"github.com/gogo/protobuf/proto"
"go.uber.org/zap"
"hash/fnv"
)
var log = logger.NewNamed("acllist")
var ErrNoSuchUser = errors.New("no such user")
var ErrFailedToDecrypt = errors.New("failed to decrypt key")
var ErrUserRemoved = errors.New("user was removed from the document")
@ -23,7 +28,7 @@ type ACLState struct {
userReadKeys map[uint64]*symmetric.Key
userStates map[string]*aclpb.ACLChangeUserState
userInvites map[string]*aclpb.ACLChangeUserInvite
signingPubKeyDecoder signingkey.PubKeyDecoder
signingPubKeyDecoder keys.Decoder
encryptionKey encryptionkey.PrivKey
identity string
}
@ -31,14 +36,14 @@ type ACLState struct {
func newACLStateWithIdentity(
identity string,
encryptionKey encryptionkey.PrivKey,
signingPubKeyDecoder signingkey.PubKeyDecoder) *ACLState {
decoder keys.Decoder) *ACLState {
return &ACLState{
identity: identity,
encryptionKey: encryptionKey,
userReadKeys: make(map[uint64]*symmetric.Key),
userStates: make(map[string]*aclpb.ACLChangeUserState),
userInvites: make(map[string]*aclpb.ACLChangeUserInvite),
signingPubKeyDecoder: signingPubKeyDecoder,
signingPubKeyDecoder: decoder,
}
}
@ -50,10 +55,18 @@ func newACLState() *ACLState {
}
}
func (st *ACLState) applyChange(change *aclpb.Change) (err error) {
func (st *ACLState) CurrentReadKeyHash() uint64 {
return st.currentReadKeyHash
}
func (st *ACLState) UserReadKeys() map[uint64]*symmetric.Key {
return st.userReadKeys
}
func (st *ACLState) applyRecord(record *aclpb.Record) (err error) {
aclData := &aclpb.ACLChangeACLData{}
err = proto.Unmarshal(change.ChangesData, aclData)
err = proto.Unmarshal(record.Data, aclData)
if err != nil {
return
}
@ -62,27 +75,27 @@ func (st *ACLState) applyChange(change *aclpb.Change) (err error) {
if err != nil {
return
}
st.currentReadKeyHash = change.CurrentReadKeyHash
st.currentReadKeyHash = record.CurrentReadKeyHash
}()
return st.applyChangeData(aclData, change.CurrentReadKeyHash, change.Identity)
return st.applyChangeData(aclData, record.CurrentReadKeyHash, record.Identity)
}
func (st *ACLState) applyChangeAndUpdate(changeWrapper *Change) (err error) {
change := changeWrapper.Content
func (st *ACLState) applyChangeAndUpdate(recordWrapper *Record) (err error) {
change := recordWrapper.Content
aclData := &aclpb.ACLChangeACLData{}
if changeWrapper.ParsedModel != nil {
aclData = changeWrapper.ParsedModel.(*aclpb.ACLChangeACLData)
if recordWrapper.ParsedModel != nil {
aclData = recordWrapper.ParsedModel.(*aclpb.ACLChangeACLData)
} else {
err = proto.Unmarshal(change.ChangesData, aclData)
err = proto.Unmarshal(change.Data, aclData)
if err != nil {
return
}
changeWrapper.ParsedModel = aclData
recordWrapper.ParsedModel = aclData
}
return st.applyChangeData(aclData, changeWrapper.Content.CurrentReadKeyHash, changeWrapper.Content.Identity)
return st.applyChangeData(aclData, recordWrapper.Content.CurrentReadKeyHash, recordWrapper.Content.Identity)
}
func (st *ACLState) applyChangeData(changeData *aclpb.ACLChangeACLData, hash uint64, identity string) (err error) {
@ -111,7 +124,7 @@ func (st *ACLState) applyChangeData(changeData *aclpb.ACLChangeACLData, hash uin
for _, ch := range changeData.GetAclContent() {
if err = st.applyChangeContent(ch); err != nil {
log.Infof("error while applying changes: %v; ignore", err)
log.Info("error while applying changes: %v; ignore", zap.Error(err))
return err
}
}
@ -174,7 +187,7 @@ func (st *ACLState) applyUserJoin(ch *aclpb.ACLChangeUserJoin) error {
return fmt.Errorf("failed to decode signing identity as bytes")
}
res, err := verificationKey.Verify(rawSignedId, signature)
res, err := verificationKey.(signingkey.PubKey).Verify(rawSignedId, signature)
if err != nil {
return fmt.Errorf("verification returned error: %w", err)
}

51
pkg/acl/list/aclstatebuilder.go Normal file
View File

@ -0,0 +1,51 @@
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
)
type aclStateBuilder struct {
log ACLList
identity string
key encryptionkey.PrivKey
decoder keys.Decoder
}
func newACLStateBuilderWithIdentity(decoder keys.Decoder, accountData *account.AccountData) *aclStateBuilder {
return &aclStateBuilder{
decoder: decoder,
identity: accountData.Identity,
key: accountData.EncKey,
}
}
func newACLStateBuilder() *aclStateBuilder {
return &aclStateBuilder{}
}
func (sb *aclStateBuilder) Init(aclLog ACLList) error {
sb.log = aclLog
return nil
}
func (sb *aclStateBuilder) Build() (*ACLState, error) {
var (
err error
state *ACLState
)
if sb.decoder != nil {
state = newACLStateWithIdentity(sb.identity, sb.key, sb.decoder)
} else {
state = newACLState()
}
sb.log.Iterate(func(c *Record) (isContinue bool) {
err = state.applyChangeAndUpdate(c)
return err == nil
})
return state, err
}

27
pkg/acl/tree/changebuilder.go → pkg/acl/list/changebuilder.go
View File

@ -1,4 +1,4 @@
package tree
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
@ -20,7 +20,7 @@ type ACLChangeBuilder interface {
type aclChangeBuilder struct {
aclState *ACLState
tree *Tree
list ACLList
acc *account.AccountData
aclData *aclpb.ACLChangeACLData
@ -33,9 +33,9 @@ func newACLChangeBuilder() *aclChangeBuilder {
return &aclChangeBuilder{}
}
func (c *aclChangeBuilder) Init(state *ACLState, tree *Tree, acc *account.AccountData) {
func (c *aclChangeBuilder) Init(state *ACLState, list ACLList, acc *account.AccountData) {
c.aclState = state
c.tree = tree
c.list = list
c.acc = acc
c.aclData = &aclpb.ACLChangeACLData{}
@ -93,30 +93,25 @@ func (c *aclChangeBuilder) UserAdd(identity string, encryptionKey encryptionkey.
return nil
}
func (c *aclChangeBuilder) BuildAndApply() (*Change, []byte, error) {
aclChange := &aclpb.Change{
TreeHeadIds: c.tree.Heads(),
SnapshotBaseId: c.tree.RootId(),
func (c *aclChangeBuilder) BuildAndApply() (*Record, []byte, error) {
aclRecord := &aclpb.Record{
PrevId: c.list.Last().Id,
CurrentReadKeyHash: c.readKeyHash,
Timestamp: int64(time.Now().Nanosecond()),
Identity: c.acc.Identity,
}
if c.aclState.currentReadKeyHash == 0 {
// setting IsSnapshot for initial change
aclChange.IsSnapshot = true
}
marshalledData, err := proto.Marshal(c.aclData)
if err != nil {
return nil, nil, err
}
aclChange.ChangesData = marshalledData
err = c.aclState.applyChange(aclChange)
aclRecord.Data = marshalledData
err = c.aclState.applyRecord(aclRecord)
if err != nil {
return nil, nil, err
}
fullMarshalledChange, err := proto.Marshal(aclChange)
fullMarshalledChange, err := proto.Marshal(aclRecord)
if err != nil {
return nil, nil, err
}
@ -128,7 +123,7 @@ func (c *aclChangeBuilder) BuildAndApply() (*Change, []byte, error) {
if err != nil {
return nil, nil, err
}
ch := NewChange(id, aclChange)
ch := NewRecord(id, aclRecord)
ch.ParsedModel = c.aclData
ch.Sign = signature

16
pkg/acl/list/list.go Normal file
View File

@ -0,0 +1,16 @@
package list
import "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
type IterFunc = func(record *Record) (IsContinue bool)
type ACLList interface {
tree.RWLocker
ID() string
ACLState() ACLState
IsAfter(first string, second string) (bool, error)
Last() *Record
Get(id string) (*Record, error)
Iterate(iterFunc IterFunc)
IterateFrom(startId string, iterFunc IterFunc)
}

17
pkg/acl/list/record.go Normal file
View File

@ -0,0 +1,17 @@
package list
import "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
type Record struct {
Id string
Content *aclpb.Record
ParsedModel interface{}
Sign []byte
}
func NewRecord(id string, aclRecord *aclpb.Record) *Record {
return &Record{
Id: id,
Content: aclRecord,
}
}

102
pkg/acl/tree/aclstatebuilder.go
View File

@ -1,102 +0,0 @@
package tree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
)
type aclStateBuilder struct {
tree *Tree
identity string
key encryptionkey.PrivKey
decoder signingkey.PubKeyDecoder
}
func newACLStateBuilderWithIdentity(decoder signingkey.PubKeyDecoder, accountData *account.AccountData) *aclStateBuilder {
return &aclStateBuilder{
decoder: decoder,
identity: accountData.Identity,
key: accountData.EncKey,
}
}
func newACLStateBuilder() *aclStateBuilder {
return &aclStateBuilder{}
}
func (sb *aclStateBuilder) Init(tree *Tree) error {
sb.tree = tree
return nil
}
func (sb *aclStateBuilder) Build() (*ACLState, error) {
state, _, err := sb.BuildBefore("")
return state, err
}
func (sb *aclStateBuilder) BuildBefore(beforeId string) (*ACLState, bool, error) {
var (
err error
startChange = sb.tree.root
state *ACLState
foundId = false
)
if sb.decoder != nil {
state = newACLStateWithIdentity(sb.identity, sb.key, sb.decoder)
} else {
state = newACLState()
}
if beforeId == startChange.Id {
return state, true, nil
}
iterFunc := func(c *Change) (isContinue bool) {
defer func() {
if err == nil {
startChange = c
} else if err != ErrDocumentForbidden {
log.Errorf("marking change %s as invalid: %v", c.Id, err)
sb.tree.RemoveInvalidChange(c.Id)
}
}()
err = state.applyChangeAndUpdate(c)
if err != nil {
return false
}
// the user can't make changes
if !state.hasPermission(c.Content.Identity, aclpb.ACLChange_Writer) && !state.hasPermission(c.Content.Identity, aclpb.ACLChange_Admin) {
err = fmt.Errorf("user %s cannot make changes", c.Content.Identity)
return false
}
if c.Id == beforeId {
foundId = true
return false
}
return true
}
for {
sb.tree.IterateSkip(sb.tree.root.Id, startChange.Id, iterFunc)
if err == nil {
break
}
// the user is forbidden to access the document
if err == ErrDocumentForbidden {
return nil, foundId, err
}
// otherwise we have to continue from the change which we had
err = nil
}
return state, foundId, err
}

525
pkg/acl/tree/acltree.go
View File

@ -1,525 +0,0 @@
package tree
import (
"context"
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"go.uber.org/zap"
"sync"
)
type AddResultSummary int
var ErrTreeWithoutIdentity = errors.New("acl tree is created without identity")
var ErrHasInvalidChanges = errors.New("the change is invalid")
const (
AddResultSummaryNothing AddResultSummary = iota
AddResultSummaryAppend
AddResultSummaryRebuild
)
type AddResult struct {
OldHeads []string
Heads []string
Added []*aclpb.RawChange
// TODO: add summary for changes
Summary AddResultSummary
}
type ACLTreeUpdateListener interface {
Update(tree ACLTree)
Rebuild(tree ACLTree)
}
type NoOpListener struct{}
func (n NoOpListener) Update(tree ACLTree) {}
func (n NoOpListener) Rebuild(tree ACLTree) {}
type RWLocker interface {
sync.Locker
RLock()
RUnlock()
}
var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot")
type ACLTree interface {
RWLocker
CommonTree
ACLState() *ACLState
AddContent(ctx context.Context, f func(builder ACLChangeBuilder) error) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, changes ...*aclpb.RawChange) (AddResult, error)
}
type aclTree struct {
treeStorage treestorage.TreeStorage
accountData *account.AccountData
updateListener ACLTreeUpdateListener
id string
header *treepb.TreeHeader
tree *Tree
aclState *ACLState
treeBuilder *treeBuilder
aclStateBuilder *aclStateBuilder
changeBuilder *aclChangeBuilder
sync.RWMutex
}
func BuildACLTreeWithIdentity(t treestorage.TreeStorage, acc *account.AccountData, listener ACLTreeUpdateListener) (ACLTree, error) {
treeBuilder := newTreeBuilder(t, acc.Decoder)
aclStateBuilder := newACLStateBuilderWithIdentity(acc.Decoder, acc)
changeBuilder := newACLChangeBuilder()
aclTree := &aclTree{
treeStorage: t,
accountData: acc,
tree: nil,
aclState: nil,
treeBuilder: treeBuilder,
aclStateBuilder: aclStateBuilder,
changeBuilder: changeBuilder,
updateListener: listener,
}
err := aclTree.rebuildFromStorage()
if err != nil {
return nil, err
}
err = aclTree.removeOrphans()
if err != nil {
return nil, err
}
err = t.SetHeads(aclTree.Heads())
if err != nil {
return nil, err
}
aclTree.id, err = t.TreeID()
if err != nil {
return nil, err
}
aclTree.header, err = t.Header()
if err != nil {
return nil, err
}
if listener != nil {
listener.Rebuild(aclTree)
}
return aclTree, nil
}
func BuildACLTree(t treestorage.TreeStorage, decoder signingkey.PubKeyDecoder, listener ACLTreeUpdateListener) (ACLTree, error) {
treeBuilder := newTreeBuilder(t, decoder)
aclStateBuilder := newACLStateBuilder()
changeBuilder := newACLChangeBuilder()
aclTree := &aclTree{
treeStorage: t,
tree: nil,
aclState: nil,
treeBuilder: treeBuilder,
aclStateBuilder: aclStateBuilder,
changeBuilder: changeBuilder,
updateListener: listener,
}
err := aclTree.rebuildFromStorage()
if err != nil {
return nil, err
}
err = aclTree.removeOrphans()
if err != nil {
return nil, err
}
err = t.SetHeads(aclTree.Heads())
if err != nil {
return nil, err
}
aclTree.id, err = t.TreeID()
if err != nil {
return nil, err
}
aclTree.header, err = t.Header()
if err != nil {
return nil, err
}
if listener != nil {
listener.Rebuild(aclTree)
}
return aclTree, nil
}
func (a *aclTree) removeOrphans() error {
// removing attached or invalid orphans
var toRemove []string
orphans, err := a.treeStorage.Orphans()
if err != nil {
return err
}
for _, orphan := range orphans {
if _, exists := a.tree.attached[orphan]; exists {
toRemove = append(toRemove, orphan)
}
if _, exists := a.tree.invalidChanges[orphan]; exists {
toRemove = append(toRemove, orphan)
}
}
return a.treeStorage.RemoveOrphans(toRemove...)
}
func (a *aclTree) rebuildFromStorage() (err error) {
a.treeBuilder.Init()
a.tree, err = a.treeBuilder.Build(true)
if err != nil {
return err
}
err = a.aclStateBuilder.Init(a.tree)
if err != nil {
return err
}
a.aclState, err = a.aclStateBuilder.Build()
if err != nil {
return err
}
return nil
}
func (a *aclTree) ID() string {
return a.id
}
func (a *aclTree) Header() *treepb.TreeHeader {
return a.header
}
func (a *aclTree) ACLState() *ACLState {
return a.aclState
}
func (a *aclTree) Storage() treestorage.TreeStorage {
return a.treeStorage
}
func (a *aclTree) AddContent(ctx context.Context, build func(builder ACLChangeBuilder) error) (*aclpb.RawChange, error) {
if a.accountData == nil {
return nil, ErrTreeWithoutIdentity
}
defer func() {
// TODO: should this be called in a separate goroutine to prevent accidental cycles (tree->updater->tree)
if a.updateListener != nil {
a.updateListener.Update(a)
}
}()
a.changeBuilder.Init(a.aclState, a.tree, a.accountData)
err := build(a.changeBuilder)
if err != nil {
return nil, err
}
ch, marshalled, err := a.changeBuilder.BuildAndApply()
if err != nil {
return nil, err
}
a.tree.AddFast(ch)
rawCh := &aclpb.RawChange{
Payload: marshalled,
Signature: ch.Signature(),
Id: ch.Id,
}
err = a.treeStorage.AddRawChange(rawCh)
if err != nil {
return nil, err
}
err = a.treeStorage.SetHeads([]string{ch.Id})
if err != nil {
return nil, err
}
return rawCh, nil
}
func (a *aclTree) AddRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (AddResult, error) {
// TODO: make proper error handling, because there are a lot of corner cases where this will break
var err error
var mode Mode
var changes []*Change // TODO: = addChangesBuf[:0] ...
var notSeenIdx []int
prevHeads := a.tree.Heads()
for idx, ch := range rawChanges {
if a.HasChange(ch.Id) {
continue
}
change, err := NewFromRawChange(ch)
// TODO: think what if we will have incorrect signatures on rawChanges, how everything will work
if err != nil {
continue
}
changes = append(changes, change)
notSeenIdx = append(notSeenIdx, idx)
}
if len(notSeenIdx) == 0 {
return AddResult{
OldHeads: prevHeads,
Heads: prevHeads,
Summary: AddResultSummaryNothing,
}, nil
}
defer func() {
if err != nil {
return
}
err = a.removeOrphans()
if err != nil {
return
}
err = a.treeStorage.SetHeads(a.tree.Heads())
if err != nil {
return
}
if a.updateListener == nil {
return
}
switch mode {
case Append:
a.updateListener.Update(a)
case Rebuild:
a.updateListener.Rebuild(a)
default:
break
}
}()
getAddedChanges := func() []*aclpb.RawChange {
var added []*aclpb.RawChange
for _, idx := range notSeenIdx {
rawChange := rawChanges[idx]
if _, exists := a.tree.attached[rawChange.Id]; exists {
added = append(added, rawChange)
}
}
return added
}
for _, ch := range changes {
err = a.treeStorage.AddChange(ch)
if err != nil {
return AddResult{}, err
}
err = a.treeStorage.AddOrphans(ch.Id)
if err != nil {
return AddResult{}, err
}
}
rebuild := func() (AddResult, error) {
err = a.rebuildFromStorage()
if err != nil {
return AddResult{}, err
}
return AddResult{
OldHeads: prevHeads,
Heads: a.tree.Heads(),
Added: getAddedChanges(),
Summary: AddResultSummaryRebuild,
}, nil
}
mode = a.tree.Add(changes...)
switch mode {
case Nothing:
for _, ch := range changes {
// rebuilding if the snapshot is different from the root
if ch.SnapshotId != a.tree.RootId() && ch.SnapshotId != "" {
return rebuild()
}
}
return AddResult{
OldHeads: prevHeads,
Heads: prevHeads,
Summary: AddResultSummaryNothing,
}, nil
case Rebuild:
return rebuild()
default:
// just rebuilding the state from start without reloading everything from tree storage
// as an optimization we could've started from current heads, but I didn't implement that
a.aclState, err = a.aclStateBuilder.Build()
if err != nil {
return AddResult{}, err
}
return AddResult{
OldHeads: prevHeads,
Heads: a.tree.Heads(),
Added: getAddedChanges(),
Summary: AddResultSummaryAppend,
}, nil
}
}
func (a *aclTree) Iterate(f func(change *Change) bool) {
a.tree.Iterate(a.tree.RootId(), f)
}
func (a *aclTree) IterateFrom(s string, f func(change *Change) bool) {
a.tree.Iterate(s, f)
}
func (a *aclTree) HasChange(s string) bool {
_, attachedExists := a.tree.attached[s]
_, unattachedExists := a.tree.unAttached[s]
_, invalidExists := a.tree.invalidChanges[s]
return attachedExists || unattachedExists || invalidExists
}
func (a *aclTree) Heads() []string {
return a.tree.Heads()
}
func (a *aclTree) Root() *Change {
return a.tree.Root()
}
func (a *aclTree) Close() error {
return nil
}
func (a *aclTree) SnapshotPath() []string {
// TODO: think about caching this
var path []string
// TODO: think that the user may have not all of the snapshots locally
currentSnapshotId := a.tree.RootId()
for currentSnapshotId != "" {
sn, err := a.treeBuilder.loadChange(currentSnapshotId)
if err != nil {
break
}
path = append(path, currentSnapshotId)
currentSnapshotId = sn.SnapshotId
}
return path
}
func (a *aclTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawChange, error) {
// TODO: think about when the clients will have their full acl tree and thus full snapshots
// but no changes after some of the snapshots
var (
isNewDocument = len(theirPath) == 0
ourPath = a.SnapshotPath()
// by default returning everything we have
commonSnapshot = ourPath[len(ourPath)-1] // TODO: root snapshot, probably it is better to have a specific method in treestorage
err error
)
// if this is non-empty request
if !isNewDocument {
commonSnapshot, err = commonSnapshotForTwoPaths(ourPath, theirPath)
if err != nil {
return nil, err
}
}
var rawChanges []*aclpb.RawChange
// using custom load function to skip verification step and save raw changes
load := func(id string) (*Change, error) {
raw, err := a.treeStorage.GetChange(context.Background(), id)
if err != nil {
return nil, err
}
ch, err := NewFromRawChange(raw)
if err != nil {
return nil, err
}
rawChanges = append(rawChanges, raw)
return ch, nil
}
// we presume that we have everything after the common snapshot, though this may not be the case in case of clients and only ACL tree changes
log.With(
zap.Strings("heads", a.tree.Heads()),
zap.String("breakpoint", commonSnapshot),
zap.String("id", a.id)).
Debug("getting all changes from common snapshot")
_, err = a.treeBuilder.dfs(a.tree.Heads(), commonSnapshot, load)
if err != nil {
return nil, err
}
if isNewDocument {
// adding snapshot to raw changes
_, err = load(commonSnapshot)
if err != nil {
return nil, err
}
}
log.With(
zap.Int("len(changes)", len(rawChanges)),
zap.String("id", a.id)).
Debug("returning all changes after common snapshot")
return rawChanges, nil
}
func (a *aclTree) DebugDump() (string, error) {
return a.tree.Graph(ACLDescriptionParser)
}
func commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) {
var i int
var j int
log.With(zap.Strings("our path", ourPath), zap.Strings("their path", theirPath)).
Debug("finding common snapshot for two paths")
OuterLoop:
// find starting point from the right
for i = len(ourPath) - 1; i >= 0; i-- {
for j = len(theirPath) - 1; j >= 0; j-- {
// most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes
if ourPath[i] == theirPath[j] {
break OuterLoop
}
}
}
if i < 0 || j < 0 {
return "", ErrNoCommonSnapshot
}
// find last common element of the sequence moving from right to left
for i >= 0 && j >= 0 {
if ourPath[i] == theirPath[j] {
i--
j--
}
}
return ourPath[i+1], nil
}

6
pkg/acl/tree/changevalidator.go
View File

@ -1,7 +1,9 @@
package tree
import "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
type DocTreeValidator interface {
ValidateTree(tree *Tree, aclTree ACLTree) error
ValidateTree(tree *Tree, aclList list.ACLList) error
}
type docTreeValidator struct{}
@ -9,7 +11,7 @@ type docTreeValidator struct{}
func newTreeValidator() DocTreeValidator {
return &docTreeValidator{}
}
func (v *docTreeValidator) ValidateTree(tree *Tree, aclTree ACLTree) error {
func (v *docTreeValidator) ValidateTree(tree *Tree, list list.ACLList) error {
// TODO: add validation logic where we check that the change refers to correct acl heads
// that means that more recent changes should refer to more recent acl heads
return nil

40
pkg/acl/tree/descriptionparser.go
View File

@ -1,13 +1,5 @@
package tree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
"strings"
"unicode"
)
type DescriptionParser interface {
ParseChange(*Change) ([]string, error)
}
@ -19,35 +11,3 @@ type noopDescriptionParser struct{}
func (n noopDescriptionParser) ParseChange(change *Change) ([]string, error) {
return []string{"DOC"}, nil
}
var ACLDescriptionParser = aclDescriptionParser{}
type aclDescriptionParser struct{}
func (a aclDescriptionParser) ParseChange(changeWrapper *Change) (res []string, err error) {
change := changeWrapper.Content
aclData := &aclpb.ACLChangeACLData{}
if changeWrapper.ParsedModel != nil {
aclData = changeWrapper.ParsedModel.(*aclpb.ACLChangeACLData)
} else {
err = proto.Unmarshal(change.ChangesData, aclData)
if err != nil {
return
}
}
var chSymbs []string
for _, chc := range aclData.AclContent {
tp := fmt.Sprintf("%T", chc.Value)
tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
res := ""
for _, ts := range tp {
if unicode.IsUpper(ts) {
res += string(ts)
}
}
chSymbs = append(chSymbs, res)
}
return chSymbs, nil
}

91
pkg/acl/tree/doctree.go
View File

@ -2,8 +2,10 @@ package tree
import (
"context"
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
@ -20,11 +22,37 @@ type TreeUpdateListener interface {
Rebuild(tree DocTree)
}
type RWLocker interface {
sync.Locker
RLock()
RUnlock()
}
var ErrHasInvalidChanges = errors.New("the change is invalid")
var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot")
var ErrTreeWithoutIdentity = errors.New("acl tree is created without identity")
type AddResultSummary int
const (
AddResultSummaryNothing AddResultSummary = iota
AddResultSummaryAppend
AddResultSummaryRebuild
)
type AddResult struct {
OldHeads []string
Heads []string
Added []*aclpb.RawChange
// TODO: add summary for changes
Summary AddResultSummary
}
type DocTree interface {
RWLocker
CommonTree
AddContent(ctx context.Context, aclTree ACLTree, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, aclTree ACLTree, changes ...*aclpb.RawChange) (AddResult, error)
AddContent(ctx context.Context, aclList list.ACLList, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, aclList list.ACLList, changes ...*aclpb.RawChange) (AddResult, error)
}
type docTree struct {
@ -48,7 +76,7 @@ type docTree struct {
sync.RWMutex
}
func BuildDocTreeWithIdentity(t treestorage.TreeStorage, acc *account.AccountData, listener TreeUpdateListener, aclTree ACLTree) (DocTree, error) {
func BuildDocTreeWithIdentity(t treestorage.TreeStorage, acc *account.AccountData, listener TreeUpdateListener, aclList list.ACLList) (DocTree, error) {
treeBuilder := newTreeBuilder(t, acc.Decoder)
validator := newTreeValidator()
@ -64,7 +92,7 @@ func BuildDocTreeWithIdentity(t treestorage.TreeStorage, acc *account.AccountDat
notSeenIdxBuf: make([]int, 0, 10),
identityKeys: make(map[string]signingkey.PubKey),
}
err := docTree.rebuildFromStorage(aclTree, nil)
err := docTree.rebuildFromStorage(aclList, nil)
if err != nil {
return nil, err
}
@ -84,7 +112,7 @@ func BuildDocTreeWithIdentity(t treestorage.TreeStorage, acc *account.AccountDat
return docTree, nil
}
func BuildDocTree(t treestorage.TreeStorage, decoder keys.Decoder, listener TreeUpdateListener, aclTree ACLTree) (DocTree, error) {
func BuildDocTree(t treestorage.TreeStorage, decoder keys.Decoder, listener TreeUpdateListener, aclList list.ACLList) (DocTree, error) {
treeBuilder := newTreeBuilder(t, decoder)
validator := newTreeValidator()
@ -99,7 +127,7 @@ func BuildDocTree(t treestorage.TreeStorage, decoder keys.Decoder, listener Tree
notSeenIdxBuf: make([]int, 0, 10),
identityKeys: make(map[string]signingkey.PubKey),
}
err := docTree.rebuildFromStorage(aclTree, nil)
err := docTree.rebuildFromStorage(aclList, nil)
if err != nil {
return nil, err
}
@ -119,7 +147,7 @@ func BuildDocTree(t treestorage.TreeStorage, decoder keys.Decoder, listener Tree
return docTree, nil
}
func (d *docTree) rebuildFromStorage(aclTree ACLTree, newChanges []*Change) (err error) {
func (d *docTree) rebuildFromStorage(aclList list.ACLList, newChanges []*Change) (err error) {
d.treeBuilder.Init(d.identityKeys)
d.tree, err = d.treeBuilder.Build(false, newChanges)
@ -127,7 +155,7 @@ func (d *docTree) rebuildFromStorage(aclTree ACLTree, newChanges []*Change) (err
return err
}
return d.validator.ValidateTree(d.tree, aclTree)
return d.validator.ValidateTree(d.tree, aclList)
}
func (d *docTree) ID() string {
@ -142,7 +170,7 @@ func (d *docTree) Storage() treestorage.TreeStorage {
return d.treeStorage
}
func (d *docTree) AddContent(ctx context.Context, aclTree ACLTree, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error) {
func (d *docTree) AddContent(ctx context.Context, aclList list.ACLList, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error) {
if d.accountData == nil {
return nil, ErrTreeWithoutIdentity
}
@ -153,12 +181,12 @@ func (d *docTree) AddContent(ctx context.Context, aclTree ACLTree, content proto
d.updateListener.Update(d)
}
}()
state := aclTree.ACLState()
state := aclList.ACLState()
change := &aclpb.Change{
TreeHeadIds: d.tree.Heads(),
AclHeadIds: aclTree.Heads(),
AclHeadId: aclList.Last().Id,
SnapshotBaseId: d.tree.RootId(),
CurrentReadKeyHash: state.currentReadKeyHash,
CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: int64(time.Now().Nanosecond()),
Identity: d.accountData.Identity,
IsSnapshot: isSnapshot,
@ -168,7 +196,7 @@ func (d *docTree) AddContent(ctx context.Context, aclTree ACLTree, content proto
if err != nil {
return nil, err
}
encrypted, err := state.userReadKeys[state.currentReadKeyHash].Encrypt(marshalledData)
encrypted, err := state.UserReadKeys()[state.CurrentReadKeyHash()].Encrypt(marshalledData)
if err != nil {
return nil, err
}
@ -213,7 +241,7 @@ func (d *docTree) AddContent(ctx context.Context, aclTree ACLTree, content proto
return rawCh, nil
}
func (d *docTree) AddRawChanges(ctx context.Context, aclTree ACLTree, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) {
func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) {
var mode Mode
// resetting buffers
@ -300,7 +328,7 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclTree ACLTree, rawChanges
// checking if we have some changes with different snapshot and then rebuilding
for _, ch := range d.tmpChangesBuf {
if ch.SnapshotId != d.tree.RootId() && ch.SnapshotId != "" {
err = d.rebuildFromStorage(aclTree, d.tmpChangesBuf)
err = d.rebuildFromStorage(aclList, d.tmpChangesBuf)
if err != nil {
return AddResult{}, err
}
@ -331,7 +359,7 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclTree ACLTree, rawChanges
default:
// just rebuilding the state from start without reloading everything from tree storage
// as an optimization we could've started from current heads, but I didn't implement that
err = d.validator.ValidateTree(d.tree, aclTree)
err = d.validator.ValidateTree(d.tree, aclList)
if err != nil {
// rolling back
for _, ch := range d.tmpChangesBuf {
@ -366,8 +394,7 @@ func (d *docTree) IterateFrom(s string, f func(change *Change) bool) {
func (d *docTree) HasChange(s string) bool {
_, attachedExists := d.tree.attached[s]
_, unattachedExists := d.tree.unAttached[s]
_, invalidExists := d.tree.invalidChanges[s]
return attachedExists || unattachedExists || invalidExists
return attachedExists || unattachedExists
}
func (d *docTree) Heads() []string {
@ -462,3 +489,31 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh
func (d *docTree) DebugDump() (string, error) {
return d.tree.Graph(NoOpDescriptionParser)
}
func commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) {
var i int
var j int
log.With(zap.Strings("our path", ourPath), zap.Strings("their path", theirPath)).
Debug("finding common snapshot for two paths")
OuterLoop:
// find starting point from the right
for i = len(ourPath) - 1; i >= 0; i-- {
for j = len(theirPath) - 1; j >= 0; j-- {
// most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes
if ourPath[i] == theirPath[j] {
break OuterLoop
}
}
}
if i < 0 || j < 0 {
return "", ErrNoCommonSnapshot
}
// find last common element of the sequence moving from right to left
for i >= 0 && j >= 0 {
if ourPath[i] == theirPath[j] {
i--
j--
}
}
return ourPath[i+1], nil
}

95
pkg/acl/tree/treestorage.go
View File

@ -3,65 +3,66 @@ package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
"time"
)
func CreateNewTreeStorageWithACL(
acc *account.AccountData,
build func(builder ACLChangeBuilder) error,
create treestorage.CreatorFunc) (treestorage.TreeStorage, error) {
bld := newACLChangeBuilder()
bld.Init(
newACLStateWithIdentity(acc.Identity, acc.EncKey, signingkey.NewEd25519PubKeyDecoder()),
&Tree{},
acc)
err := build(bld)
if err != nil {
return nil, err
}
change, payload, err := bld.BuildAndApply()
if err != nil {
return nil, err
}
rawChange := &aclpb.RawChange{
Payload: payload,
Signature: change.Signature(),
Id: change.CID(),
}
header, id, err := createTreeHeaderAndId(rawChange, treepb.TreeHeader_ACLTree, "")
if err != nil {
return nil, err
}
thr, err := create(id, header, []*aclpb.RawChange{rawChange})
if err != nil {
return nil, err
}
err = thr.SetHeads([]string{change.CID()})
if err != nil {
return nil, err
}
return thr, nil
}
//
//func CreateNewTreeStorageWithACL(
// acc *account.AccountData,
// build func(builder list.ACLChangeBuilder) error,
// create treestorage.CreatorFunc) (treestorage.TreeStorage, error) {
// bld := list.newACLChangeBuilder()
// bld.Init(
// list.newACLStateWithIdentity(acc.Identity, acc.EncKey, signingkey.NewEd25519PubKeyDecoder()),
// &Tree{},
// acc)
// err := build(bld)
// if err != nil {
// return nil, err
// }
//
// change, payload, err := bld.BuildAndApply()
// if err != nil {
// return nil, err
// }
//
// rawChange := &aclpb.RawChange{
// Payload: payload,
// Signature: change.Signature(),
// Id: change.CID(),
// }
// header, id, err := createTreeHeaderAndId(rawChange, treepb.TreeHeader_ACLTree, "")
// if err != nil {
// return nil, err
// }
//
// thr, err := create(id, header, []*aclpb.RawChange{rawChange})
// if err != nil {
// return nil, err
// }
//
// err = thr.SetHeads([]string{change.CID()})
// if err != nil {
// return nil, err
// }
// return thr, nil
//}
func CreateNewTreeStorage(
acc *account.AccountData,
aclTree ACLTree,
aclList list.ACLList,
content proto.Marshaler,
create treestorage.CreatorFunc) (treestorage.TreeStorage, error) {
state := aclTree.ACLState()
state := aclList.ACLState()
change := &aclpb.Change{
AclHeadIds: aclTree.Heads(),
CurrentReadKeyHash: state.currentReadKeyHash,
AclHeadId: aclList.Last().Id,
CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: int64(time.Now().Nanosecond()),
Identity: acc.Identity,
IsSnapshot: true,
@ -71,7 +72,7 @@ func CreateNewTreeStorage(
if err != nil {
return nil, err
}
encrypted, err := state.userReadKeys[state.currentReadKeyHash].Encrypt(marshalledData)
encrypted, err := state.UserReadKeys()[state.CurrentReadKeyHash()].Encrypt(marshalledData)
if err != nil {
return nil, err
}
@ -95,7 +96,7 @@ func CreateNewTreeStorage(
Signature: signature,
Id: changeId,
}
header, treeId, err := createTreeHeaderAndId(rawChange, treepb.TreeHeader_DocTree, aclTree.ID())
header, treeId, err := createTreeHeaderAndId(rawChange, treepb.TreeHeader_DocTree, aclList.ID())
if err != nil {
return nil, err
}

3
service/document/service.go
View File

@ -6,6 +6,7 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
@ -131,7 +132,7 @@ func (s *service) CreateACLTree(ctx context.Context) (id string, err error) {
heads []string
)
t, err := tree.CreateNewTreeStorageWithACL(acc, func(builder tree.ACLChangeBuilder) error {
t, err := tree.CreateNewTreeStorageWithACL(acc, func(builder list.ACLChangeBuilder) error {
err := builder.UserAdd(acc.Identity, acc.EncKey.GetPublic(), aclpb.ACLChange_Admin)
if err != nil {
return err