From b026d228dfe3ee0dae771564f0137b36606322c0 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 5 Sep 2022 11:48:50 +0200 Subject: [PATCH] Further changes to treebuilder etc --- pkg/acl/list/aclstate.go | 9 + pkg/acl/tree/change.go | 21 +-- pkg/acl/tree/doctree.go | 315 ++++++++++++++++---------------- pkg/acl/tree/keychain.go | 30 +++ pkg/acl/tree/signablecontent.go | 13 ++ pkg/acl/tree/tree.go | 25 ++- pkg/acl/tree/treebuilder.go | 123 ++++--------- pkg/acl/tree/treereduce.go | 7 +- pkg/acl/tree/treestorage.go | 53 +----- pkg/acl/tree/util.go | 27 +++ service/document/service.go | 8 +- 11 files changed, 322 insertions(+), 309 deletions(-) create mode 100644 pkg/acl/tree/keychain.go create mode 100644 pkg/acl/tree/signablecontent.go create mode 100644 pkg/acl/tree/util.go diff --git a/pkg/acl/list/aclstate.go b/pkg/acl/list/aclstate.go index c5a2d63f..c34e73e8 100644 --- a/pkg/acl/list/aclstate.go +++ b/pkg/acl/list/aclstate.go @@ -24,6 +24,7 @@ var ErrDocumentForbidden = errors.New("your user was forbidden access to the doc var ErrUserAlreadyExists = errors.New("user already exists") var ErrNoSuchRecord = errors.New("no such record") var ErrInsufficientPermissions = errors.New("insufficient permissions") +var ErrNoReadKey = errors.New("acl state doesn't have a read key") type UserPermissionPair struct { Identity string @@ -70,6 +71,14 @@ func (st *ACLState) CurrentReadKeyHash() uint64 { return st.currentReadKeyHash } +func (st *ACLState) CurrentReadKey() (*symmetric.Key, error) { + key, exists := st.userReadKeys[st.currentReadKeyHash] + if !exists { + return nil, ErrNoReadKey + } + return key, nil +} + func (st *ACLState) UserReadKeys() map[uint64]*symmetric.Key { return st.userReadKeys } diff --git a/pkg/acl/tree/change.go b/pkg/acl/tree/change.go index b3243fdb..2e1d3dce 100644 --- a/pkg/acl/tree/change.go +++ b/pkg/acl/tree/change.go @@ -3,8 +3,6 @@ package tree import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" - "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys" - "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey" "github.com/gogo/protobuf/proto" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric" @@ -52,7 +50,7 @@ func (ch *Change) DecryptContents(key *symmetric.Key) error { return nil } -func NewFromRawChange(rawChange *aclpb.RawChange) (*Change, error) { +func NewChangeFromRaw(rawChange *aclpb.RawChange) (*Change, error) { unmarshalled := &aclpb.Change{} err := proto.Unmarshal(rawChange.Payload, unmarshalled) if err != nil { @@ -64,25 +62,20 @@ func NewFromRawChange(rawChange *aclpb.RawChange) (*Change, error) { return ch, nil } -func NewFromVerifiedRawChange( +func NewVerifiedChangeFromRaw( rawChange *aclpb.RawChange, - identityKeys map[string]signingkey.PubKey, - decoder keys.Decoder) (*Change, error) { + kch *keychain) (*Change, error) { unmarshalled := &aclpb.Change{} err := proto.Unmarshal(rawChange.Payload, unmarshalled) if err != nil { return nil, err } - identityKey, exists := identityKeys[unmarshalled.Identity] - if !exists { - key, err := decoder.DecodeFromString(unmarshalled.Identity) - if err != nil { - return nil, err - } - identityKey = key.(signingkey.PubKey) - identityKeys[unmarshalled.Identity] = identityKey + identityKey, err := kch.getOrAdd(unmarshalled.Identity) + if err != nil { + return nil, err } + res, err := identityKey.Verify(rawChange.Payload, rawChange.Signature) if err != nil { return nil, err diff --git a/pkg/acl/tree/doctree.go b/pkg/acl/tree/doctree.go index 4aa1b061..91bbe925 100644 --- a/pkg/acl/tree/doctree.go +++ b/pkg/acl/tree/doctree.go @@ -3,13 +3,10 @@ package tree import ( "context" "errors" - "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid" - "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys" - "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice" "github.com/gogo/protobuf/proto" "go.uber.org/zap" @@ -30,7 +27,6 @@ type RWLocker interface { var ErrHasInvalidChanges = errors.New("the change is invalid") var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot") -var ErrTreeWithoutIdentity = errors.New("acl tree is created without identity") type AddResultSummary int @@ -51,13 +47,12 @@ type AddResult struct { type DocTree interface { RWLocker CommonTree - AddContent(ctx context.Context, aclList list.ACLList, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error) + AddContent(ctx context.Context, aclList list.ACLList, content SignableChangeContent) (*aclpb.RawChange, error) AddRawChanges(ctx context.Context, aclList list.ACLList, changes ...*aclpb.RawChange) (AddResult, error) } type docTree struct { treeStorage storage.TreeStorage - accountData *account.AccountData updateListener TreeUpdateListener id string @@ -66,45 +61,31 @@ type docTree struct { treeBuilder *treeBuilder validator DocTreeValidator + kch *keychain + // buffers difSnapshotBuf []*aclpb.RawChange tmpChangesBuf []*Change + newSnapshots []*Change notSeenIdxBuf []int - identityKeys map[string]signingkey.PubKey - sync.RWMutex } -func BuildDocTreeWithIdentity(t storage.TreeStorage, acc *account.AccountData, listener TreeUpdateListener, aclList list.ACLList) (DocTree, error) { - return buildDocTreeWithAccount(t, acc, acc.Decoder, listener, aclList) -} - -func BuildDocTree(t storage.TreeStorage, decoder keys.Decoder, listener TreeUpdateListener, aclList list.ACLList) (DocTree, error) { - return buildDocTreeWithAccount(t, nil, decoder, listener, aclList) -} - -func buildDocTreeWithAccount( - t storage.TreeStorage, - acc *account.AccountData, - decoder keys.Decoder, - listener TreeUpdateListener, - aclList list.ACLList) (DocTree, error) { - - treeBuilder := newTreeBuilder(t, decoder) +func BuildDocTree(t storage.TreeStorage, listener TreeUpdateListener, aclList list.ACLList) (DocTree, error) { + treeBuilder := newTreeBuilder(t) validator := newTreeValidator() docTree := &docTree{ treeStorage: t, tree: nil, - accountData: acc, treeBuilder: treeBuilder, validator: validator, updateListener: listener, tmpChangesBuf: make([]*Change, 0, 10), difSnapshotBuf: make([]*aclpb.RawChange, 0, 10), notSeenIdxBuf: make([]int, 0, 10), - identityKeys: make(map[string]signingkey.PubKey), + kch: newKeychain(), } err := docTree.rebuildFromStorage(aclList, nil) if err != nil { @@ -143,13 +124,17 @@ func buildDocTreeWithAccount( } func (d *docTree) rebuildFromStorage(aclList list.ACLList, newChanges []*Change) (err error) { - d.treeBuilder.Init(d.identityKeys) + d.treeBuilder.Init(d.kch) - d.tree, err = d.treeBuilder.Build(false, newChanges) + d.tree, err = d.treeBuilder.Build(newChanges) if err != nil { return err } + // during building the tree we may have marked some changes as possible roots, + // but obviously they are not roots, because of the way how we construct the tree + d.tree.clearPossibleRoots() + return d.validator.ValidateTree(d.tree, aclList) } @@ -165,152 +150,159 @@ func (d *docTree) Storage() storage.TreeStorage { return d.treeStorage } -func (d *docTree) AddContent(ctx context.Context, aclList list.ACLList, content proto.Marshaler, isSnapshot bool) (*aclpb.RawChange, error) { - if d.accountData == nil { - return nil, ErrTreeWithoutIdentity - } - +func (d *docTree) AddContent(ctx context.Context, aclList list.ACLList, content SignableChangeContent) (rawChange *aclpb.RawChange, err error) { defer func() { - // TODO: should this be called in a separate goroutine to prevent accidental cycles (tree->updater->tree) if d.updateListener != nil { d.updateListener.Update(d) } }() - state := aclList.ACLState() - change := &aclpb.Change{ + state := aclList.ACLState() // special method for own keys + aclChange := &aclpb.Change{ TreeHeadIds: d.tree.Heads(), AclHeadId: aclList.Head().Id, SnapshotBaseId: d.tree.RootId(), CurrentReadKeyHash: state.CurrentReadKeyHash(), Timestamp: int64(time.Now().Nanosecond()), - Identity: d.accountData.Identity, - IsSnapshot: isSnapshot, + Identity: content.Identity, + IsSnapshot: content.IsSnapshot, } - marshalledData, err := content.Marshal() + marshalledData, err := content.Proto.Marshal() if err != nil { return nil, err } - encrypted, err := state.UserReadKeys()[state.CurrentReadKeyHash()].Encrypt(marshalledData) - if err != nil { - return nil, err - } - change.ChangesData = encrypted - fullMarshalledChange, err := proto.Marshal(change) + readKey, err := state.CurrentReadKey() if err != nil { return nil, err } - signature, err := d.accountData.SignKey.Sign(fullMarshalledChange) + + encrypted, err := readKey.Encrypt(marshalledData) if err != nil { return nil, err } + aclChange.ChangesData = encrypted + + fullMarshalledChange, err := proto.Marshal(aclChange) + if err != nil { + return nil, err + } + + signature, err := content.Key.Sign(fullMarshalledChange) + if err != nil { + return nil, err + } + id, err := cid.NewCIDFromBytes(fullMarshalledChange) if err != nil { return nil, err } - ch := NewChange(id, change) - ch.ParsedModel = content - ch.Sign = signature - if isSnapshot { + docChange := NewChange(id, aclChange) + docChange.ParsedModel = content + docChange.Sign = signature + + if content.IsSnapshot { // clearing tree, because we already fixed everything in the last snapshot d.tree = &Tree{} } - err = d.tree.AddMergedHead(ch) + err = d.tree.AddMergedHead(docChange) if err != nil { - panic("error in adding head") + panic(err) } - rawCh := &aclpb.RawChange{ + rawChange = &aclpb.RawChange{ Payload: fullMarshalledChange, - Signature: ch.Signature(), - Id: ch.Id, + Signature: docChange.Signature(), + Id: docChange.Id, } - err = d.treeStorage.AddRawChange(rawCh) + err = d.treeStorage.AddRawChange(rawChange) if err != nil { - return nil, err + return } - err = d.treeStorage.SetHeads([]string{ch.Id}) - if err != nil { - return nil, err - } - return rawCh, nil + err = d.treeStorage.SetHeads([]string{docChange.Id}) + return } func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) { var mode Mode + mode, addResult, err = d.addRawChanges(ctx, aclList, rawChanges...) + if err != nil { + return + } + // reducing tree if we have new roots + d.tree.reduceTree() + + // adding to database all the added changes only after they are good + for _, ch := range addResult.Added { + err = d.treeStorage.AddRawChange(ch) + if err != nil { + return + } + } + + // setting heads + err = d.treeStorage.SetHeads(d.tree.Heads()) + if err != nil { + return + } + + if d.updateListener == nil { + return + } + + switch mode { + case Append: + d.updateListener.Update(d) + case Rebuild: + d.updateListener.Rebuild(d) + default: + break + } + return +} + +func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawChanges ...*aclpb.RawChange) (mode Mode, addResult AddResult, err error) { // resetting buffers d.tmpChangesBuf = d.tmpChangesBuf[:0] d.notSeenIdxBuf = d.notSeenIdxBuf[:0] d.difSnapshotBuf = d.difSnapshotBuf[:0] + d.newSnapshots = d.newSnapshots[:0] prevHeads := d.tree.Heads() + // TODO: if we can use new snapshot -> update tree, check if some snapshot, dfsPrev? // filtering changes, verifying and unmarshalling them for idx, ch := range rawChanges { if d.HasChange(ch.Id) { continue } - // if we already added the change to invalid ones - if _, exists := d.tree.invalidChanges[ch.Id]; exists { - return AddResult{}, ErrHasInvalidChanges - } var change *Change - change, err = NewFromVerifiedRawChange(ch, d.identityKeys, d.treeBuilder.signingPubKeyDecoder) + change, err = NewVerifiedChangeFromRaw(ch, d.kch) if err != nil { - return AddResult{}, err + return } + if change.IsSnapshot { + d.newSnapshots = append(d.newSnapshots, change) + } d.tmpChangesBuf = append(d.tmpChangesBuf, change) d.notSeenIdxBuf = append(d.notSeenIdxBuf, idx) } // if no new changes, then returning if len(d.notSeenIdxBuf) == 0 { - return AddResult{ + addResult = AddResult{ OldHeads: prevHeads, Heads: prevHeads, Summary: AddResultSummaryNothing, - }, nil + } + return } - defer func() { - if err != nil { - return - } - - // adding to database all the added changes only after they are good - for _, ch := range addResult.Added { - err = d.treeStorage.AddRawChange(ch) - if err != nil { - return - } - } - - // setting heads - err = d.treeStorage.SetHeads(d.tree.Heads()) - if err != nil { - return - } - - if d.updateListener == nil { - return - } - - switch mode { - case Append: - d.updateListener.Update(d) - case Rebuild: - d.updateListener.Rebuild(d) - default: - break - } - }() - // returns changes that we added to the tree getAddedChanges := func() []*aclpb.RawChange { var added []*aclpb.RawChange @@ -333,13 +325,28 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawCh } } + // checks if we need to go to database + isOldSnapshot := func(ch *Change) bool { + if ch.SnapshotId == d.tree.RootId() { + return false + } + for _, sn := range d.newSnapshots { + // if change refers to newly received snapshot + if ch.SnapshotId == sn.Id { + return false + } + } + return true + } + // checking if we have some changes with different snapshot and then rebuilding for _, ch := range d.tmpChangesBuf { - if ch.SnapshotId != d.tree.RootId() && ch.SnapshotId != "" { + if isOldSnapshot(ch) { err = d.rebuildFromStorage(aclList, d.tmpChangesBuf) if err != nil { - rollback() - return AddResult{}, err + // rebuilding without new changes + d.rebuildFromStorage(aclList, nil) + return } addResult = AddResult{ @@ -348,7 +355,6 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawCh Added: getAddedChanges(), Summary: AddResultSummaryRebuild, } - err = nil return } } @@ -362,7 +368,6 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawCh Heads: prevHeads, Summary: AddResultSummaryNothing, } - err = nil return default: @@ -371,7 +376,8 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawCh err = d.validator.ValidateTree(d.tree, aclList) if err != nil { rollback() - return AddResult{}, ErrHasInvalidChanges + err = ErrHasInvalidChanges + return } addResult = AddResult{ @@ -380,7 +386,6 @@ func (d *docTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawCh Added: getAddedChanges(), Summary: AddResultSummaryAppend, } - err = nil } return } @@ -429,14 +434,11 @@ func (d *docTree) SnapshotPath() []string { } func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawChange, error) { - // TODO: think about when the clients will have their full acl tree and thus full snapshots - // but no changes after some of the snapshots - var ( isNewDocument = len(theirPath) == 0 ourPath = d.SnapshotPath() // by default returning everything we have - commonSnapshot = ourPath[len(ourPath)-1] // TODO: root snapshot, probably it is better to have a specific method in treestorage + commonSnapshot = ourPath[len(ourPath)-1] err error ) @@ -447,16 +449,55 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh return nil, err } } - // TODO: if the snapshot is in the tree we probably can skip going to the DB + + log.With( + zap.Strings("heads", d.tree.Heads()), + zap.String("breakpoint", commonSnapshot), + zap.String("id", d.id)). + Debug("getting all changes from common snapshot") + + if commonSnapshot == d.tree.RootId() { + return d.getChangesFromTree(isNewDocument) + } else { + return d.getChangesFromDB(commonSnapshot, isNewDocument) + } +} + +func (d *docTree) getChangesFromTree(isNewDocument bool) (marshalledChanges []*aclpb.RawChange, err error) { + if !isNewDocument { + // ignoring root change + d.tree.Root().visited = true + } + d.tree.dfsPrev(d.tree.HeadsChanges(), func(ch *Change) bool { + var marshalled []byte + marshalled, err = ch.Content.Marshal() + if err != nil { + return false + } + raw := &aclpb.RawChange{ + Payload: marshalled, + Signature: ch.Signature(), + Id: ch.Id, + } + marshalledChanges = append(marshalledChanges, raw) + return true + }, func(changes []*Change) {}) + + if err != nil { + return nil, err + } + return +} + +func (d *docTree) getChangesFromDB(commonSnapshot string, isNewDocument bool) (marshalledChanges []*aclpb.RawChange, err error) { var rawChanges []*aclpb.RawChange - // using custom load function to skip verification step and save raw changes load := func(id string) (*Change, error) { raw, err := d.treeStorage.GetRawChange(context.Background(), id) if err != nil { return nil, err } - ch, err := NewFromRawChange(raw) + ch, err := NewChangeFromRaw(raw) if err != nil { return nil, err } @@ -464,16 +505,12 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh rawChanges = append(rawChanges, raw) return ch, nil } - // we presume that we have everything after the common snapshot, though this may not be the case in case of clients and only ACL tree changes - log.With( - zap.Strings("heads", d.tree.Heads()), - zap.String("breakpoint", commonSnapshot), - zap.String("id", d.id)). - Debug("getting all changes from common snapshot") + _, err = d.treeBuilder.dfs(d.tree.Heads(), commonSnapshot, load) if err != nil { return nil, err } + if isNewDocument { // adding snapshot to raw changes _, err = load(commonSnapshot) @@ -481,10 +518,6 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh return nil, err } } - log.With( - zap.Int("len(changes)", len(rawChanges)), - zap.String("id", d.id)). - Debug("returning all changes after common snapshot") return rawChanges, nil } @@ -492,31 +525,3 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh func (d *docTree) DebugDump() (string, error) { return d.tree.Graph(NoOpDescriptionParser) } - -func commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) { - var i int - var j int - log.With(zap.Strings("our path", ourPath), zap.Strings("their path", theirPath)). - Debug("finding common snapshot for two paths") -OuterLoop: - // find starting point from the right - for i = len(ourPath) - 1; i >= 0; i-- { - for j = len(theirPath) - 1; j >= 0; j-- { - // most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes - if ourPath[i] == theirPath[j] { - break OuterLoop - } - } - } - if i < 0 || j < 0 { - return "", ErrNoCommonSnapshot - } - // find last common element of the sequence moving from right to left - for i >= 0 && j >= 0 { - if ourPath[i] == theirPath[j] { - i-- - j-- - } - } - return ourPath[i+1], nil -} diff --git a/pkg/acl/tree/keychain.go b/pkg/acl/tree/keychain.go new file mode 100644 index 00000000..2e6eca83 --- /dev/null +++ b/pkg/acl/tree/keychain.go @@ -0,0 +1,30 @@ +package tree + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys" + "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey" +) + +type keychain struct { + decoder keys.Decoder + keys map[string]signingkey.PubKey +} + +func newKeychain() *keychain { + return &keychain{ + decoder: signingkey.NewEDPubKeyDecoder(), + } +} + +func (k *keychain) getOrAdd(identity string) (signingkey.PubKey, error) { + if key, exists := k.keys[identity]; exists { + return key, nil + } + res, err := k.decoder.DecodeFromString(identity) + if err != nil { + return nil, err + } + + k.keys[identity] = res.(signingkey.PubKey) + return res.(signingkey.PubKey), nil +} diff --git a/pkg/acl/tree/signablecontent.go b/pkg/acl/tree/signablecontent.go new file mode 100644 index 00000000..f97ed44a --- /dev/null +++ b/pkg/acl/tree/signablecontent.go @@ -0,0 +1,13 @@ +package tree + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey" + "github.com/gogo/protobuf/proto" +) + +type SignableChangeContent struct { + Proto proto.Marshaler + Key signingkey.PrivKey + Identity string + IsSnapshot bool +} diff --git a/pkg/acl/tree/tree.go b/pkg/acl/tree/tree.go index aa36d667..e251d911 100644 --- a/pkg/acl/tree/tree.go +++ b/pkg/acl/tree/tree.go @@ -270,9 +270,16 @@ func (t *Tree) after(id1, id2 string) (found bool) { return } -func (t *Tree) dfsPrev(stack []*Change, visit func(ch *Change), afterVisit func([]*Change)) { +func (t *Tree) dfsPrev(stack []*Change, visit func(ch *Change) (isContinue bool), afterVisit func([]*Change)) { t.visitedBuf = t.visitedBuf[:0] + defer func() { + afterVisit(t.visitedBuf) + for _, ch := range t.visitedBuf { + ch.visited = false + } + }() + for len(stack) > 0 { ch := stack[len(stack)-1] stack = stack[:len(stack)-1] @@ -289,11 +296,9 @@ func (t *Tree) dfsPrev(stack []*Change, visit func(ch *Change), afterVisit func( stack = append(stack, prevCh) } } - visit(ch) - } - afterVisit(t.visitedBuf) - for _, ch := range t.visitedBuf { - ch.visited = false + if !visit(ch) { + return + } } } @@ -373,6 +378,14 @@ func (t *Tree) Heads() []string { return t.headIds } +func (t *Tree) HeadsChanges() []*Change { + var heads []*Change + for _, head := range t.headIds { + heads = append(heads, t.attached[head]) + } + return heads +} + func (t *Tree) String() string { var buf = bytes.NewBuffer(nil) t.Iterate(t.RootId(), func(c *Change) (isContinue bool) { diff --git a/pkg/acl/tree/treebuilder.go b/pkg/acl/tree/treebuilder.go index 11e95955..af229cd5 100644 --- a/pkg/acl/tree/treebuilder.go +++ b/pkg/acl/tree/treebuilder.go @@ -19,112 +19,55 @@ var ( ) type treeBuilder struct { - cache map[string]*Change - identityKeys map[string]signingkey.PubKey - signingPubKeyDecoder keys.Decoder - tree *Tree - treeStorage storage.TreeStorage + cache map[string]*Change + kch *keychain + tree *Tree + treeStorage storage.TreeStorage + + // buffers + idStack []string + loadBuffer []*Change } -func newTreeBuilder(t storage.TreeStorage, decoder keys.Decoder) *treeBuilder { +func newTreeBuilder(t storage.TreeStorage) *treeBuilder { return &treeBuilder{ - signingPubKeyDecoder: decoder, - treeStorage: t, + treeStorage: t, } } -func (tb *treeBuilder) Init(identityKeys map[string]signingkey.PubKey) { +func (tb *treeBuilder) Init(kch *keychain) { tb.cache = make(map[string]*Change) - tb.identityKeys = identityKeys + tb.kch = kch tb.tree = &Tree{} } -func (tb *treeBuilder) Build(fromStart bool, newChanges []*Change) (*Tree, error) { - var headsAndOrphans []string +func (tb *treeBuilder) Build(newChanges []*Change) (*Tree, error) { + var headsAndNewChanges []string heads, err := tb.treeStorage.Heads() if err != nil { return nil, err } - headsAndOrphans = append(headsAndOrphans, heads...) + headsAndNewChanges = append(headsAndNewChanges, heads...) tb.cache = make(map[string]*Change) for _, ch := range newChanges { - headsAndOrphans = append(headsAndOrphans, ch.Id) + headsAndNewChanges = append(headsAndNewChanges, ch.Id) tb.cache[ch.Id] = ch } log.With(zap.Strings("heads", heads)).Debug("building tree") - if fromStart { - if err := tb.buildTreeFromStart(headsAndOrphans); err != nil { - return nil, fmt.Errorf("buildTree error: %v", err) - } - } else { - breakpoint, err := tb.findBreakpoint(headsAndOrphans) - if err != nil { - return nil, fmt.Errorf("findBreakpoint error: %v", err) - } + breakpoint, err := tb.findBreakpoint(headsAndNewChanges) + if err != nil { + return nil, fmt.Errorf("findBreakpoint error: %v", err) + } - if err = tb.buildTree(headsAndOrphans, breakpoint); err != nil { - return nil, fmt.Errorf("buildTree error: %v", err) - } + if err = tb.buildTree(headsAndNewChanges, breakpoint); err != nil { + return nil, fmt.Errorf("buildTree error: %v", err) } return tb.tree, nil } -func (tb *treeBuilder) buildTreeFromStart(heads []string) (err error) { - changes, root, err := tb.dfsFromStart(heads) - if err != nil { - return err - } - - tb.tree.AddFast(root) - tb.tree.AddFast(changes...) - return -} - -func (tb *treeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change, err error) { - var possibleRoots []*Change - stack := make([]string, len(heads), len(heads)*2) - copy(stack, heads) - - buf = make([]*Change, 0, len(stack)*2) - uniqMap := make(map[string]struct{}) - for len(stack) > 0 { - id := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if _, exists := uniqMap[id]; exists { - continue - } - - ch, err := tb.loadChange(id) - if err != nil { - continue - } - - uniqMap[id] = struct{}{} - buf = append(buf, ch) - - for _, prev := range ch.PreviousIds { - stack = append(stack, prev) - } - if len(ch.PreviousIds) == 0 { - possibleRoots = append(possibleRoots, ch) - } - } - header, err := tb.treeStorage.Header() - if err != nil { - return nil, nil, err - } - for _, r := range possibleRoots { - if r.Id == header.FirstId { - return buf, r, nil - } - } - - return nil, nil, fmt.Errorf("could not find root change") -} - func (tb *treeBuilder) buildTree(heads []string, breakpoint string) (err error) { ch, err := tb.loadChange(breakpoint) if err != nil { @@ -141,11 +84,17 @@ func (tb *treeBuilder) dfs( heads []string, breakpoint string, load func(string) (*Change, error)) (buf []*Change, err error) { - stack := make([]string, len(heads), len(heads)*2) + tb.idStack = tb.idStack[:0] + tb.loadBuffer = tb.loadBuffer[:0] + buf = tb.loadBuffer + + var ( + stack = tb.idStack + uniqMap = map[string]struct{}{breakpoint: {}} + ) + copy(stack, heads) - buf = make([]*Change, 0, len(stack)*2) - uniqMap := map[string]struct{}{breakpoint: {}} for len(stack) > 0 { id := stack[len(stack)-1] stack = stack[:len(stack)-1] @@ -162,6 +111,9 @@ func (tb *treeBuilder) dfs( buf = append(buf, ch) for _, prev := range ch.PreviousIds { + if _, exists := uniqMap[id]; exists { + continue + } stack = append(stack, prev) } } @@ -173,7 +125,6 @@ func (tb *treeBuilder) loadChange(id string) (ch *Change, err error) { return ch, nil } - // TODO: Add virtual changes logic ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -182,8 +133,7 @@ func (tb *treeBuilder) loadChange(id string) (ch *Change, err error) { return nil, err } - // TODO: maybe we can use unverified changes here, because we shouldn't put bad changes in the DB in the first place - ch, err = NewFromVerifiedRawChange(change, tb.identityKeys, tb.signingPubKeyDecoder) + ch, err = NewVerifiedChangeFromRaw(change, tb.identityKeys, tb.signingPubKeyDecoder) if err != nil { return nil, err } @@ -302,11 +252,9 @@ func (tb *treeBuilder) findCommonForTwoSnapshots(s1, s2 string) (s string, err e } isEmptySnapshot := func(ch *Change) bool { - // TODO: add more sophisticated checks in Change for snapshots return !ch.IsSnapshot } - // TODO: can we even have empty snapshots? // prefer not empty snapshot if isEmptySnapshot(ch1) && !isEmptySnapshot(ch2) { log.Warnf("changes build Tree: prefer %s(not empty) over %s(empty)", s2, s1) @@ -316,7 +264,6 @@ func (tb *treeBuilder) findCommonForTwoSnapshots(s1, s2 string) (s string, err e return s1, nil } - // TODO: add virtual change mechanics // unexpected behavior - just return lesser id if s1 < s2 { log.Warnf("changes build Tree: prefer %s (%s<%s)", s1, s1, s2) diff --git a/pkg/acl/tree/treereduce.go b/pkg/acl/tree/treereduce.go index d1a788ac..7677f1cb 100644 --- a/pkg/acl/tree/treereduce.go +++ b/pkg/acl/tree/treereduce.go @@ -21,8 +21,9 @@ func (t *Tree) checkRoot(change *Change) (total int) { change.visited = true t.dfsPrev( stack, - func(ch *Change) { + func(ch *Change) bool { total += 1 + return true }, func(changes []*Change) { if t.root.visited { @@ -45,7 +46,9 @@ func (t *Tree) makeRootAndRemove(start *Change) { t.dfsPrev( stack, - func(ch *Change) {}, + func(ch *Change) bool { + return true + }, func(changes []*Change) { for _, ch := range changes { delete(t.unAttached, ch.Id) diff --git a/pkg/acl/tree/treestorage.go b/pkg/acl/tree/treestorage.go index a82a1ce6..af31d339 100644 --- a/pkg/acl/tree/treestorage.go +++ b/pkg/acl/tree/treestorage.go @@ -10,48 +10,6 @@ import ( "time" ) -// -//func CreateNewTreeStorageWithACL( -// acc *account.AccountData, -// build func(builder list.ACLChangeBuilder) error, -// create treestorage.CreatorFunc) (treestorage.Storage, error) { -// bld := list.newACLChangeBuilder() -// bld.Init( -// list.newACLStateWithIdentity(acc.Identity, acc.EncKey, signingkey.NewEd25519PubKeyDecoder()), -// &Tree{}, -// acc) -// err := build(bld) -// if err != nil { -// return nil, err -// } -// -// change, payload, err := bld.BuildAndApply() -// if err != nil { -// return nil, err -// } -// -// rawChange := &aclpb.RawChange{ -// Payload: payload, -// Signature: change.Signature(), -// Id: change.CID(), -// } -// header, id, err := createTreeHeaderAndId(rawChange, treepb.TreeHeader_ACLTree, "") -// if err != nil { -// return nil, err -// } -// -// thr, err := create(id, header, []*aclpb.RawChange{rawChange}) -// if err != nil { -// return nil, err -// } -// -// err = thr.SetHeads([]string{change.CID()}) -// if err != nil { -// return nil, err -// } -// return thr, nil -//} - func CreateNewTreeStorage( acc *account.AccountData, aclList list.ACLList, @@ -71,20 +29,29 @@ func CreateNewTreeStorage( if err != nil { return nil, err } - encrypted, err := state.UserReadKeys()[state.CurrentReadKeyHash()].Encrypt(marshalledData) + + readKey, err := state.CurrentReadKey() if err != nil { return nil, err } + + encrypted, err := readKey.Encrypt(marshalledData) + if err != nil { + return nil, err + } + change.ChangesData = encrypted fullMarshalledChange, err := proto.Marshal(change) if err != nil { return nil, err } + signature, err := acc.SignKey.Sign(fullMarshalledChange) if err != nil { return nil, err } + changeId, err := cid.NewCIDFromBytes(fullMarshalledChange) if err != nil { return nil, err diff --git a/pkg/acl/tree/util.go b/pkg/acl/tree/util.go new file mode 100644 index 00000000..37a08d31 --- /dev/null +++ b/pkg/acl/tree/util.go @@ -0,0 +1,27 @@ +package tree + +func commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) { + var i int + var j int +OuterLoop: + // find starting point from the right + for i = len(ourPath) - 1; i >= 0; i-- { + for j = len(theirPath) - 1; j >= 0; j-- { + // most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes + if ourPath[i] == theirPath[j] { + break OuterLoop + } + } + } + if i < 0 || j < 0 { + return "", ErrNoCommonSnapshot + } + // find last common element of the sequence moving from right to left + for i >= 0 && j >= 0 { + if ourPath[i] == theirPath[j] { + i-- + j-- + } + } + return ourPath[i+1], nil +} diff --git a/service/document/service.go b/service/document/service.go index 30350a2d..cf62124e 100644 --- a/service/document/service.go +++ b/service/document/service.go @@ -97,7 +97,13 @@ func (s *service) UpdateDocumentTree(ctx context.Context, id, text string) (err defer aclTree.RUnlock() content := createAppendTextChange(text) - ch, err = docTree.AddContent(ctx, aclTree, content, false) + signable := tree.SignableChangeContent{ + Proto: content, + Key: s.account.Account().SignKey, + Identity: s.account.Account().Identity, + IsSnapshot: false, + } + ch, err = docTree.AddContent(ctx, aclTree, signable) if err != nil { return err }