Merge branch 'main' of github.com:anytypeio/any-sync into update-fileproto

This commit is contained in:
Sergey Cherepanov 2023-03-19 19:21:20 +03:00
commit 22a597389d
No known key found for this signature in database
GPG Key ID: 87F8EDE8FBDF637C
57 changed files with 3429 additions and 586 deletions

View File

@ -172,7 +172,7 @@ func (app *App) Start(ctx context.Context) (err error) {
for i, s := range app.components {
if err = s.Init(app); err != nil {
closeServices(i)
return fmt.Errorf("can't init service '%s': %v", s.Name(), err)
return fmt.Errorf("can't init service '%s': %w", s.Name(), err)
}
}
@ -181,7 +181,7 @@ func (app *App) Start(ctx context.Context) (err error) {
start := time.Now()
if err = serviceRun.Run(ctx); err != nil {
closeServices(i)
return fmt.Errorf("can't run service '%s': %v", serviceRun.Name(), err)
return fmt.Errorf("can't run service '%s': %w", serviceRun.Name(), err)
}
spent := time.Since(start).Milliseconds()
app.startStat.SpentMsTotal += spent

View File

@ -1,33 +1,87 @@
package logger
import "go.uber.org/zap"
import (
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"github.com/anytypeio/any-sync/util/slice"
)
type LogFormat int
const (
ColorizedOutput LogFormat = iota
PlaintextOutput
JSONOutput
)
type Config struct {
Production bool `yaml:"production"`
DefaultLevel string `yaml:"defaultLevel"`
NamedLevels map[string]string `yaml:"namedLevels"`
AddOutputPaths []string `yaml:"outputPaths"`
DisableStdErr bool `yaml:"disableStdErr"`
Format LogFormat `yaml:"format"`
ZapConfig *zap.Config `yaml:"-"` // optional, if set it will be used instead of other config options
}
func (l Config) ApplyGlobal() {
var conf zap.Config
if l.ZapConfig != nil {
conf = *l.ZapConfig
} else {
if l.Production {
conf = zap.NewProductionConfig()
} else {
conf = zap.NewDevelopmentConfig()
}
encConfig := conf.EncoderConfig
switch l.Format {
case PlaintextOutput:
encConfig.EncodeLevel = zapcore.CapitalLevelEncoder
conf.Encoding = "console"
case JSONOutput:
encConfig.MessageKey = "msg"
encConfig.TimeKey = "ts"
encConfig.LevelKey = "level"
encConfig.NameKey = "logger"
encConfig.CallerKey = "caller"
encConfig.EncodeTime = zapcore.ISO8601TimeEncoder
conf.Encoding = "json"
default:
// default is ColorizedOutput
conf.Encoding = "console"
encConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
}
conf.EncoderConfig = encConfig
if len(l.AddOutputPaths) > 0 {
conf.OutputPaths = append(conf.OutputPaths, l.AddOutputPaths...)
}
if l.DisableStdErr {
conf.OutputPaths = slice.Filter(conf.OutputPaths, func(path string) bool {
return path != "stderr"
})
}
if defaultLevel, err := zap.ParseAtomicLevel(l.DefaultLevel); err == nil {
conf.Level = defaultLevel
}
var levels = make(map[string]zap.AtomicLevel)
}
var lvl = make(map[string]zap.AtomicLevel)
for k, v := range l.NamedLevels {
if lev, err := zap.ParseAtomicLevel(v); err != nil {
levels[k] = lev
if lev, err := zap.ParseAtomicLevel(v); err == nil {
lvl[k] = lev
// we need to have a minimum level of all named loggers for the main logger
if lev.Level() < conf.Level.Level() {
conf.Level.SetLevel(lev.Level())
}
}
defaultLogger, err := conf.Build()
}
lg, err := conf.Build()
if err != nil {
Default().Fatal("can't build logger", zap.Error(err))
}
SetDefault(defaultLogger)
SetNamedLevels(levels)
SetDefault(lg)
SetNamedLevels(lvl)
}

View File

@ -32,6 +32,7 @@ func CtxGetFields(ctx context.Context) (fields []zap.Field) {
type CtxLogger struct {
*zap.Logger
name string
}
func (cl CtxLogger) DebugCtx(ctx context.Context, msg string, fields ...zap.Field) {
@ -51,5 +52,9 @@ func (cl CtxLogger) ErrorCtx(ctx context.Context, msg string, fields ...zap.Fiel
}
func (cl CtxLogger) With(fields ...zap.Field) CtxLogger {
return CtxLogger{cl.Logger.With(fields...)}
return CtxLogger{cl.Logger.With(fields...), cl.name}
}
func (cl CtxLogger) Sugar() *zap.SugaredLogger {
return NewNamedSugared(cl.name)
}

View File

@ -1,51 +1,132 @@
package logger
import (
"go.uber.org/zap"
"sync"
"github.com/gobwas/glob"
"go.uber.org/zap"
)
var (
mu sync.Mutex
defaultLogger *zap.Logger
levels = make(map[string]zap.AtomicLevel)
loggers = make(map[string]CtxLogger)
logger *zap.Logger
loggerConfig zap.Config
namedLevels = make(map[string]zap.AtomicLevel)
namedGlobs = make(map[string]glob.Glob)
namedLoggers = make(map[string]CtxLogger)
namedSugarLoggers = make(map[string]*zap.SugaredLogger)
)
func init() {
defaultLogger, _ = zap.NewDevelopment()
zap.NewProduction()
loggerConfig = zap.NewDevelopmentConfig()
logger, _ = loggerConfig.Build()
}
// SetDefault replaces the default logger
// you need to call SetNamedLevels after in case you have named loggers,
// otherwise they will use the old logger
func SetDefault(l *zap.Logger) {
mu.Lock()
defer mu.Unlock()
*defaultLogger = *l
for name, l := range loggers {
*l.Logger = *defaultLogger.Named(name)
}
*logger = *l
}
// SetNamedLevels sets the namedLevels for named loggers
// it also supports glob patterns for names, like "app*"
// can be racy in case there are existing named loggers
// so consider to call only once at the beginning
func SetNamedLevels(l map[string]zap.AtomicLevel) {
mu.Lock()
defer mu.Unlock()
levels = l
namedLevels = l
var minLevel = logger.Level()
for k, l := range namedLevels {
g, err := glob.Compile(k)
if err == nil {
namedGlobs[k] = g
}
namedLevels[k] = l
if l.Level() < minLevel {
minLevel = l.Level()
}
}
if minLevel < logger.Level() {
// recreate logger if the min level is lower than the current min one
loggerConfig.Level = zap.NewAtomicLevelAt(minLevel)
logger, _ = loggerConfig.Build()
}
for name, nl := range namedLoggers {
level := getLevel(name)
newCore := zap.New(logger.Core()).Named(name).WithOptions(
zap.IncreaseLevel(level),
)
*(nl.Logger) = *newCore
}
for name, nl := range namedSugarLoggers {
level := getLevel(name)
newCore := zap.New(logger.Core()).Named(name).WithOptions(
zap.IncreaseLevel(level),
).Sugar()
*(nl) = *newCore
}
}
func Default() *zap.Logger {
mu.Lock()
defer mu.Unlock()
return defaultLogger
return logger
}
func getLevel(name string) zap.AtomicLevel {
level, ok := namedLevels[name]
if !ok {
var found bool
for globName, glob := range namedGlobs {
if glob.Match(name) {
found = true
level, _ = namedLevels[globName]
// no need to check ok, because we know that globName exists
break
}
}
if !found {
level = loggerConfig.Level
}
}
return level
}
func NewNamed(name string, fields ...zap.Field) CtxLogger {
mu.Lock()
defer mu.Unlock()
l := defaultLogger.Named(name)
if len(fields) > 0 {
l = l.With(fields...)
if l, nameExists := namedLoggers[name]; nameExists {
return l
}
ctxL := CtxLogger{l}
loggers[name] = ctxL
level := getLevel(name)
l := zap.New(logger.Core()).Named(name).WithOptions(zap.IncreaseLevel(level),
zap.Fields(fields...))
ctxL := CtxLogger{Logger: l, name: name}
namedLoggers[name] = ctxL
return ctxL
}
func NewNamedSugared(name string) *zap.SugaredLogger {
mu.Lock()
defer mu.Unlock()
if l, nameExists := namedSugarLoggers[name]; nameExists {
return l
}
level := getLevel(name)
l := zap.New(logger.Core()).Named(name).Sugar().WithOptions(zap.IncreaseLevel(level))
namedSugarLoggers[name] = l
return l
}

120
app/ocache/entry.go Normal file
View File

@ -0,0 +1,120 @@
package ocache
import (
"context"
"go.uber.org/zap"
"sync"
"time"
)
type entryState int
const (
entryStateLoading = iota
entryStateActive
entryStateClosing
entryStateClosed
)
type entry struct {
id string
state entryState
lastUsage time.Time
load chan struct{}
loadErr error
value Object
close chan struct{}
mx sync.Mutex
}
func newEntry(id string, value Object, state entryState) *entry {
return &entry{
id: id,
load: make(chan struct{}),
lastUsage: time.Now(),
state: state,
value: value,
}
}
func (e *entry) isActive() bool {
e.mx.Lock()
defer e.mx.Unlock()
return e.state == entryStateActive
}
func (e *entry) isClosing() bool {
e.mx.Lock()
defer e.mx.Unlock()
return e.state == entryStateClosed || e.state == entryStateClosing
}
func (e *entry) waitLoad(ctx context.Context, id string) (value Object, err error) {
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object load", zap.String("id", id))
return nil, ctx.Err()
case <-e.load:
return e.value, e.loadErr
}
}
func (e *entry) waitClose(ctx context.Context, id string) (res bool, err error) {
e.mx.Lock()
switch e.state {
case entryStateClosing:
waitCh := e.close
e.mx.Unlock()
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object close", zap.String("id", id))
return false, ctx.Err()
case <-waitCh:
return true, nil
}
case entryStateClosed:
e.mx.Unlock()
return true, nil
default:
e.mx.Unlock()
return false, nil
}
}
func (e *entry) setClosing(wait bool) (prevState, curState entryState) {
e.mx.Lock()
prevState = e.state
curState = e.state
if e.state == entryStateClosing {
waitCh := e.close
e.mx.Unlock()
if !wait {
return
}
<-waitCh
e.mx.Lock()
}
if e.state != entryStateClosed {
e.state = entryStateClosing
e.close = make(chan struct{})
}
curState = e.state
e.mx.Unlock()
return
}
func (e *entry) setActive(chClose bool) {
e.mx.Lock()
defer e.mx.Unlock()
if chClose {
close(e.close)
}
e.state = entryStateActive
}
func (e *entry) setClosed() {
e.mx.Lock()
defer e.mx.Unlock()
close(e.close)
e.state = entryStateClosed
}

View File

@ -1,11 +1,18 @@
package ocache
import "github.com/prometheus/client_golang/prometheus"
import (
"github.com/prometheus/client_golang/prometheus"
"strings"
)
func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Option {
if subsystem == "" {
subsystem = "cache"
}
nameSplit := strings.Split(namespace, ".")
subSplit := strings.Split(subsystem, ".")
namespace = strings.Join(nameSplit, "_")
subsystem = strings.Join(subSplit, "_")
if reg == nil {
return nil
}

View File

@ -0,0 +1,19 @@
package ocache
import (
"context"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"strings"
"testing"
)
func TestWithPrometheus_MetricsConvertsDots(t *testing.T) {
opt := WithPrometheus(prometheus.NewRegistry(), "some.name", "some.system")
cache := New(func(ctx context.Context, id string) (value Object, err error) {
return &testObject{}, nil
}, opt).(*oCache)
_, err := cache.Get(context.Background(), "id")
require.NoError(t, err)
require.True(t, strings.Contains(cache.metrics.hit.Desc().String(), "some_name_some_system_hit"))
}

View File

@ -44,12 +44,6 @@ var WithGCPeriod = func(gcPeriod time.Duration) Option {
}
}
var WithRefCounter = func(enable bool) Option {
return func(cache *oCache) {
cache.refCounter = enable
}
}
func New(loadFunc LoadFunc, opts ...Option) OCache {
c := &oCache{
data: make(map[string]*entry),
@ -73,33 +67,7 @@ func New(loadFunc LoadFunc, opts ...Option) OCache {
type Object interface {
Close() (err error)
}
type ObjectLocker interface {
Object
Locked() bool
}
type ObjectLastUsage interface {
LastUsage() time.Time
}
type entry struct {
id string
lastUsage time.Time
refCount uint32
isClosing bool
load chan struct{}
loadErr error
value Object
close chan struct{}
}
func (e *entry) locked() bool {
if locker, ok := e.value.(ObjectLocker); ok {
return locker.Locked()
}
return false
TryClose(objectTTL time.Duration) (res bool, err error)
}
type OCache interface {
@ -116,12 +84,8 @@ type OCache interface {
// Add adds new object to cache
// Returns error when object exists
Add(id string, value Object) (err error)
// Release decreases the refs counter
Release(id string) bool
// Reset sets refs counter to 0
Reset(id string) bool
// Remove closes and removes object
Remove(id string) (ok bool, err error)
Remove(ctx context.Context, id string) (ok bool, err error)
// ForEach iterates over all loaded objects, breaks when callback returns false
ForEach(f func(v Object) (isContinue bool))
// GC frees not used and expired objects
@ -144,7 +108,6 @@ type oCache struct {
closeCh chan struct{}
log *zap.SugaredLogger
metrics *metrics
refCounter bool
}
func (c *oCache) Get(ctx context.Context, id string) (value Object, err error) {
@ -160,69 +123,36 @@ Load:
return nil, ErrClosed
}
if e, ok = c.data[id]; !ok {
e = newEntry(id, nil, entryStateLoading)
load = true
e = &entry{
id: id,
load: make(chan struct{}),
}
c.data[id] = e
}
closing := e.isClosing
if !e.isClosing {
e.lastUsage = c.timeNow()
if c.refCounter {
e.refCount++
}
}
e.lastUsage = time.Now()
c.mu.Unlock()
if closing {
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object close", zap.String("id", id))
return nil, ctx.Err()
case <-e.close:
reload, err := e.waitClose(ctx, id)
if err != nil {
return nil, err
}
if reload {
goto Load
}
}
if load {
go c.load(ctx, id, e)
}
if c.metrics != nil {
if load {
c.metrics.miss.Inc()
} else {
c.metrics.hit.Inc()
}
}
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object load", zap.String("id", id))
return nil, ctx.Err()
case <-e.load:
}
return e.value, e.loadErr
c.metricsGet(!load)
return e.waitLoad(ctx, id)
}
func (c *oCache) Pick(ctx context.Context, id string) (value Object, err error) {
c.mu.Lock()
val, ok := c.data[id]
if !ok || val.isClosing {
if !ok || val.isClosing() {
c.mu.Unlock()
return nil, ErrNotExists
}
c.mu.Unlock()
if c.metrics != nil {
c.metrics.hit.Inc()
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-val.load:
return val.value, val.loadErr
}
c.metricsGet(true)
return val.waitLoad(ctx, id)
}
func (c *oCache) load(ctx context.Context, id string, e *entry) {
@ -236,63 +166,39 @@ func (c *oCache) load(ctx context.Context, id string, e *entry) {
delete(c.data, id)
} else {
e.value = value
e.setActive(false)
}
}
func (c *oCache) Release(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return false
}
if e, ok := c.data[id]; ok {
if c.refCounter && e.refCount > 0 {
e.refCount--
return true
}
}
return false
}
func (c *oCache) Reset(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return false
}
if e, ok := c.data[id]; ok {
e.refCount = 0
return true
}
return false
}
func (c *oCache) Remove(id string) (ok bool, err error) {
func (c *oCache) Remove(ctx context.Context, id string) (ok bool, err error) {
c.mu.Lock()
if c.closed {
c.mu.Unlock()
err = ErrClosed
return
}
var e *entry
e, ok = c.data[id]
if !ok || e.isClosing {
e, ok := c.data[id]
if !ok {
c.mu.Unlock()
return
return false, ErrNotExists
}
e.isClosing = true
e.close = make(chan struct{})
c.mu.Unlock()
return c.remove(ctx, e)
}
<-e.load
if e.value != nil {
err = e.value.Close()
func (c *oCache) remove(ctx context.Context, e *entry) (ok bool, err error) {
if _, err = e.waitLoad(ctx, e.id); err != nil {
return false, err
}
_, curState := e.setClosing(true)
if curState == entryStateClosing {
ok = true
err = e.value.Close()
c.mu.Lock()
close(e.close)
e.setClosed()
delete(c.data, e.id)
c.mu.Unlock()
}
return
}
@ -314,13 +220,7 @@ func (c *oCache) Add(id string, value Object) (err error) {
if _, ok := c.data[id]; ok {
return ErrExists
}
e := &entry{
id: id,
lastUsage: time.Now(),
refCount: 0,
load: make(chan struct{}),
value: value,
}
e := newEntry(id, value, entryStateActive)
close(e.load)
c.data[id] = e
return
@ -332,7 +232,7 @@ func (c *oCache) ForEach(f func(obj Object) (isContinue bool)) {
for _, v := range c.data {
select {
case <-v.load:
if v.value != nil && !v.isClosing {
if v.value != nil && !v.isClosing() {
objects = append(objects, v.value)
}
default:
@ -368,41 +268,36 @@ func (c *oCache) GC() {
deadline := c.timeNow().Add(-c.ttl)
var toClose []*entry
for _, e := range c.data {
if e.isClosing {
continue
}
lu := e.lastUsage
if lug, ok := e.value.(ObjectLastUsage); ok {
lu = lug.LastUsage()
}
if !e.locked() && e.refCount <= 0 && lu.Before(deadline) {
e.isClosing = true
if e.isActive() && e.lastUsage.Before(deadline) {
e.close = make(chan struct{})
toClose = append(toClose, e)
}
}
size := len(c.data)
c.mu.Unlock()
closedNum := 0
for _, e := range toClose {
<-e.load
if e.value != nil {
if err := e.value.Close(); err != nil {
prevState, _ := e.setClosing(false)
if prevState == entryStateClosing || prevState == entryStateClosed {
continue
}
closed, err := e.value.TryClose(c.ttl)
if err != nil {
c.log.With("object_id", e.id).Warnf("GC: object close error: %v", err)
}
}
}
c.log.Infof("GC: removed %d; cache size: %d", len(toClose), size)
if len(toClose) > 0 && c.metrics != nil {
c.metrics.gc.Add(float64(len(toClose)))
}
if !closed {
e.setActive(true)
continue
} else {
closedNum++
c.mu.Lock()
for _, e := range toClose {
close(e.close)
e.setClosed()
delete(c.data, e.id)
}
c.mu.Unlock()
}
}
c.metricsClosed(closedNum, size)
}
func (c *oCache) Len() int {
c.mu.Lock()
@ -418,25 +313,34 @@ func (c *oCache) Close() (err error) {
}
c.closed = true
close(c.closeCh)
var toClose, alreadyClosing []*entry
var toClose []*entry
for _, e := range c.data {
if e.isClosing {
alreadyClosing = append(alreadyClosing, e)
} else {
toClose = append(toClose, e)
}
}
c.mu.Unlock()
for _, e := range toClose {
<-e.load
if e.value != nil {
if clErr := e.value.Close(); clErr != nil {
c.log.With("object_id", e.id).Warnf("cache close: object close error: %v", clErr)
if _, err := c.remove(context.Background(), e); err != nil && err != ErrNotExists {
c.log.With("object_id", e.id).Warnf("cache close: object close error: %v", err)
}
}
}
for _, e := range alreadyClosing {
<-e.close
}
return nil
}
func (c *oCache) metricsGet(hit bool) {
if c.metrics == nil {
return
}
if hit {
c.metrics.hit.Inc()
} else {
c.metrics.miss.Inc()
}
}
func (c *oCache) metricsClosed(closedLen, size int) {
c.log.Infof("GC: removed %d; cache size: %d", closedLen, size)
if c.metrics == nil || closedLen == 0 {
return
}
c.metrics.gc.Add(float64(closedLen))
}

View File

@ -3,6 +3,8 @@ package ocache
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
@ -11,26 +13,48 @@ import (
"github.com/stretchr/testify/require"
)
var ctx = context.Background()
type testObject struct {
name string
closeErr error
closeCh chan struct{}
tryReturn bool
closeCalled bool
tryCloseCalled bool
}
func NewTestObject(name string, closeCh chan struct{}) *testObject {
func NewTestObject(name string, tryReturn bool, closeCh chan struct{}) *testObject {
return &testObject{
name: name,
closeCh: closeCh,
tryReturn: tryReturn,
}
}
func (t *testObject) Close() (err error) {
if t.closeCalled || (t.tryCloseCalled && t.tryReturn) {
panic("close called twice")
}
t.closeCalled = true
if t.closeCh != nil {
<-t.closeCh
}
return t.closeErr
}
func (t *testObject) TryClose(objectTTL time.Duration) (res bool, err error) {
if t.closeCalled || (t.tryCloseCalled && t.tryReturn) {
panic("close called twice")
}
t.tryCloseCalled = true
if t.closeCh != nil {
<-t.closeCh
return t.tryReturn, t.closeErr
}
return t.tryReturn, nil
}
func TestOCache_Get(t *testing.T) {
t.Run("successful", func(t *testing.T) {
c := New(func(ctx context.Context, id string) (value Object, err error) {
@ -116,42 +140,37 @@ func TestOCache_Get(t *testing.T) {
}
func TestOCache_GC(t *testing.T) {
t.Run("test without close wait", func(t *testing.T) {
t.Run("test gc expired object", func(t *testing.T) {
c := New(func(ctx context.Context, id string) (value Object, err error) {
return &testObject{name: id}, nil
}, WithTTL(time.Millisecond*10), WithRefCounter(true))
return NewTestObject(id, true, nil), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
c.GC()
assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 30)
c.GC()
assert.Equal(t, 1, c.Len())
assert.True(t, c.Release("id"))
time.Sleep(time.Millisecond * 20)
c.GC()
assert.Equal(t, 0, c.Len())
assert.False(t, c.Release("id"))
})
t.Run("test with close wait", func(t *testing.T) {
t.Run("test gc tryClose true, close before get", func(t *testing.T) {
closeCh := make(chan struct{})
getCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, closeCh), nil
}, WithTTL(time.Millisecond*10), WithRefCounter(true))
return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
assert.True(t, c.Release("id"))
// making ttl pass
time.Sleep(time.Millisecond * 40)
time.Sleep(time.Millisecond * 20)
// first gc will be run after 20 secs, so calling it manually
go c.GC()
// waiting until all objects are marked as closing
time.Sleep(time.Millisecond * 40)
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
_, err := c.Get(context.TODO(), "id")
@ -160,33 +179,114 @@ func TestOCache_GC(t *testing.T) {
events = append(events, "get")
close(getCh)
}()
events = append(events, "close")
// sleeping to make sure that Get is called
time.Sleep(time.Millisecond * 40)
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-getCh
require.Equal(t, []string{"close", "get"}, events)
})
t.Run("test gc tryClose false, many parallel get", func(t *testing.T) {
timesCalled := &atomic.Int32{}
obj := NewTestObject("id", false, nil)
c := New(func(ctx context.Context, id string) (value Object, err error) {
timesCalled.Add(1)
return obj, nil
}, WithTTL(0))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
begin := make(chan struct{})
wg := sync.WaitGroup{}
once := sync.Once{}
wg.Add(1)
go func() {
<-begin
c.GC()
wg.Done()
}()
for i := 0; i < 50; i++ {
wg.Add(1)
go func(i int) {
once.Do(func() {
close(begin)
})
if i%2 != 0 {
time.Sleep(time.Millisecond)
}
_, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
wg.Done()
}(i)
}
require.NoError(t, err)
wg.Wait()
require.Equal(t, timesCalled.Load(), int32(1))
require.True(t, obj.tryCloseCalled)
})
t.Run("test gc tryClose different, many objects", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
called := make(map[string]int)
max := 1000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
called[id] = called[id] + 1
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Millisecond*10))
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
assert.Equal(t, max, c.Len())
time.Sleep(time.Millisecond * 20)
c.GC()
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
require.Equal(t, called[getId(i)], i%2+1)
}
})
}
func Test_OCache_Remove(t *testing.T) {
t.Run("remove simple", func(t *testing.T) {
closeCh := make(chan struct{})
getCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, closeCh), nil
return NewTestObject(id, false, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
// removing the object, so we will wait on closing
go func() {
_, err := c.Remove("id")
_, err := c.Remove(ctx, "id")
require.NoError(t, err)
}()
time.Sleep(time.Millisecond * 40)
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
@ -196,11 +296,196 @@ func Test_OCache_Remove(t *testing.T) {
events = append(events, "get")
close(getCh)
}()
events = append(events, "close")
// sleeping to make sure that Get is called
time.Sleep(time.Millisecond * 40)
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-getCh
require.Equal(t, []string{"close", "get"}, events)
})
t.Run("test remove while gc, tryClose false", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, false, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 20)
go c.GC()
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.True(t, ok)
events = append(events, "remove")
close(removeCh)
}()
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-removeCh
require.Equal(t, []string{"close", "remove"}, events)
})
t.Run("test remove while gc, tryClose true", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 20)
go c.GC()
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.False(t, ok)
events = append(events, "remove")
close(removeCh)
}()
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-removeCh
require.Equal(t, []string{"close", "remove"}, events)
})
t.Run("test gc while remove, tryClose true", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.True(t, ok)
close(removeCh)
}()
time.Sleep(20 * time.Millisecond)
c.GC()
close(closeCh)
<-removeCh
})
}
func TestOCacheFuzzy(t *testing.T) {
t.Run("test many objects gc, get and remove simultaneously, close after", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
max := 2000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Nanosecond))
stopGC := make(chan struct{})
wg := sync.WaitGroup{}
go func() {
for {
select {
case <-stopGC:
return
default:
c.GC()
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
c.Remove(ctx, getId(i))
}
}
}()
wg.Wait()
close(stopGC)
err := c.Close()
require.NoError(t, err)
require.Equal(t, 0, c.Len())
})
t.Run("test many objects gc, get, remove and close simultaneously", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
max := 2000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Nanosecond))
go func() {
for {
c.GC()
}
}()
go func() {
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
if err == ErrClosed {
return
}
require.NoError(t, err)
require.NotNil(t, val)
}
}
}()
go func() {
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
c.Remove(ctx, getId(i))
}
}
}()
time.Sleep(time.Millisecond)
err := c.Close()
require.NoError(t, err)
require.Equal(t, 0, c.Len())
})
}

View File

@ -0,0 +1,52 @@
//go:generate mockgen -destination mock_credentialprovider/mock_credentialprovider.go github.com/anytypeio/any-sync/commonspace/credentialprovider CredentialProvider
package credentialprovider
import (
"context"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/coordinator/coordinatorclient"
"github.com/gogo/protobuf/proto"
)
const CName = "common.commonspace.credentialprovider"
func New() app.Component {
return &credentialProvider{}
}
func NewNoOp() CredentialProvider {
return &noOpProvider{}
}
type CredentialProvider interface {
GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error)
}
type noOpProvider struct {
}
func (n noOpProvider) GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
return nil, nil
}
type credentialProvider struct {
client coordinatorclient.CoordinatorClient
}
func (c *credentialProvider) Init(a *app.App) (err error) {
c.client = a.MustComponent(coordinatorclient.CName).(coordinatorclient.CoordinatorClient)
return
}
func (c *credentialProvider) Name() (name string) {
return CName
}
func (c *credentialProvider) GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
receipt, err := c.client.SpaceSign(ctx, spaceHeader.Id, spaceHeader.RawHeader)
if err != nil {
return nil, err
}
return proto.Marshal(receipt)
}

View File

@ -0,0 +1,51 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/anytypeio/any-sync/commonspace/credentialprovider (interfaces: CredentialProvider)
// Package mock_credentialprovider is a generated GoMock package.
package mock_credentialprovider
import (
context "context"
reflect "reflect"
spacesyncproto "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
gomock "github.com/golang/mock/gomock"
)
// MockCredentialProvider is a mock of CredentialProvider interface.
type MockCredentialProvider struct {
ctrl *gomock.Controller
recorder *MockCredentialProviderMockRecorder
}
// MockCredentialProviderMockRecorder is the mock recorder for MockCredentialProvider.
type MockCredentialProviderMockRecorder struct {
mock *MockCredentialProvider
}
// NewMockCredentialProvider creates a new mock instance.
func NewMockCredentialProvider(ctrl *gomock.Controller) *MockCredentialProvider {
mock := &MockCredentialProvider{ctrl: ctrl}
mock.recorder = &MockCredentialProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockCredentialProvider) EXPECT() *MockCredentialProviderMockRecorder {
return m.recorder
}
// GetCredential mocks base method.
func (m *MockCredentialProvider) GetCredential(arg0 context.Context, arg1 *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetCredential", arg0, arg1)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetCredential indicates an expected call of GetCredential.
func (mr *MockCredentialProviderMockRecorder) GetCredential(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredential", reflect.TypeOf((*MockCredentialProvider)(nil).GetCredential), arg0, arg1)
}

View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/peermanager"
@ -33,6 +34,7 @@ func newDiffSyncer(
storage spacestorage.SpaceStorage,
clientFactory spacesyncproto.ClientFactory,
syncStatus syncstatus.StatusUpdater,
credentialProvider credentialprovider.CredentialProvider,
log logger.CtxLogger) DiffSyncer {
return &diffSyncer{
diff: diff,
@ -41,6 +43,7 @@ func newDiffSyncer(
storage: storage,
peerManager: peerManager,
clientFactory: clientFactory,
credentialProvider: credentialProvider,
log: log,
syncStatus: syncStatus,
}
@ -55,6 +58,7 @@ type diffSyncer struct {
clientFactory spacesyncproto.ClientFactory
log logger.CtxLogger
deletionState settingsstate.ObjectDeletionState
credentialProvider credentialprovider.CredentialProvider
syncStatus syncstatus.StatusUpdater
}
@ -86,6 +90,7 @@ func (d *diffSyncer) UpdateHeads(id string, heads []string) {
}
func (d *diffSyncer) Sync(ctx context.Context) error {
// TODO: split diffsyncer into components
st := time.Now()
// diffing with responsible peers according to configuration
peers, err := d.peerManager.GetResponsiblePeers(ctx)
@ -117,6 +122,10 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error)
newIds, changedIds, removedIds, err := d.diff.Diff(ctx, rdiff)
err = rpcerr.Unwrap(err)
if err != nil && err != spacesyncproto.ErrSpaceMissing {
if err == spacesyncproto.ErrSpaceIsDeleted {
d.log.Debug("got space deleted while syncing")
d.syncTrees(ctx, p.Id(), []string{d.storage.SpaceSettingsId()})
}
d.syncStatus.SetNodesOnline(p.Id(), false)
return fmt.Errorf("diff error: %v", err)
}
@ -192,6 +201,10 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, peerId string, cl
return
}
cred, err := d.credentialProvider.GetCredential(ctx, header)
if err != nil {
return
}
spacePayload := &spacesyncproto.SpacePayload{
SpaceHeader: header,
AclPayload: root.Payload,
@ -201,6 +214,7 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, peerId string, cl
}
_, err = cl.SpacePush(ctx, &spacesyncproto.SpacePushRequest{
Payload: spacePayload,
Credential: cred,
})
if err != nil {
return

View File

@ -1,11 +1,13 @@
package headsync
import (
"bytes"
"context"
"fmt"
"github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/ldiff/mock_ldiff"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider/mock_credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@ -30,6 +32,7 @@ type pushSpaceRequestMatcher struct {
spaceId string
aclRootId string
settingsId string
credential []byte
spaceHeader *spacesyncproto.RawSpaceHeaderWithId
}
@ -39,7 +42,7 @@ func (p pushSpaceRequestMatcher) Matches(x interface{}) bool {
return false
}
return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader && res.Payload.SpaceSettingsPayloadId == p.settingsId
return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader && res.Payload.SpaceSettingsPayloadId == p.settingsId && bytes.Equal(p.credential, res.Credential)
}
func (p pushSpaceRequestMatcher) String() string {
@ -48,6 +51,10 @@ func (p pushSpaceRequestMatcher) String() string {
type mockPeer struct{}
func (m mockPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, m.Close()
}
func (m mockPeer) Id() string {
return "mockId"
}
@ -83,11 +90,13 @@ func newPushSpaceRequestMatcher(
spaceId string,
aclRootId string,
settingsId string,
credential []byte,
spaceHeader *spacesyncproto.RawSpaceHeaderWithId) *pushSpaceRequestMatcher {
return &pushSpaceRequestMatcher{
spaceId: spaceId,
aclRootId: aclRootId,
settingsId: settingsId,
credential: credential,
spaceHeader: spaceHeader,
}
}
@ -106,11 +115,12 @@ func TestDiffSyncer_Sync(t *testing.T) {
factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient {
return clientMock
})
credentialProvider := mock_credentialprovider.NewMockCredentialProvider(ctrl)
delState := mock_settingsstate.NewMockObjectDeletionState(ctrl)
spaceId := "spaceId"
aclRootId := "aclRootId"
l := logger.NewNamed(spaceId)
diffSyncer := newDiffSyncer(spaceId, diffMock, peerManagerMock, cacheMock, stMock, factory, syncstatus.NewNoOpSyncStatus(), l)
diffSyncer := newDiffSyncer(spaceId, diffMock, peerManagerMock, cacheMock, stMock, factory, syncstatus.NewNoOpSyncStatus(), credentialProvider, l)
delState.EXPECT().AddObserver(gomock.Any())
diffSyncer.Init(delState)
@ -172,6 +182,7 @@ func TestDiffSyncer_Sync(t *testing.T) {
}
spaceHeader := &spacesyncproto.RawSpaceHeaderWithId{}
spaceSettingsId := "spaceSettingsId"
credential := []byte("credential")
peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()).
@ -189,15 +200,18 @@ func TestDiffSyncer_Sync(t *testing.T) {
aclStorageMock.EXPECT().
Root().
Return(aclRoot, nil)
credentialProvider.EXPECT().
GetCredential(gomock.Any(), spaceHeader).
Return(credential, nil)
clientMock.EXPECT().
SpacePush(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, settingsId, spaceHeader)).
SpacePush(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, settingsId, credential, spaceHeader)).
Return(nil, nil)
peerManagerMock.EXPECT().SendPeer(gomock.Any(), "mockId", gomock.Any())
require.NoError(t, diffSyncer.Sync(ctx))
})
t.Run("diff syncer sync other error", func(t *testing.T) {
t.Run("diff syncer sync unexpected", func(t *testing.T) {
peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()).
Return([]peer.Peer{mockPeer{}}, nil)
@ -207,4 +221,19 @@ func TestDiffSyncer_Sync(t *testing.T) {
require.NoError(t, diffSyncer.Sync(ctx))
})
t.Run("diff syncer sync space is deleted error", func(t *testing.T) {
peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()).
Return([]peer.Peer{mockPeer{}}, nil)
diffMock.EXPECT().
Diff(gomock.Any(), gomock.Eq(NewRemoteDiff(spaceId, clientMock))).
Return(nil, nil, nil, spacesyncproto.ErrSpaceIsDeleted)
stMock.EXPECT().SpaceSettingsId().Return("settingsId")
cacheMock.EXPECT().
GetTree(gomock.Any(), spaceId, "settingsId").
Return(nil, nil)
require.NoError(t, diffSyncer.Sync(ctx))
})
}

View File

@ -5,6 +5,7 @@ import (
"context"
"github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
@ -60,12 +61,13 @@ func NewHeadSync(
peerManager peermanager.PeerManager,
cache treegetter.TreeGetter,
syncStatus syncstatus.StatusUpdater,
credentialProvider credentialprovider.CredentialProvider,
log logger.CtxLogger) HeadSync {
diff := ldiff.New(16, 16)
l := log.With(zap.String("spaceId", spaceId))
factory := spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient)
syncer := newDiffSyncer(spaceId, diff, peerManager, cache, storage, factory, syncStatus, l)
syncer := newDiffSyncer(spaceId, diff, peerManager, cache, storage, factory, syncStatus, credentialProvider, l)
sync := func(ctx context.Context) (err error) {
// for clients cancelling the sync process
if spaceIsDeleted.Load() && !configuration.IsResponsible(spaceId) {

View File

@ -88,7 +88,7 @@ func (a *aclRecordBuilder) BuildUserJoin(acceptPrivKeyBytes []byte, encSymKeyByt
Identity: state.Identity(),
Data: marshalledJoin,
CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: time.Now().UnixNano(),
Timestamp: time.Now().Unix(),
}
marshalledRecord, err := aclRecord.Marshal()
if err != nil {

View File

@ -4,8 +4,10 @@ import (
"bytes"
"errors"
"fmt"
"hash/fnv"
"github.com/anytypeio/any-sync/app/logger"
aclrecordproto "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/keychain"
"github.com/anytypeio/any-sync/util/keys"
"github.com/anytypeio/any-sync/util/keys/asymmetric/encryptionkey"
@ -13,10 +15,9 @@ import (
"github.com/anytypeio/any-sync/util/keys/symmetric"
"github.com/gogo/protobuf/proto"
"go.uber.org/zap"
"hash/fnv"
)
var log = logger.NewNamed("acllist").Sugar()
var log = logger.NewNamedSugared("common.commonspace.acllist")
var (
ErrNoSuchUser = errors.New("no such user")

View File

@ -126,7 +126,7 @@ func (t *AclListStorageBuilder) parseRecord(rec *Record, prevId string) *aclreco
Identity: []byte(t.keychain.GetIdentity(rec.Identity)),
Data: bytes,
CurrentReadKeyHash: k.Hash,
Timestamp: time.Now().UnixNano(),
Timestamp: time.Now().Unix(),
}
}

View File

@ -3,6 +3,7 @@ package objecttree
import (
"errors"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/gogo/protobuf/proto"
)
var (
@ -48,6 +49,11 @@ func NewChange(id string, ch *treechangeproto.TreeChange, signature []byte) *Cha
}
func NewChangeFromRoot(id string, ch *treechangeproto.RootChange, signature []byte) *Change {
changeInfo := &treechangeproto.TreeChangeInfo{
ChangeType: ch.ChangeType,
ChangePayload: ch.ChangePayload,
}
data, _ := proto.Marshal(changeInfo)
return &Change{
Next: nil,
AclHeadId: ch.AclHeadId,
@ -56,7 +62,8 @@ func NewChangeFromRoot(id string, ch *treechangeproto.RootChange, signature []by
Timestamp: ch.Timestamp,
Identity: string(ch.Identity),
Signature: signature,
Data: []byte(ch.ChangeType),
Data: data,
Model: changeInfo,
}
}

View File

@ -32,6 +32,7 @@ type InitialContent struct {
SpaceId string
Seed []byte
ChangeType string
ChangePayload []byte
Timestamp int64
}
@ -126,37 +127,31 @@ func (c *changeBuilder) BuildRoot(payload InitialContent) (ch *Change, rawIdChan
Timestamp: payload.Timestamp,
Identity: payload.Identity,
ChangeType: payload.ChangeType,
ChangePayload: payload.ChangePayload,
SpaceId: payload.SpaceId,
Seed: payload.Seed,
}
marshalledChange, err := proto.Marshal(change)
if err != nil {
return
}
signature, err := payload.SigningKey.Sign(marshalledChange)
if err != nil {
return
}
raw := &treechangeproto.RawTreeChange{
Payload: marshalledChange,
Signature: signature,
}
marshalledRawChange, err := proto.Marshal(raw)
if err != nil {
return
}
id, err := cidutil.NewCidFromBytes(marshalledRawChange)
if err != nil {
return
}
ch = NewChangeFromRoot(id, change, signature)
rawIdChange = &treechangeproto.RawTreeChangeWithId{
RawChange: marshalledRawChange,
Id: id,
@ -170,7 +165,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
AclHeadId: payload.AclHeadId,
SnapshotBaseId: payload.SnapshotBaseId,
CurrentReadKeyHash: payload.CurrentReadKeyHash,
Timestamp: time.Now().UnixNano(),
Timestamp: time.Now().Unix(),
Identity: payload.Identity,
IsSnapshot: payload.IsSnapshot,
}
@ -184,34 +179,27 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
} else {
change.ChangesData = payload.Content
}
marshalledChange, err := proto.Marshal(change)
if err != nil {
return
}
signature, err := payload.SigningKey.Sign(marshalledChange)
if err != nil {
return
}
raw := &treechangeproto.RawTreeChange{
Payload: marshalledChange,
Signature: signature,
}
marshalledRawChange, err := proto.Marshal(raw)
if err != nil {
return
}
id, err := cidutil.NewCidFromBytes(marshalledRawChange)
if err != nil {
return
}
ch = NewChange(id, change, signature)
rawIdChange = &treechangeproto.RawTreeChangeWithId{
RawChange: marshalledRawChange,
Id: id,
@ -220,7 +208,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
}
func (c *changeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) {
if ch.Id == c.rootChange.Id {
if c.isRoot(ch.Id) {
return c.rootChange, nil
}
treeChange := &treechangeproto.TreeChange{
@ -255,7 +243,7 @@ func (c *changeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChange
}
func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, id string) (ch *Change, err error) {
if c.rootChange.Id == id {
if c.isRoot(id) {
unmarshalled := &treechangeproto.RootChange{}
err = proto.Unmarshal(raw.Payload, unmarshalled)
if err != nil {
@ -264,7 +252,6 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
ch = NewChangeFromRoot(id, unmarshalled, raw.Signature)
return
}
unmarshalled := &treechangeproto.TreeChange{}
err = proto.Unmarshal(raw.Payload, unmarshalled)
if err != nil {
@ -274,3 +261,10 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
ch = NewChange(id, unmarshalled, raw.Signature)
return
}
func (c *changeBuilder) isRoot(id string) bool {
if c.rootChange != nil {
return c.rootChange.Id == id
}
return false
}

View File

@ -7,6 +7,7 @@ package mock_objecttree
import (
context "context"
reflect "reflect"
time "time"
list "github.com/anytypeio/any-sync/commonspace/object/acl/list"
objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -82,6 +83,20 @@ func (mr *MockObjectTreeMockRecorder) AddRawChanges(arg0, arg1 interface{}) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockObjectTree)(nil).AddRawChanges), arg0, arg1)
}
// ChangeInfo mocks base method.
func (m *MockObjectTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeInfo")
ret0, _ := ret[0].(*treechangeproto.TreeChangeInfo)
return ret0
}
// ChangeInfo indicates an expected call of ChangeInfo.
func (mr *MockObjectTreeMockRecorder) ChangeInfo() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeInfo", reflect.TypeOf((*MockObjectTree)(nil).ChangeInfo))
}
// ChangesAfterCommonSnapshot mocks base method.
func (m *MockObjectTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
m.ctrl.T.Helper()
@ -336,6 +351,21 @@ func (mr *MockObjectTreeMockRecorder) Storage() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Storage", reflect.TypeOf((*MockObjectTree)(nil).Storage))
}
// TryClose mocks base method.
func (m *MockObjectTree) TryClose(arg0 time.Duration) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TryClose", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// TryClose indicates an expected call of TryClose.
func (mr *MockObjectTreeMockRecorder) TryClose(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryClose", reflect.TypeOf((*MockObjectTree)(nil).TryClose), arg0)
}
// TryLock mocks base method.
func (m *MockObjectTree) TryLock() bool {
m.ctrl.T.Helper()

View File

@ -5,6 +5,7 @@ import (
"context"
"errors"
"sync"
"time"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
@ -52,6 +53,7 @@ type ReadableObjectTree interface {
Id() string
Header() *treechangeproto.RawTreeChangeWithId
UnmarshalledHeader() *Change
ChangeInfo() *treechangeproto.TreeChangeInfo
Heads() []string
Root() *Change
@ -81,6 +83,7 @@ type ObjectTree interface {
Delete() error
Close() error
TryClose(objectTTL time.Duration) (bool, error)
}
type objectTree struct {
@ -142,6 +145,10 @@ func (ot *objectTree) UnmarshalledHeader() *Change {
return ot.root
}
func (ot *objectTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
return ot.root.Model.(*treechangeproto.TreeChangeInfo)
}
func (ot *objectTree) Storage() treestorage.TreeStorage {
return ot.treeStorage
}
@ -555,6 +562,10 @@ func (ot *objectTree) Root() *Change {
return ot.tree.Root()
}
func (ot *objectTree) TryClose(objectTTL time.Duration) (bool, error) {
return true, ot.Close()
}
func (ot *objectTree) Close() error {
return nil
}

View File

@ -14,6 +14,7 @@ import (
type ObjectTreeCreatePayload struct {
SignKey signingkey.PrivKey
ChangeType string
ChangePayload []byte
SpaceId string
Identity []byte
IsEncrypted bool
@ -75,7 +76,7 @@ func CreateObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList)
if err != nil {
return
}
return createObjectTreeRoot(payload, time.Now().UnixNano(), bytes, aclList)
return createObjectTreeRoot(payload, time.Now().Unix(), bytes, aclList)
}
func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) (root *treechangeproto.RawTreeChangeWithId, err error) {
@ -125,7 +126,7 @@ func CreateObjectTree(
if err != nil {
return
}
return createObjectTree(payload, time.Now().UnixNano(), bytes, aclList, createStorage)
return createObjectTree(payload, time.Now().Unix(), bytes, aclList, createStorage)
}
func createObjectTree(
@ -170,6 +171,7 @@ func createObjectTreeRoot(
SigningKey: payload.SignKey,
SpaceId: payload.SpaceId,
ChangeType: payload.ChangeType,
ChangePayload: payload.ChangePayload,
Timestamp: timestamp,
Seed: seed,
}

View File

@ -125,7 +125,7 @@ func TestTree_Hash(t *testing.T) {
}
func TestTree_AddFuzzy(t *testing.T) {
rand.Seed(time.Now().UnixNano())
rand.Seed(time.Now().Unix())
getChanges := func() []*Change {
changes := []*Change{
newChange("1", "root", "root"),

View File

@ -4,15 +4,16 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/util/slice"
"go.uber.org/zap"
"time"
)
var (
log = logger.NewNamed("acltree").Sugar()
log = logger.NewNamedSugared("common.commonspace.objecttree")
ErrEmpty = errors.New("logs empty")
)

View File

@ -7,6 +7,7 @@ package mock_synctree
import (
context "context"
reflect "reflect"
time "time"
list "github.com/anytypeio/any-sync/commonspace/object/acl/list"
objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -193,6 +194,20 @@ func (mr *MockSyncTreeMockRecorder) AddRawChanges(arg0, arg1 interface{}) *gomoc
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockSyncTree)(nil).AddRawChanges), arg0, arg1)
}
// ChangeInfo mocks base method.
func (m *MockSyncTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeInfo")
ret0, _ := ret[0].(*treechangeproto.TreeChangeInfo)
return ret0
}
// ChangeInfo indicates an expected call of ChangeInfo.
func (mr *MockSyncTreeMockRecorder) ChangeInfo() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeInfo", reflect.TypeOf((*MockSyncTree)(nil).ChangeInfo))
}
// ChangesAfterCommonSnapshot mocks base method.
func (m *MockSyncTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
m.ctrl.T.Helper()
@ -487,6 +502,21 @@ func (mr *MockSyncTreeMockRecorder) SyncWithPeer(arg0, arg1 interface{}) *gomock
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncWithPeer", reflect.TypeOf((*MockSyncTree)(nil).SyncWithPeer), arg0, arg1)
}
// TryClose mocks base method.
func (m *MockSyncTree) TryClose(arg0 time.Duration) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TryClose", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// TryClose indicates an expected call of TryClose.
func (mr *MockSyncTreeMockRecorder) TryClose(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryClose", reflect.TypeOf((*MockSyncTree)(nil).TryClose), arg0)
}
// TryLock mocks base method.
func (m *MockSyncTree) TryLock() bool {
m.ctrl.T.Helper()

View File

@ -3,6 +3,8 @@ package synctree
import (
"context"
"errors"
"time"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -10,7 +12,7 @@ import (
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/objectsync"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/syncstatus"
"github.com/anytypeio/any-sync/net/peer"
"github.com/anytypeio/any-sync/nodeconf"
@ -50,7 +52,7 @@ type syncTree struct {
isDeleted bool
}
var log = logger.NewNamed("commonspace.synctree")
var log = logger.NewNamed("common.commonspace.synctree")
var buildObjectTree = objecttree.BuildObjectTree
var createSyncClient = newSyncClient
@ -208,6 +210,10 @@ func (s *syncTree) Delete() (err error) {
return
}
func (s *syncTree) TryClose(objectTTL time.Duration) (bool, error) {
return true, s.Close()
}
func (s *syncTree) Close() (err error) {
log.Debug("closing sync tree", zap.String("id", s.Id()))
defer func() {

View File

@ -16,6 +16,8 @@ message RootChange {
bytes seed = 5;
// Identity is a public key of the tree's creator
bytes identity = 6;
// ChangePayload is a payload related to ChangeType
bytes changePayload = 7;
}
// TreeChange is a change of a tree
@ -94,3 +96,9 @@ message TreeFullSyncResponse {
message TreeErrorResponse {
string error = 1;
}
// TreeChangeInfo is used internally in Tree implementation for ease of marshalling
message TreeChangeInfo {
string changeType = 1;
bytes changePayload = 2;
}

View File

@ -36,6 +36,8 @@ type RootChange struct {
Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"`
// Identity is a public key of the tree's creator
Identity []byte `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"`
// ChangePayload is a payload related to ChangeType
ChangePayload []byte `protobuf:"bytes,7,opt,name=changePayload,proto3" json:"changePayload,omitempty"`
}
func (m *RootChange) Reset() { *m = RootChange{} }
@ -113,6 +115,13 @@ func (m *RootChange) GetIdentity() []byte {
return nil
}
func (m *RootChange) GetChangePayload() []byte {
if m != nil {
return m.ChangePayload
}
return nil
}
// TreeChange is a change of a tree
type TreeChange struct {
// TreeHeadIds are previous ids for this TreeChange
@ -725,6 +734,59 @@ func (m *TreeErrorResponse) GetError() string {
return ""
}
// TreeChangeInfo is used internally in Tree implementation for ease of marshalling
type TreeChangeInfo struct {
ChangeType string `protobuf:"bytes,1,opt,name=changeType,proto3" json:"changeType,omitempty"`
ChangePayload []byte `protobuf:"bytes,2,opt,name=changePayload,proto3" json:"changePayload,omitempty"`
}
func (m *TreeChangeInfo) Reset() { *m = TreeChangeInfo{} }
func (m *TreeChangeInfo) String() string { return proto.CompactTextString(m) }
func (*TreeChangeInfo) ProtoMessage() {}
func (*TreeChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_5033f0301ef9b772, []int{10}
}
func (m *TreeChangeInfo) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *TreeChangeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_TreeChangeInfo.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *TreeChangeInfo) XXX_Merge(src proto.Message) {
xxx_messageInfo_TreeChangeInfo.Merge(m, src)
}
func (m *TreeChangeInfo) XXX_Size() int {
return m.Size()
}
func (m *TreeChangeInfo) XXX_DiscardUnknown() {
xxx_messageInfo_TreeChangeInfo.DiscardUnknown(m)
}
var xxx_messageInfo_TreeChangeInfo proto.InternalMessageInfo
func (m *TreeChangeInfo) GetChangeType() string {
if m != nil {
return m.ChangeType
}
return ""
}
func (m *TreeChangeInfo) GetChangePayload() []byte {
if m != nil {
return m.ChangePayload
}
return nil
}
func init() {
proto.RegisterType((*RootChange)(nil), "treechange.RootChange")
proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange")
@ -736,6 +798,7 @@ func init() {
proto.RegisterType((*TreeFullSyncRequest)(nil), "treechange.TreeFullSyncRequest")
proto.RegisterType((*TreeFullSyncResponse)(nil), "treechange.TreeFullSyncResponse")
proto.RegisterType((*TreeErrorResponse)(nil), "treechange.TreeErrorResponse")
proto.RegisterType((*TreeChangeInfo)(nil), "treechange.TreeChangeInfo")
}
func init() {
@ -743,48 +806,51 @@ func init() {
}
var fileDescriptor_5033f0301ef9b772 = []byte{
// 656 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xc1, 0x6e, 0xd3, 0x40,
0x10, 0xf5, 0x3a, 0x69, 0xd3, 0x4e, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x07, 0x08,
0x97, 0x56, 0x2a, 0x27, 0x10, 0x52, 0x45, 0x4b, 0x8b, 0xab, 0x0a, 0x84, 0xb6, 0x05, 0x24, 0x6e,
0x5b, 0x7b, 0x68, 0x8c, 0x12, 0xdb, 0x78, 0x37, 0x54, 0xf9, 0x00, 0x2e, 0x20, 0x21, 0x3e, 0x81,
0x6f, 0xe0, 0x0f, 0xb8, 0x71, 0xec, 0x91, 0x23, 0x6a, 0x7e, 0x04, 0xed, 0x3a, 0x4e, 0xd6, 0x6e,
0x0e, 0xbd, 0xf5, 0xb2, 0xc9, 0xbc, 0x9d, 0x79, 0xfb, 0xe6, 0xcd, 0x6e, 0x02, 0x3b, 0x61, 0xda,
0xef, 0xa7, 0x89, 0xc8, 0x78, 0x88, 0x5b, 0xe9, 0xe9, 0x47, 0x0c, 0xe5, 0x96, 0xcc, 0x11, 0xf5,
0x12, 0x76, 0x79, 0x72, 0x86, 0x59, 0x9e, 0xca, 0x74, 0x4b, 0xaf, 0xc2, 0x80, 0x37, 0x35, 0x42,
0x61, 0x8a, 0xf8, 0xbf, 0x08, 0x00, 0x4b, 0x53, 0xb9, 0xa7, 0x43, 0x7a, 0x07, 0x16, 0x79, 0xd8,
0x0b, 0x90, 0x47, 0x87, 0x91, 0x43, 0x3c, 0xd2, 0x59, 0x64, 0x53, 0x80, 0x3a, 0xd0, 0xd2, 0xa7,
0x1e, 0x46, 0x8e, 0xad, 0xf7, 0xca, 0x90, 0xba, 0x00, 0x05, 0xe1, 0xc9, 0x30, 0x43, 0xa7, 0xa1,
0x37, 0x0d, 0x44, 0xf1, 0xca, 0xb8, 0x8f, 0x42, 0xf2, 0x7e, 0xe6, 0x34, 0x3d, 0xd2, 0x69, 0xb0,
0x29, 0x40, 0x29, 0x34, 0x05, 0x62, 0xe4, 0xcc, 0x79, 0xa4, 0xd3, 0x66, 0xfa, 0x3b, 0xdd, 0x80,
0x85, 0x38, 0xc2, 0x44, 0xc6, 0x72, 0xe8, 0xcc, 0x6b, 0x7c, 0x12, 0xfb, 0x3f, 0x6d, 0x80, 0x93,
0x1c, 0x71, 0x2c, 0xda, 0x83, 0x25, 0xd5, 0x51, 0x21, 0x52, 0x38, 0xc4, 0x6b, 0x74, 0x16, 0x99,
0x09, 0x55, 0xdb, 0xb2, 0xeb, 0x6d, 0xdd, 0x87, 0x15, 0x91, 0xf0, 0x4c, 0x74, 0x53, 0xb9, 0xcb,
0x85, 0xea, 0xae, 0x68, 0xa0, 0x86, 0xaa, 0x73, 0x8a, 0x96, 0xc4, 0x73, 0x2e, 0xb9, 0x6e, 0xa3,
0xcd, 0x4c, 0x88, 0x6e, 0x02, 0x0d, 0x07, 0x79, 0x8e, 0x89, 0x64, 0xc8, 0xa3, 0x23, 0x1c, 0x06,
0x5c, 0x74, 0x75, 0x5b, 0x4d, 0x36, 0x63, 0xa7, 0x6a, 0xcb, 0x7c, 0xdd, 0x16, 0xd3, 0x82, 0x56,
0xd5, 0x02, 0x65, 0x78, 0x2c, 0x8e, 0xc7, 0xfa, 0x9c, 0x05, 0x8f, 0x74, 0x16, 0x98, 0x81, 0xf8,
0x2f, 0x60, 0x99, 0xf1, 0x73, 0xc3, 0x24, 0x07, 0x5a, 0x19, 0x1f, 0xf6, 0x52, 0x5e, 0xcc, 0xb5,
0xcd, 0xca, 0x50, 0x89, 0x10, 0xf1, 0x59, 0xc2, 0xe5, 0x20, 0x47, 0x6d, 0x4e, 0x9b, 0x4d, 0x01,
0x7f, 0x0f, 0xd6, 0x2a, 0x44, 0xef, 0x62, 0xd9, 0x3d, 0xd4, 0x45, 0x39, 0x3f, 0x2f, 0xa0, 0x31,
0xe1, 0x14, 0xa0, 0x2b, 0x60, 0xc7, 0xa5, 0xd1, 0x76, 0x1c, 0xf9, 0xdf, 0x09, 0xac, 0x2a, 0x8a,
0xe3, 0x61, 0x12, 0xbe, 0x44, 0x21, 0xf8, 0x19, 0xd2, 0x27, 0xd0, 0x0a, 0xd3, 0x44, 0x62, 0x22,
0x75, 0xfd, 0xd2, 0xb6, 0xb7, 0x69, 0xdc, 0xd4, 0x32, 0x7b, 0xaf, 0x48, 0x79, 0xcb, 0x7b, 0x03,
0x64, 0x65, 0x01, 0xdd, 0x01, 0xc8, 0x27, 0x97, 0x56, 0x9f, 0xb3, 0xb4, 0x7d, 0xcf, 0x2c, 0x9f,
0x21, 0x99, 0x19, 0x25, 0xfe, 0x6f, 0x1b, 0xd6, 0x67, 0x1d, 0x41, 0x9f, 0x02, 0x74, 0x91, 0x47,
0x6f, 0xb2, 0x88, 0x4b, 0x1c, 0x0b, 0xdb, 0xa8, 0x0b, 0x0b, 0x26, 0x19, 0x81, 0xc5, 0x8c, 0x7c,
0x7a, 0x04, 0xab, 0x1f, 0x06, 0xbd, 0x9e, 0x62, 0x65, 0xf8, 0x69, 0x80, 0x42, 0xce, 0x12, 0xa7,
0x28, 0x0e, 0xaa, 0x69, 0x81, 0xc5, 0xea, 0x95, 0xf4, 0x15, 0xdc, 0x9a, 0x42, 0x22, 0x4b, 0x13,
0x51, 0xbc, 0xac, 0x19, 0x4e, 0x1d, 0xd4, 0xf2, 0x02, 0x8b, 0x5d, 0xa9, 0xa5, 0xfb, 0xb0, 0x8c,
0x79, 0x9e, 0xe6, 0x13, 0xb2, 0xa6, 0x26, 0xbb, 0x5b, 0x27, 0xdb, 0x37, 0x93, 0x02, 0x8b, 0x55,
0xab, 0x76, 0x5b, 0x30, 0xf7, 0x59, 0x59, 0xe5, 0x7f, 0x21, 0xb0, 0x52, 0x75, 0x83, 0xae, 0xc3,
0x9c, 0x72, 0xa3, 0x7c, 0x83, 0x45, 0x40, 0x1f, 0x43, 0x6b, 0xfc, 0x48, 0x1c, 0xdb, 0x6b, 0x5c,
0x67, 0x54, 0x65, 0x3e, 0xf5, 0xa1, 0x5d, 0x3e, 0xc2, 0xd7, 0x5c, 0x76, 0x9d, 0x86, 0xe6, 0xad,
0x60, 0xfe, 0x57, 0x02, 0x6b, 0x33, 0x2c, 0xbd, 0x19, 0x31, 0xdf, 0x48, 0x71, 0xb1, 0xea, 0x13,
0xb9, 0x19, 0x35, 0x0f, 0xe1, 0xf6, 0x95, 0x89, 0x2a, 0x25, 0x7a, 0xa2, 0xe3, 0xdf, 0xf7, 0x22,
0xd8, 0x7d, 0xf6, 0xe7, 0xd2, 0x25, 0x17, 0x97, 0x2e, 0xf9, 0x77, 0xe9, 0x92, 0x1f, 0x23, 0xd7,
0xba, 0x18, 0xb9, 0xd6, 0xdf, 0x91, 0x6b, 0xbd, 0x7f, 0x70, 0xcd, 0xff, 0x9b, 0xd3, 0x79, 0xfd,
0xf1, 0xe8, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4e, 0xa5, 0xe2, 0x7c, 0xa1, 0x06, 0x00, 0x00,
// 690 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xc1, 0x4e, 0xdb, 0x4a,
0x14, 0xf5, 0x38, 0x01, 0xc3, 0x25, 0xc0, 0x7b, 0x03, 0x0b, 0x0b, 0xbd, 0xe7, 0x5a, 0x56, 0xd5,
0xa6, 0x1b, 0x90, 0xe8, 0xaa, 0x55, 0x25, 0x54, 0x28, 0xd4, 0x11, 0x6a, 0x85, 0x06, 0x4a, 0xa5,
0xee, 0x06, 0xfb, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0x99, 0x14, 0xe5, 0x03, 0xba, 0x69, 0xa5, 0xaa,
0x9f, 0xd0, 0x5f, 0xe9, 0xae, 0x4b, 0x96, 0x2c, 0x2b, 0xf8, 0x91, 0xca, 0x63, 0x3b, 0xb1, 0x1d,
0x2f, 0xd8, 0xb1, 0x71, 0x72, 0x8f, 0xef, 0x3d, 0x73, 0xee, 0xb9, 0x33, 0x63, 0xd8, 0xf1, 0xa2,
0xe1, 0x30, 0x0a, 0x45, 0xcc, 0x3d, 0xdc, 0x8a, 0xce, 0x3e, 0xa2, 0x27, 0xb7, 0x64, 0x82, 0xa8,
0x1e, 0x5e, 0x9f, 0x87, 0x17, 0x18, 0x27, 0x91, 0x8c, 0xb6, 0xd4, 0x53, 0x94, 0xe0, 0x4d, 0x85,
0x50, 0x98, 0x22, 0xce, 0x35, 0x01, 0x60, 0x51, 0x24, 0xf7, 0x54, 0x48, 0xff, 0x83, 0x45, 0xee,
0x0d, 0x5c, 0xe4, 0x7e, 0xcf, 0x37, 0x89, 0x4d, 0xba, 0x8b, 0x6c, 0x0a, 0x50, 0x13, 0x0c, 0xb5,
0x6a, 0xcf, 0x37, 0x75, 0xf5, 0xae, 0x08, 0xa9, 0x05, 0x90, 0x11, 0x9e, 0x8c, 0x63, 0x34, 0x5b,
0xea, 0x65, 0x09, 0x49, 0x79, 0x65, 0x30, 0x44, 0x21, 0xf9, 0x30, 0x36, 0xdb, 0x36, 0xe9, 0xb6,
0xd8, 0x14, 0xa0, 0x14, 0xda, 0x02, 0xd1, 0x37, 0xe7, 0x6c, 0xd2, 0xed, 0x30, 0xf5, 0x9f, 0x6e,
0xc0, 0x42, 0xe0, 0x63, 0x28, 0x03, 0x39, 0x36, 0xe7, 0x15, 0x3e, 0x89, 0xe9, 0x43, 0x58, 0xce,
0xb8, 0x8f, 0xf8, 0x78, 0x10, 0x71, 0xdf, 0x34, 0x54, 0x42, 0x15, 0x74, 0x7e, 0xea, 0x00, 0x27,
0x09, 0x62, 0xde, 0x9a, 0x0d, 0x4b, 0x69, 0xdf, 0x59, 0x2b, 0xc2, 0x24, 0x76, 0xab, 0xbb, 0xc8,
0xca, 0x50, 0xb5, 0x79, 0xbd, 0xde, 0xfc, 0x23, 0x58, 0x11, 0x21, 0x8f, 0x45, 0x3f, 0x92, 0xbb,
0x5c, 0xa4, 0x1e, 0x64, 0x6d, 0xd6, 0xd0, 0x74, 0x9d, 0x4c, 0x87, 0x78, 0xc5, 0x25, 0x57, 0xcd,
0x76, 0x58, 0x19, 0xa2, 0x9b, 0x40, 0xbd, 0x51, 0x92, 0x60, 0x28, 0x19, 0x72, 0xff, 0x10, 0xc7,
0x2e, 0x17, 0x7d, 0xd5, 0x7c, 0x9b, 0x35, 0xbc, 0xa9, 0x9a, 0x37, 0x5f, 0x37, 0xaf, 0x6c, 0x94,
0x51, 0x33, 0xca, 0x02, 0x08, 0xc4, 0x71, 0xae, 0xcf, 0x5c, 0xb0, 0x49, 0x77, 0x81, 0x95, 0x10,
0xe7, 0x35, 0x2c, 0x33, 0x7e, 0x59, 0x32, 0xc9, 0x04, 0x23, 0xce, 0x3d, 0x25, 0x8a, 0xab, 0x08,
0x53, 0x11, 0x22, 0xb8, 0x08, 0xb9, 0x1c, 0x25, 0xa8, 0xcc, 0xe9, 0xb0, 0x29, 0xe0, 0xec, 0xc1,
0x5a, 0x85, 0xe8, 0x7d, 0x20, 0xfb, 0x3d, 0x55, 0x94, 0xf0, 0xcb, 0x0c, 0xca, 0x09, 0xa7, 0x00,
0x5d, 0x01, 0x3d, 0x28, 0x8c, 0xd6, 0x03, 0xdf, 0xf9, 0x4e, 0x60, 0x35, 0xa5, 0x38, 0x1e, 0x87,
0xde, 0x1b, 0x14, 0x82, 0x5f, 0x20, 0x7d, 0x0e, 0x86, 0x17, 0x85, 0x12, 0x43, 0xa9, 0xea, 0x97,
0xb6, 0xed, 0xcd, 0xd2, 0x7e, 0x2e, 0xb2, 0xf7, 0xb2, 0x94, 0x53, 0x3e, 0x18, 0x21, 0x2b, 0x0a,
0xe8, 0x0e, 0x40, 0x32, 0xd9, 0xda, 0x6a, 0x9d, 0xa5, 0xed, 0x07, 0xe5, 0xf2, 0x06, 0xc9, 0xac,
0x54, 0xe2, 0xfc, 0xd2, 0x61, 0xbd, 0x69, 0x09, 0xfa, 0x02, 0xa0, 0x8f, 0xdc, 0x7f, 0x17, 0xfb,
0x5c, 0x62, 0x2e, 0x6c, 0xa3, 0x2e, 0xcc, 0x9d, 0x64, 0xb8, 0x1a, 0x2b, 0xe5, 0xd3, 0x43, 0x58,
0x3d, 0x1f, 0x0d, 0x06, 0x29, 0x2b, 0xc3, 0x4f, 0x23, 0x14, 0xb2, 0x49, 0x5c, 0x4a, 0x71, 0x50,
0x4d, 0x73, 0x35, 0x56, 0xaf, 0xa4, 0x6f, 0xe1, 0x9f, 0x29, 0x24, 0xe2, 0x28, 0x14, 0xd9, 0xf9,
0x6b, 0x70, 0xea, 0xa0, 0x96, 0xe7, 0x6a, 0x6c, 0xa6, 0x96, 0xee, 0xc3, 0x32, 0x26, 0x49, 0x94,
0x4c, 0xc8, 0xda, 0x8a, 0xec, 0xff, 0x3a, 0xd9, 0x7e, 0x39, 0xc9, 0xd5, 0x58, 0xb5, 0x6a, 0xd7,
0x80, 0xb9, 0xcf, 0xa9, 0x55, 0xce, 0x17, 0x02, 0x2b, 0x55, 0x37, 0xe8, 0x3a, 0xcc, 0xa5, 0x6e,
0x14, 0x67, 0x30, 0x0b, 0xe8, 0x33, 0x30, 0xf2, 0x43, 0x62, 0xea, 0x76, 0xeb, 0x2e, 0xa3, 0x2a,
0xf2, 0xa9, 0x03, 0x9d, 0xe2, 0x10, 0x1e, 0x71, 0xd9, 0x37, 0x5b, 0x8a, 0xb7, 0x82, 0x39, 0x5f,
0x09, 0xac, 0x35, 0x58, 0x7a, 0x3f, 0x62, 0xbe, 0x91, 0x6c, 0x63, 0xd5, 0x27, 0x72, 0x3f, 0x6a,
0x9e, 0xc0, 0xbf, 0x33, 0x13, 0x4d, 0x95, 0xa8, 0x89, 0xe6, 0x5f, 0x81, 0x2c, 0x70, 0x4e, 0xb3,
0x61, 0x66, 0x6b, 0xf5, 0xc2, 0xf3, 0xa8, 0x76, 0xf3, 0x93, 0x99, 0x9b, 0x7f, 0xe6, 0xae, 0xd6,
0x1b, 0xee, 0xea, 0xdd, 0x97, 0xbf, 0x6f, 0x2c, 0x72, 0x75, 0x63, 0x91, 0x3f, 0x37, 0x16, 0xf9,
0x71, 0x6b, 0x69, 0x57, 0xb7, 0x96, 0x76, 0x7d, 0x6b, 0x69, 0x1f, 0x1e, 0xdf, 0xf1, 0x6b, 0x77,
0x36, 0xaf, 0x7e, 0x9e, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x24, 0x93, 0x3b, 0x00, 0x1f, 0x07,
0x00, 0x00,
}
func (m *RootChange) Marshal() (dAtA []byte, err error) {
@ -807,6 +873,13 @@ func (m *RootChange) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.ChangePayload) > 0 {
i -= len(m.ChangePayload)
copy(dAtA[i:], m.ChangePayload)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangePayload)))
i--
dAtA[i] = 0x3a
}
if len(m.Identity) > 0 {
i -= len(m.Identity)
copy(dAtA[i:], m.Identity)
@ -1362,6 +1435,43 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *TreeChangeInfo) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *TreeChangeInfo) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *TreeChangeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.ChangePayload) > 0 {
i -= len(m.ChangePayload)
copy(dAtA[i:], m.ChangePayload)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangePayload)))
i--
dAtA[i] = 0x12
}
if len(m.ChangeType) > 0 {
i -= len(m.ChangeType)
copy(dAtA[i:], m.ChangeType)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangeType)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintTreechange(dAtA []byte, offset int, v uint64) int {
offset -= sovTreechange(v)
base := offset
@ -1402,6 +1512,10 @@ func (m *RootChange) Size() (n int) {
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
l = len(m.ChangePayload)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
return n
}
@ -1650,6 +1764,23 @@ func (m *TreeErrorResponse) Size() (n int) {
return n
}
func (m *TreeChangeInfo) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.ChangeType)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
l = len(m.ChangePayload)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
return n
}
func sovTreechange(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
@ -1868,6 +1999,40 @@ func (m *RootChange) Unmarshal(dAtA []byte) error {
m.Identity = []byte{}
}
iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangePayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangePayload = append(m.ChangePayload[:0], dAtA[iNdEx:postIndex]...)
if m.ChangePayload == nil {
m.ChangePayload = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTreechange(dAtA[iNdEx:])
@ -3233,6 +3398,122 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *TreeChangeInfo) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: TreeChangeInfo: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: TreeChangeInfo: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangeType", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangeType = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangePayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangePayload = append(m.ChangePayload[:0], dAtA[iNdEx:postIndex]...)
if m.ChangePayload == nil {
m.ChangePayload = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTreechange(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTreechange
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipTreechange(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0

View File

@ -3,7 +3,6 @@ package objectsync
import (
"context"
"fmt"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
"github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
@ -15,9 +14,13 @@ import (
"time"
)
type LastUsage interface {
LastUsage() time.Time
}
// MessagePool can be made generic to work with different streams
type MessagePool interface {
ocache.ObjectLastUsage
LastUsage
synchandler.SyncHandler
peermanager.PeerManager
SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error)

View File

@ -2,23 +2,24 @@ package objectsync
import (
"context"
"sync/atomic"
"time"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
"github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/nodeconf"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"sync/atomic"
"time"
)
var log = logger.NewNamed("commonspace.objectsync")
var log = logger.NewNamed("common.commonspace.objectsync")
type ObjectSync interface {
ocache.ObjectLastUsage
LastUsage
synchandler.SyncHandler
MessagePool() MessagePool
@ -32,6 +33,7 @@ type objectSync struct {
messagePool MessagePool
objectGetter syncobjectgetter.SyncObjectGetter
configuration nodeconf.Configuration
spaceStorage spacestorage.SpaceStorage
syncCtx context.Context
cancelSync context.CancelFunc
@ -43,13 +45,15 @@ func NewObjectSync(
spaceIsDeleted *atomic.Bool,
configuration nodeconf.Configuration,
peerManager peermanager.PeerManager,
objectGetter syncobjectgetter.SyncObjectGetter) ObjectSync {
objectGetter syncobjectgetter.SyncObjectGetter,
storage spacestorage.SpaceStorage) ObjectSync {
syncCtx, cancel := context.WithCancel(context.Background())
os := newObjectSync(
spaceId,
spaceIsDeleted,
configuration,
objectGetter,
storage,
syncCtx,
cancel)
msgPool := newMessagePool(peerManager, os.handleMessage)
@ -62,11 +66,13 @@ func newObjectSync(
spaceIsDeleted *atomic.Bool,
configuration nodeconf.Configuration,
objectGetter syncobjectgetter.SyncObjectGetter,
spaceStorage spacestorage.SpaceStorage,
syncCtx context.Context,
cancel context.CancelFunc,
) *objectSync {
return &objectSync{
objectGetter: objectGetter,
spaceStorage: spaceStorage,
spaceId: spaceId,
syncCtx: syncCtx,
cancelSync: cancel,
@ -95,8 +101,8 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp
log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId))
if s.spaceIsDeleted.Load() {
log = log.With(zap.Bool("isDeleted", true))
// preventing sync with other clients
if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) {
// preventing sync with other clients if they are not just syncing the settings tree
if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() {
return spacesyncproto.ErrSpaceIsDeleted
}
}

View File

@ -14,7 +14,7 @@ import (
)
const (
SpaceSettingsChangeType = "reserved.spacesettings"
SpaceReserved = "any-sync.space"
SpaceDerivationScheme = "derivation.standard"
)
@ -37,8 +37,9 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
}
header := &spacesyncproto.SpaceHeader{
Identity: identity,
Timestamp: time.Now().UnixNano(),
Timestamp: time.Now().Unix(),
SpaceType: payload.SpaceType,
SpaceHeaderPayload: payload.SpacePayload,
ReplicationKey: payload.ReplicationKey,
Seed: spaceHeaderSeed,
}
@ -81,7 +82,7 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
SpaceId: spaceId,
EncryptedReadKey: encReadKey,
CurrentReadKeyHash: readKeyHash,
Timestamp: time.Now().UnixNano(),
Timestamp: time.Now().Unix(),
}
rawWithId, err := marshalAclRoot(aclRoot, payload.SigningKey)
if err != nil {
@ -101,8 +102,8 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
SigningKey: payload.SigningKey,
SpaceId: spaceId,
Seed: spaceSettingsSeed,
ChangeType: SpaceSettingsChangeType,
Timestamp: time.Now().UnixNano(),
ChangeType: SpaceReserved,
Timestamp: time.Now().Unix(),
})
if err != nil {
return
@ -147,7 +148,8 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp
// preparing header and space id
header := &spacesyncproto.SpaceHeader{
Identity: identity,
SpaceType: SpaceTypeDerived,
SpaceType: payload.SpaceType,
SpaceHeaderPayload: payload.SpacePayload,
ReplicationKey: repKey,
}
marshalled, err := header.Marshal()
@ -201,7 +203,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp
Identity: aclRoot.Identity,
SigningKey: payload.SigningKey,
SpaceId: spaceId,
ChangeType: SpaceSettingsChangeType,
ChangeType: SpaceReserved,
})
if err != nil {
return

View File

@ -5,15 +5,17 @@ import (
"context"
"errors"
"fmt"
"github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/keychain"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/nodeconf"
"github.com/gogo/protobuf/proto"
@ -242,17 +244,7 @@ func (s *settingsObject) verifyDeleteSpace(raw *treechangeproto.RawTreeChangeWit
if err != nil {
return
}
content := &spacesyncproto.SettingsData{}
err = proto.Unmarshal(data, content)
if err != nil {
return
}
if len(content.GetContent()) != 1 ||
content.GetContent()[0].GetSpaceDelete() == nil ||
content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() == "" {
return fmt.Errorf("incorrect delete change payload")
}
return
return verifyDeleteContent(data, "")
}
func (s *settingsObject) addContent(data []byte) (err error) {
@ -271,3 +263,30 @@ func (s *settingsObject) addContent(data []byte) (err error) {
s.Update(s)
return
}
func VerifyDeleteChange(raw *treechangeproto.RawTreeChangeWithId, identity []byte, peerId string) (err error) {
changeBuilder := objecttree.NewChangeBuilder(keychain.NewKeychain(), nil)
res, err := changeBuilder.Unmarshall(raw, true)
if err != nil {
return
}
if res.Identity != string(identity) {
return fmt.Errorf("incorrect identity")
}
return verifyDeleteContent(res.Data, peerId)
}
func verifyDeleteContent(data []byte, peerId string) (err error) {
content := &spacesyncproto.SettingsData{}
err = proto.Unmarshal(data, content)
if err != nil {
return
}
if len(content.GetContent()) != 1 ||
content.GetContent()[0].GetSpaceDelete() == nil ||
(peerId == "" && content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() == "") ||
(peerId != "" && content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() != peerId) {
return fmt.Errorf("incorrect delete change payload")
}
return
}

View File

@ -48,7 +48,7 @@ func (s *stateBuilder) Build(tr objecttree.ObjectTree, oldState *State) (state *
func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, state *State) *State {
// ignoring root change which has empty model or startId change
if change.Model == nil || state.LastIteratedId == change.Id {
if len(change.PreviousIds) == 0 || state.LastIteratedId == change.Id {
return state
}

View File

@ -42,6 +42,7 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
t.Run("changeId is equal to rootId", func(t *testing.T) {
ch := &objecttree.Change{}
ch.PreviousIds = []string{"someId"}
ch.Model = &spacesyncproto.SettingsData{
Snapshot: &spacesyncproto.SpaceSettingsSnapshot{
DeletedIds: []string{"id1", "id2"},
@ -56,6 +57,7 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
t.Run("changeId is not equal to lastIteratedId or rootId", func(t *testing.T) {
ch := &objecttree.Change{}
ch.PreviousIds = []string{"someId"}
ch.Model = &spacesyncproto.SettingsData{
Content: []*spacesyncproto.SpaceSettingsContent{
{Value: &spacesyncproto.SpaceSettingsContent_ObjectDelete{

View File

@ -6,7 +6,6 @@ import (
"fmt"
"github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/headsync"
"github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/acl/syncacl"
@ -52,6 +51,8 @@ type SpaceCreatePayload struct {
ReadKey []byte
// ReplicationKey is a key which is to be used to determine the node where the space should be held
ReplicationKey uint64
// SpacePayload is an arbitrary payload related to space type
SpacePayload []byte
}
type HandleMessage struct {
@ -61,11 +62,11 @@ type HandleMessage struct {
Message *spacesyncproto.ObjectSyncMessage
}
const SpaceTypeDerived = "derived.space"
type SpaceDerivePayload struct {
SigningKey signingkey.PrivKey
EncryptionKey encryptionkey.PrivKey
SpaceType string
SpacePayload []byte
}
type SpaceDescription struct {
@ -81,9 +82,6 @@ func NewSpaceId(id string, repKey uint64) string {
}
type Space interface {
ocache.ObjectLocker
ocache.ObjectLastUsage
Id() string
Init(ctx context.Context) error
@ -108,6 +106,7 @@ type Space interface {
HandleMessage(ctx context.Context, msg HandleMessage) (err error)
TryClose(objectTTL time.Duration) (close bool, err error)
Close() error
}
@ -134,16 +133,6 @@ type space struct {
treesUsed *atomic.Int32
}
func (s *space) LastUsage() time.Time {
return s.objectSync.LastUsage()
}
func (s *space) Locked() bool {
locked := s.treesUsed.Load() > 1
log.With(zap.Int32("trees used", s.treesUsed.Load()), zap.Bool("locked", locked), zap.String("spaceId", s.id)).Debug("space lock status check")
return locked
}
func (s *space) Id() string {
return s.id
}
@ -306,7 +295,13 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea
SyncStatus: s.syncStatus,
PeerGetter: s.peerManager,
}
return synctree.PutSyncTree(ctx, payload, deps)
t, err = synctree.PutSyncTree(ctx, payload, deps)
if err != nil {
return
}
s.treesUsed.Add(1)
log.Debug("incrementing counter", zap.String("id", payload.RootRawChange.Id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
return
}
type BuildTreeOpts struct {
@ -341,8 +336,8 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t
if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil {
return nil, err
}
log.Debug("incrementing counter", zap.String("id", id), zap.String("spaceId", s.id))
s.treesUsed.Add(1)
log.Debug("incrementing counter", zap.String("id", id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
return
}
@ -417,8 +412,8 @@ func (s *space) handleMessage(msg HandleMessage) {
}
func (s *space) onObjectClose(id string) {
log.Debug("decrementing counter", zap.String("id", id), zap.String("spaceId", s.id))
s.treesUsed.Add(-1)
log.Debug("decrementing counter", zap.String("id", id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
_ = s.handleQueue.CloseThread(id)
}
@ -462,3 +457,15 @@ func (s *space) Close() error {
log.With(zap.String("id", s.id)).Debug("space closed")
return mError.Err()
}
func (s *space) TryClose(objectTTL time.Duration) (close bool, err error) {
if time.Now().Sub(s.objectSync.LastUsage()) < objectTTL {
return false, nil
}
locked := s.treesUsed.Load() > 1
log.With(zap.Int32("trees used", s.treesUsed.Load()), zap.Bool("locked", locked), zap.String("spaceId", s.id)).Debug("space lock status check")
if locked {
return false, nil
}
return true, s.Close()
}

View File

@ -5,6 +5,7 @@ import (
"github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/headsync"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@ -16,6 +17,7 @@ import (
"github.com/anytypeio/any-sync/commonspace/syncstatus"
"github.com/anytypeio/any-sync/net/peer"
"github.com/anytypeio/any-sync/net/pool"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
"github.com/anytypeio/any-sync/nodeconf"
"sync/atomic"
)
@ -34,6 +36,7 @@ const AddSpaceCtxKey ctxKey = 0
type SpaceService interface {
DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (string, error)
DeriveId(ctx context.Context, payload SpaceDerivePayload) (string, error)
CreateSpace(ctx context.Context, payload SpaceCreatePayload) (string, error)
NewSpace(ctx context.Context, id string) (sp Space, err error)
app.Component
@ -45,6 +48,7 @@ type spaceService struct {
configurationService nodeconf.Service
storageProvider spacestorage.SpaceStorageProvider
peermanagerProvider peermanager.PeerManagerProvider
credentialProvider credentialprovider.CredentialProvider
treeGetter treegetter.TreeGetter
pool pool.Pool
}
@ -56,6 +60,12 @@ func (s *spaceService) Init(a *app.App) (err error) {
s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service)
s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter)
s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider)
credProvider := a.Component(credentialprovider.CName)
if credProvider != nil {
s.credentialProvider = credProvider.(credentialprovider.CredentialProvider)
} else {
s.credentialProvider = credentialprovider.NewNoOp()
}
s.pool = a.MustComponent(pool.CName).(pool.Pool)
return nil
}
@ -80,6 +90,15 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload SpaceCreatePaylo
return store.Id(), nil
}
func (s *spaceService) DeriveId(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload)
if err != nil {
return
}
id = storageCreate.SpaceHeaderWithId.Id
return
}
func (s *spaceService) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload)
if err != nil {
@ -139,8 +158,8 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) {
return nil, err
}
headSync := headsync.NewHeadSync(id, spaceIsDeleted, s.config.SyncPeriod, lastConfiguration, st, peerManager, getter, syncStatus, log)
objectSync := objectsync.NewObjectSync(id, spaceIsDeleted, lastConfiguration, peerManager, getter)
headSync := headsync.NewHeadSync(id, spaceIsDeleted, s.config.SyncPeriod, lastConfiguration, st, peerManager, getter, syncStatus, s.credentialProvider, log)
objectSync := objectsync.NewObjectSync(id, spaceIsDeleted, lastConfiguration, peerManager, getter, st)
sp := &space{
id: id,
objectSync: objectSync,
@ -198,6 +217,7 @@ func (s *spaceService) getSpaceStorageFromRemote(ctx context.Context, id string)
cl := spacesyncproto.NewDRPCSpaceSyncClient(p)
res, err := cl.SpacePull(ctx, &spacesyncproto.SpacePullRequest{Id: id})
if err != nil {
err = rpcerr.Unwrap(err)
return
}

View File

@ -2,14 +2,20 @@
package spacestorage
import (
"bytes"
"context"
"errors"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/liststorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/util/cidutil"
"github.com/anytypeio/any-sync/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
"strings"
)
const CName = "common.commonspace.spacestorage"
@ -17,6 +23,7 @@ const CName = "common.commonspace.spacestorage"
var (
ErrSpaceStorageExists = errors.New("space storage exists")
ErrSpaceStorageMissing = errors.New("space storage missing")
ErrIncorrectSpaceHeader = errors.New("incorrect space header")
ErrTreeStorageAlreadyDeleted = errors.New("tree storage already deleted")
)
@ -63,3 +70,38 @@ func ValidateSpaceStorageCreatePayload(payload SpaceStorageCreatePayload) (err e
// TODO: add proper validation
return nil
}
func ValidateSpaceHeader(spaceId string, header, identity []byte) (err error) {
split := strings.Split(spaceId, ".")
if len(split) != 2 {
return ErrIncorrectSpaceHeader
}
if !cidutil.VerifyCid(header, split[0]) {
err = objecttree.ErrIncorrectCid
return
}
raw := &spacesyncproto.RawSpaceHeader{}
err = proto.Unmarshal(header, raw)
if err != nil {
return
}
payload := &spacesyncproto.SpaceHeader{}
err = proto.Unmarshal(raw.SpaceHeader, payload)
if err != nil {
return
}
if identity != nil && !bytes.Equal(identity, payload.Identity) {
err = ErrIncorrectSpaceHeader
return
}
key, err := signingkey.NewSigningEd25519PubKeyFromBytes(payload.Identity)
if err != nil {
return
}
res, err := key.Verify(raw.SpaceHeader, raw.Signature)
if err != nil || !res {
err = ErrIncorrectSpaceHeader
return
}
return
}

View File

@ -66,6 +66,7 @@ message ObjectSyncMessage {
// SpacePushRequest is a request to add space on a node containing only one acl record
message SpacePushRequest {
SpacePayload payload = 1;
bytes Credential = 2;
}
// SpacePushResponse is an empty response
@ -97,6 +98,7 @@ message SpaceHeader {
string spaceType = 3;
uint64 replicationKey = 4;
bytes seed = 5;
bytes spaceHeaderPayload = 6;
}
// RawSpaceHeader is raw header for SpaceHeader

View File

@ -438,6 +438,7 @@ func (m *ObjectSyncMessage) GetObjectId() string {
// SpacePushRequest is a request to add space on a node containing only one acl record
type SpacePushRequest struct {
Payload *SpacePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
Credential []byte `protobuf:"bytes,2,opt,name=Credential,proto3" json:"Credential,omitempty"`
}
func (m *SpacePushRequest) Reset() { *m = SpacePushRequest{} }
@ -480,6 +481,13 @@ func (m *SpacePushRequest) GetPayload() *SpacePayload {
return nil
}
func (m *SpacePushRequest) GetCredential() []byte {
if m != nil {
return m.Credential
}
return nil
}
// SpacePushResponse is an empty response
type SpacePushResponse struct {
}
@ -691,6 +699,7 @@ type SpaceHeader struct {
SpaceType string `protobuf:"bytes,3,opt,name=spaceType,proto3" json:"spaceType,omitempty"`
ReplicationKey uint64 `protobuf:"varint,4,opt,name=replicationKey,proto3" json:"replicationKey,omitempty"`
Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"`
SpaceHeaderPayload []byte `protobuf:"bytes,6,opt,name=spaceHeaderPayload,proto3" json:"spaceHeaderPayload,omitempty"`
}
func (m *SpaceHeader) Reset() { *m = SpaceHeader{} }
@ -761,6 +770,13 @@ func (m *SpaceHeader) GetSeed() []byte {
return nil
}
func (m *SpaceHeader) GetSpaceHeaderPayload() []byte {
if m != nil {
return m.SpaceHeaderPayload
}
return nil
}
// RawSpaceHeader is raw header for SpaceHeader
type RawSpaceHeader struct {
SpaceHeader []byte `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"`
@ -1232,71 +1248,73 @@ func init() {
}
var fileDescriptor_80e49f1f4ac27799 = []byte{
// 1019 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0x4b, 0x6f, 0xdb, 0x46,
0x10, 0x16, 0xe9, 0xa7, 0xc6, 0xb2, 0xcc, 0x6c, 0x9c, 0x44, 0x55, 0x0c, 0x45, 0x58, 0x14, 0x85,
0x91, 0x43, 0x1e, 0x72, 0x51, 0x20, 0x69, 0x7b, 0x48, 0x6c, 0xa5, 0x11, 0x8a, 0xd4, 0xc6, 0xaa,
0x41, 0x81, 0x02, 0x39, 0xac, 0xc9, 0xb1, 0xc4, 0x96, 0x22, 0x59, 0xee, 0xaa, 0xb6, 0x8e, 0x3d,
0xf5, 0xda, 0x73, 0x7b, 0xea, 0x7f, 0xe8, 0x8f, 0xe8, 0x31, 0xc7, 0x1e, 0x0b, 0xfb, 0x8f, 0x14,
0xbb, 0x5c, 0x3e, 0x24, 0x51, 0x39, 0xf4, 0x22, 0x73, 0xbf, 0x99, 0xf9, 0xe6, 0xb5, 0x3b, 0x63,
0x78, 0xea, 0x46, 0x93, 0x49, 0x14, 0x8a, 0x98, 0xbb, 0xf8, 0x58, 0xff, 0x8a, 0x59, 0xe8, 0xc6,
0x49, 0x24, 0xa3, 0xc7, 0xfa, 0x57, 0x14, 0xe8, 0x23, 0x0d, 0x90, 0x7a, 0x0e, 0xd0, 0x01, 0xec,
0xbe, 0x46, 0xee, 0x0d, 0x67, 0xa1, 0xcb, 0x78, 0x38, 0x42, 0x42, 0x60, 0xfd, 0x22, 0x89, 0x26,
0x2d, 0xab, 0x6b, 0x1d, 0xae, 0x33, 0xfd, 0x4d, 0x9a, 0x60, 0xcb, 0xa8, 0x65, 0x6b, 0xc4, 0x96,
0x11, 0xd9, 0x87, 0x8d, 0xc0, 0x9f, 0xf8, 0xb2, 0xb5, 0xd6, 0xb5, 0x0e, 0x77, 0x59, 0x7a, 0xa0,
0x57, 0xd0, 0xcc, 0xa9, 0x50, 0x4c, 0x03, 0xa9, 0xb8, 0xc6, 0x5c, 0x8c, 0x35, 0x57, 0x83, 0xe9,
0x6f, 0xf2, 0x05, 0x6c, 0x63, 0x80, 0x13, 0x0c, 0xa5, 0x68, 0xd9, 0xdd, 0xb5, 0xc3, 0x9d, 0x5e,
0xf7, 0x51, 0x11, 0xdf, 0x3c, 0x41, 0x3f, 0x55, 0x64, 0xb9, 0x85, 0xf2, 0xec, 0x46, 0xd3, 0x30,
0xf7, 0xac, 0x0f, 0xf4, 0x73, 0xb8, 0x53, 0x69, 0xa8, 0x02, 0xf7, 0x3d, 0xed, 0xbe, 0xce, 0x6c,
0xdf, 0xd3, 0x01, 0x21, 0xf7, 0x74, 0x2a, 0x75, 0xa6, 0xbf, 0xe9, 0x3b, 0xd8, 0x2b, 0x8c, 0x7f,
0x9a, 0xa2, 0x90, 0xa4, 0x05, 0x5b, 0x3a, 0xa4, 0x41, 0x66, 0x9b, 0x1d, 0xc9, 0x13, 0xd8, 0x4c,
0x54, 0x99, 0xb2, 0xd8, 0x5b, 0x55, 0xb1, 0x2b, 0x05, 0x66, 0xf4, 0xe8, 0x57, 0xe0, 0x94, 0x62,
0x8b, 0xa3, 0x50, 0x20, 0x39, 0x82, 0xad, 0x44, 0xc7, 0x29, 0x5a, 0x96, 0xa6, 0xf9, 0x68, 0x65,
0x09, 0x58, 0xa6, 0x49, 0xff, 0xb0, 0xe0, 0xd6, 0xe9, 0xf9, 0x0f, 0xe8, 0x4a, 0x25, 0x7d, 0x83,
0x42, 0xf0, 0x11, 0x7e, 0x20, 0xd4, 0x03, 0xa8, 0x27, 0x69, 0x3e, 0x83, 0x2c, 0xe1, 0x02, 0x50,
0x76, 0x09, 0xc6, 0xc1, 0x6c, 0xe0, 0xe9, 0x52, 0xd6, 0x59, 0x76, 0x54, 0x92, 0x98, 0xcf, 0x82,
0x88, 0x7b, 0xad, 0x75, 0xdd, 0xb7, 0xec, 0x48, 0xda, 0xb0, 0x1d, 0xe9, 0x00, 0x06, 0x5e, 0x6b,
0x43, 0x1b, 0xe5, 0x67, 0xda, 0x07, 0x67, 0xa8, 0x1c, 0x9f, 0x4d, 0xc5, 0x38, 0x2b, 0xe3, 0xd3,
0x82, 0x49, 0xc5, 0xb6, 0xd3, 0xbb, 0x57, 0x4a, 0x33, 0xd5, 0x4e, 0xc5, 0xb9, 0x0b, 0x7a, 0x1b,
0x6e, 0x95, 0x68, 0xd2, 0x72, 0x51, 0x9a, 0x73, 0x07, 0x41, 0xc6, 0xbd, 0xd0, 0x59, 0xfa, 0x2a,
0x37, 0x54, 0x3a, 0xa6, 0xce, 0xff, 0x23, 0x80, 0x5f, 0x6c, 0x68, 0x94, 0x25, 0xe4, 0x05, 0xec,
0x68, 0x1b, 0xd5, 0x16, 0x4c, 0x0c, 0xcf, 0x83, 0x12, 0x0f, 0xe3, 0x97, 0xc3, 0x42, 0xe1, 0x3b,
0x5f, 0x8e, 0x07, 0x1e, 0x2b, 0xdb, 0x90, 0x0e, 0x00, 0x77, 0x03, 0x43, 0xa8, 0x5b, 0xd1, 0x60,
0x25, 0x84, 0x50, 0x68, 0x14, 0xa7, 0xbc, 0x21, 0x73, 0x18, 0xe9, 0xc1, 0xbe, 0xa6, 0x1c, 0xa2,
0x94, 0x7e, 0x38, 0x12, 0x67, 0x73, 0x2d, 0xaa, 0x94, 0x91, 0xcf, 0xe0, 0x6e, 0x15, 0x9e, 0x77,
0x6f, 0x85, 0x94, 0xfe, 0x69, 0xc1, 0x4e, 0x29, 0x25, 0xd5, 0x77, 0xdf, 0xc3, 0x50, 0xfa, 0x72,
0x66, 0x9e, 0x72, 0x7e, 0x56, 0xb7, 0x4c, 0xfa, 0x13, 0x14, 0x92, 0x4f, 0x62, 0x9d, 0xda, 0x1a,
0x2b, 0x00, 0x25, 0xd5, 0x3e, 0xbe, 0x9d, 0xc5, 0x68, 0xd2, 0x2a, 0x00, 0xf2, 0x09, 0x34, 0xd5,
0xa5, 0xf3, 0x5d, 0x2e, 0xfd, 0x28, 0xfc, 0x1a, 0x67, 0x3a, 0x9b, 0x75, 0xb6, 0x80, 0xaa, 0x57,
0x2b, 0x10, 0xd3, 0xa8, 0x1b, 0x4c, 0x7f, 0xd3, 0x33, 0x68, 0xce, 0x17, 0x9e, 0x74, 0x97, 0x1b,
0xd5, 0x98, 0xef, 0x83, 0x8a, 0xc6, 0x1f, 0x85, 0x5c, 0x4e, 0x13, 0x34, 0x6d, 0x28, 0x00, 0x7a,
0x02, 0xfb, 0x55, 0xad, 0xd4, 0xef, 0x88, 0x5f, 0xce, 0xb1, 0x16, 0x80, 0xb9, 0x87, 0x76, 0x7e,
0x0f, 0x7f, 0xb7, 0x60, 0x7f, 0x58, 0x2e, 0xeb, 0x71, 0x14, 0x4a, 0x35, 0x8a, 0xbe, 0x84, 0x46,
0xfa, 0x58, 0x4e, 0x30, 0x40, 0x89, 0x15, 0x17, 0xf2, 0xb4, 0x24, 0x7e, 0x5d, 0x63, 0x73, 0xea,
0xe4, 0xb9, 0xc9, 0xce, 0x58, 0xdb, 0xda, 0xfa, 0xee, 0xe2, 0x75, 0xce, 0x8d, 0xcb, 0xca, 0x2f,
0xb7, 0x60, 0xe3, 0x67, 0x1e, 0x4c, 0x91, 0x76, 0xa0, 0x51, 0x76, 0xb2, 0xf4, 0x88, 0x8e, 0x4c,
0xdf, 0x8d, 0xf8, 0x63, 0xd8, 0xf5, 0xf4, 0x57, 0x72, 0x86, 0x98, 0xe4, 0x13, 0x66, 0x1e, 0xa4,
0xef, 0xe0, 0xce, 0x5c, 0xc2, 0xc3, 0x90, 0xc7, 0x62, 0x1c, 0x49, 0x75, 0xed, 0x53, 0x4d, 0x6f,
0xe0, 0xa5, 0x83, 0xae, 0xce, 0x4a, 0xc8, 0x32, 0xbd, 0x5d, 0x45, 0xff, 0xab, 0x05, 0x8d, 0x8c,
0xfa, 0x84, 0x4b, 0x4e, 0x9e, 0xc1, 0x96, 0x9b, 0xd6, 0xd4, 0x0c, 0xcf, 0x07, 0x8b, 0x55, 0x58,
0x28, 0x3d, 0xcb, 0xf4, 0xd5, 0xee, 0x11, 0x26, 0x3a, 0x53, 0xc1, 0xee, 0x2a, 0xdb, 0x2c, 0x0b,
0x96, 0x5b, 0xd0, 0x1f, 0xcd, 0x88, 0x19, 0x4e, 0xcf, 0x85, 0x9b, 0xf8, 0xb1, 0xba, 0x9e, 0xea,
0x6d, 0x98, 0x81, 0x9b, 0xa5, 0x98, 0x9f, 0xc9, 0x73, 0xd8, 0xe4, 0xae, 0xd2, 0xd2, 0xce, 0x9a,
0x3d, 0xba, 0xe4, 0xac, 0xc4, 0xf4, 0x42, 0x6b, 0x32, 0x63, 0xf1, 0xf0, 0x12, 0xb6, 0xfb, 0x49,
0x72, 0x1c, 0x79, 0x28, 0x48, 0x13, 0xe0, 0x6d, 0x88, 0x57, 0x31, 0xba, 0x12, 0x3d, 0xa7, 0x46,
0x1c, 0x33, 0xa2, 0xde, 0xf8, 0x42, 0xf8, 0xe1, 0xc8, 0xb1, 0xc8, 0x9e, 0x69, 0x5c, 0xff, 0xca,
0x17, 0x52, 0x38, 0x36, 0xb9, 0x0d, 0x7b, 0x1a, 0xf8, 0x26, 0x92, 0x83, 0xf0, 0x98, 0xbb, 0x63,
0x74, 0xd6, 0x08, 0x81, 0xa6, 0x06, 0x07, 0x22, 0x6d, 0xb0, 0xe7, 0xac, 0x2b, 0xcb, 0x7e, 0x92,
0x44, 0xc9, 0xe9, 0xc5, 0x85, 0x40, 0xe9, 0x78, 0x0f, 0x9f, 0xc1, 0xbd, 0x15, 0xb1, 0x91, 0x5d,
0xa8, 0x1b, 0xf4, 0x1c, 0x9d, 0x9a, 0x32, 0x7d, 0x1b, 0x8a, 0x1c, 0xb0, 0x7a, 0x7f, 0xd9, 0x50,
0x4f, 0x6d, 0x67, 0xa1, 0x4b, 0x8e, 0x61, 0x3b, 0x5b, 0x65, 0xa4, 0x5d, 0xb9, 0xdf, 0xf4, 0x24,
0x6f, 0xdf, 0xaf, 0xde, 0x7d, 0xe9, 0x04, 0x7f, 0x65, 0x18, 0xd5, 0x3e, 0x20, 0xf7, 0x97, 0xa6,
0x77, 0xb1, 0x6c, 0xda, 0x07, 0xd5, 0xc2, 0x25, 0x9e, 0x20, 0xa8, 0xe2, 0xc9, 0x17, 0x4b, 0x15,
0x4f, 0x69, 0xa3, 0x30, 0x70, 0x8a, 0x1d, 0x3c, 0x94, 0x09, 0xf2, 0x09, 0x39, 0x58, 0x7a, 0xc3,
0xa5, 0x05, 0xdd, 0xfe, 0xa0, 0xf4, 0xd0, 0x7a, 0x62, 0xbd, 0xfc, 0xf4, 0xef, 0xeb, 0x8e, 0xf5,
0xfe, 0xba, 0x63, 0xfd, 0x7b, 0xdd, 0xb1, 0x7e, 0xbb, 0xe9, 0xd4, 0xde, 0xdf, 0x74, 0x6a, 0xff,
0xdc, 0x74, 0x6a, 0xdf, 0xb7, 0x57, 0xff, 0x6b, 0x77, 0xbe, 0xa9, 0xff, 0x1c, 0xfd, 0x17, 0x00,
0x00, 0xff, 0xff, 0xa1, 0xe4, 0x04, 0x3d, 0xff, 0x09, 0x00, 0x00,
// 1042 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0x4f, 0x6f, 0x1b, 0x45,
0x14, 0xf7, 0x6e, 0xd2, 0x24, 0x7e, 0x71, 0x9c, 0xed, 0x34, 0x6d, 0x8d, 0x1b, 0xb9, 0xd6, 0x08,
0xa1, 0xa8, 0x87, 0xb4, 0x75, 0x10, 0x52, 0x0b, 0x1c, 0x5a, 0x27, 0xa5, 0x16, 0x2a, 0x89, 0xc6,
0x54, 0x48, 0x48, 0x3d, 0x4c, 0x76, 0x5f, 0xec, 0x85, 0xf5, 0xee, 0xb2, 0x33, 0x26, 0xf1, 0x91,
0x13, 0x57, 0xce, 0xf0, 0x35, 0xf8, 0x10, 0x1c, 0xcb, 0x8d, 0x23, 0x4a, 0xbe, 0x08, 0x9a, 0xd9,
0xd9, 0x3f, 0xb6, 0xd7, 0x95, 0xb8, 0x38, 0x3b, 0xbf, 0xf7, 0xde, 0xef, 0xfd, 0x9b, 0x79, 0x2f,
0xf0, 0xd4, 0x8d, 0x26, 0x93, 0x28, 0x14, 0x31, 0x77, 0xf1, 0xb1, 0xfe, 0x15, 0xb3, 0xd0, 0x8d,
0x93, 0x48, 0x46, 0x8f, 0xf5, 0xaf, 0x28, 0xd0, 0x43, 0x0d, 0x90, 0x7a, 0x0e, 0xd0, 0x01, 0xec,
0xbc, 0x46, 0xee, 0x0d, 0x67, 0xa1, 0xcb, 0x78, 0x38, 0x42, 0x42, 0x60, 0xfd, 0x22, 0x89, 0x26,
0x2d, 0xab, 0x6b, 0x1d, 0xac, 0x33, 0xfd, 0x4d, 0x9a, 0x60, 0xcb, 0xa8, 0x65, 0x6b, 0xc4, 0x96,
0x11, 0xd9, 0x83, 0x5b, 0x81, 0x3f, 0xf1, 0x65, 0x6b, 0xad, 0x6b, 0x1d, 0xec, 0xb0, 0xf4, 0x40,
0xaf, 0xa0, 0x99, 0x53, 0xa1, 0x98, 0x06, 0x52, 0x71, 0x8d, 0xb9, 0x18, 0x6b, 0xae, 0x06, 0xd3,
0xdf, 0xe4, 0x0b, 0xd8, 0xc2, 0x00, 0x27, 0x18, 0x4a, 0xd1, 0xb2, 0xbb, 0x6b, 0x07, 0xdb, 0xbd,
0xee, 0x61, 0x11, 0xdf, 0x3c, 0xc1, 0x49, 0xaa, 0xc8, 0x72, 0x0b, 0xe5, 0xd9, 0x8d, 0xa6, 0x61,
0xee, 0x59, 0x1f, 0xe8, 0xe7, 0x70, 0xb7, 0xd2, 0x50, 0x05, 0xee, 0x7b, 0xda, 0x7d, 0x9d, 0xd9,
0xbe, 0xa7, 0x03, 0x42, 0xee, 0xe9, 0x54, 0xea, 0x4c, 0x7f, 0xd3, 0x77, 0xb0, 0x5b, 0x18, 0xff,
0x34, 0x45, 0x21, 0x49, 0x0b, 0x36, 0x75, 0x48, 0x83, 0xcc, 0x36, 0x3b, 0x92, 0x27, 0xb0, 0x91,
0xa8, 0x32, 0x65, 0xb1, 0xb7, 0xaa, 0x62, 0x57, 0x0a, 0xcc, 0xe8, 0xd1, 0xaf, 0xc0, 0x29, 0xc5,
0x16, 0x47, 0xa1, 0x40, 0x72, 0x04, 0x9b, 0x89, 0x8e, 0x53, 0xb4, 0x2c, 0x4d, 0xf3, 0xd1, 0xca,
0x12, 0xb0, 0x4c, 0x93, 0xfe, 0x61, 0xc1, 0xed, 0xd3, 0xf3, 0x1f, 0xd0, 0x95, 0x4a, 0xfa, 0x06,
0x85, 0xe0, 0x23, 0xfc, 0x40, 0xa8, 0xfb, 0x50, 0x4f, 0xd2, 0x7c, 0x06, 0x59, 0xc2, 0x05, 0xa0,
0xec, 0x12, 0x8c, 0x83, 0xd9, 0xc0, 0xd3, 0xa5, 0xac, 0xb3, 0xec, 0xa8, 0x24, 0x31, 0x9f, 0x05,
0x11, 0xf7, 0x5a, 0xeb, 0xba, 0x6f, 0xd9, 0x91, 0xb4, 0x61, 0x2b, 0xd2, 0x01, 0x0c, 0xbc, 0xd6,
0x2d, 0x6d, 0x94, 0x9f, 0x29, 0x82, 0x33, 0x54, 0x8e, 0xcf, 0xa6, 0x62, 0x9c, 0x95, 0xf1, 0x69,
0xc1, 0xa4, 0x62, 0xdb, 0xee, 0xdd, 0x2f, 0xa5, 0x99, 0x6a, 0xa7, 0xe2, 0xc2, 0x45, 0x07, 0xa0,
0x9f, 0xa0, 0x87, 0xa1, 0xf4, 0x79, 0xa0, 0xa3, 0x6e, 0xb0, 0x12, 0x42, 0xef, 0xc0, 0xed, 0x92,
0x9b, 0xb4, 0x9c, 0x94, 0xe6, 0xbe, 0x83, 0x20, 0xf3, 0xbd, 0xd0, 0x79, 0xfa, 0x2a, 0x37, 0x54,
0x3a, 0xa6, 0x0f, 0xff, 0x3f, 0x40, 0xfa, 0x8b, 0x0d, 0x8d, 0xb2, 0x84, 0xbc, 0x80, 0x6d, 0x6d,
0xa3, 0xda, 0x86, 0x89, 0xe1, 0x79, 0x58, 0xe2, 0x61, 0xfc, 0x72, 0x58, 0x28, 0x7c, 0xe7, 0xcb,
0xf1, 0xc0, 0x63, 0x65, 0x1b, 0x95, 0x34, 0x77, 0x03, 0x43, 0x98, 0x25, 0x5d, 0x20, 0x84, 0x42,
0xa3, 0x38, 0xe5, 0x0d, 0x9b, 0xc3, 0x48, 0x0f, 0xf6, 0x34, 0xe5, 0x10, 0xa5, 0xf4, 0xc3, 0x91,
0x38, 0x9b, 0x6b, 0x61, 0xa5, 0x8c, 0x7c, 0x06, 0xf7, 0xaa, 0xf0, 0xbc, 0xbb, 0x2b, 0xa4, 0xf4,
0x6f, 0x0b, 0xb6, 0x4b, 0x29, 0xa9, 0x7b, 0xe1, 0xeb, 0x06, 0xc9, 0x99, 0x79, 0xea, 0xf9, 0x59,
0xdd, 0x42, 0xe9, 0x4f, 0x50, 0x48, 0x3e, 0x89, 0x75, 0x6a, 0x6b, 0xac, 0x00, 0x94, 0x54, 0xfb,
0xf8, 0x76, 0x16, 0xa3, 0x49, 0xab, 0x00, 0xc8, 0x27, 0xd0, 0x54, 0x97, 0xd2, 0x77, 0xb9, 0xf4,
0xa3, 0xf0, 0x6b, 0x9c, 0xe9, 0x6c, 0xd6, 0xd9, 0x02, 0xaa, 0x5e, 0xb5, 0x40, 0x4c, 0xa3, 0x6e,
0x30, 0xfd, 0x4d, 0x0e, 0x81, 0x94, 0x4a, 0x9c, 0x55, 0x63, 0x43, 0x6b, 0x54, 0x48, 0xe8, 0x19,
0x34, 0xe7, 0x1b, 0x45, 0xba, 0xcb, 0x8d, 0x6d, 0xcc, 0xf7, 0x4d, 0x45, 0xef, 0x8f, 0x42, 0x2e,
0xa7, 0x09, 0x9a, 0xb6, 0x15, 0x00, 0x3d, 0x86, 0xbd, 0xaa, 0xd6, 0xeb, 0x77, 0xc9, 0x2f, 0xe7,
0x58, 0x0b, 0xc0, 0xdc, 0x5b, 0x3b, 0xbf, 0xb7, 0xbf, 0x5b, 0xb0, 0x37, 0x2c, 0xb7, 0xa1, 0x1f,
0x85, 0x52, 0x8d, 0xb6, 0x2f, 0xa1, 0x91, 0x3e, 0xbe, 0x63, 0x0c, 0x50, 0x62, 0xc5, 0x05, 0x3e,
0x2d, 0x89, 0x5f, 0xd7, 0xd8, 0x9c, 0x3a, 0x79, 0x6e, 0xb2, 0x33, 0xd6, 0xb6, 0xb6, 0xbe, 0xb7,
0x78, 0xfd, 0x73, 0xe3, 0xb2, 0xf2, 0xcb, 0x4d, 0xb8, 0xf5, 0x33, 0x0f, 0xa6, 0x48, 0x3b, 0xd0,
0x28, 0x3b, 0x59, 0x7a, 0x74, 0x47, 0xe6, 0x9e, 0x18, 0xf1, 0xc7, 0xb0, 0xe3, 0xe9, 0xaf, 0xe4,
0x0c, 0x31, 0xc9, 0x27, 0xd6, 0x3c, 0x48, 0xdf, 0xc1, 0xdd, 0xb9, 0x84, 0x87, 0x21, 0x8f, 0xc5,
0x38, 0x92, 0xea, 0x99, 0xa4, 0x9a, 0xde, 0xc0, 0x4b, 0x07, 0x67, 0x9d, 0x95, 0x90, 0x65, 0x7a,
0xbb, 0x8a, 0xfe, 0x57, 0x0b, 0x1a, 0x19, 0xf5, 0x31, 0x97, 0x9c, 0x3c, 0x83, 0x4d, 0x37, 0xad,
0xa9, 0x19, 0xc6, 0x0f, 0x17, 0xab, 0xb0, 0x50, 0x7a, 0x96, 0xe9, 0xab, 0x5d, 0x26, 0x4c, 0x74,
0xa6, 0x82, 0xdd, 0x55, 0xb6, 0x59, 0x16, 0x2c, 0xb7, 0xa0, 0x3f, 0x9a, 0x91, 0x34, 0x9c, 0x9e,
0x0b, 0x37, 0xf1, 0x63, 0x75, 0x9d, 0xd5, 0x5b, 0x32, 0x03, 0x3c, 0x4b, 0x31, 0x3f, 0x93, 0xe7,
0xb0, 0xc1, 0x5d, 0xa5, 0xa5, 0x9d, 0x35, 0x7b, 0x74, 0xc9, 0x59, 0x89, 0xe9, 0x85, 0xd6, 0x64,
0xc6, 0xe2, 0xd1, 0x25, 0x6c, 0x9d, 0x24, 0x49, 0x3f, 0xf2, 0x50, 0x90, 0x26, 0xc0, 0xdb, 0x10,
0xaf, 0x62, 0x74, 0x25, 0x7a, 0x4e, 0x8d, 0x38, 0x66, 0xa4, 0xbd, 0xf1, 0x85, 0xf0, 0xc3, 0x91,
0x63, 0x91, 0x5d, 0xd3, 0xb8, 0x93, 0x2b, 0x5f, 0x48, 0xe1, 0xd8, 0xe4, 0x0e, 0xec, 0x6a, 0xe0,
0x9b, 0x48, 0x0e, 0xc2, 0x3e, 0x77, 0xc7, 0xe8, 0xac, 0x11, 0x02, 0x4d, 0x0d, 0x0e, 0x44, 0xda,
0x60, 0xcf, 0x59, 0x57, 0x96, 0x27, 0x49, 0x12, 0x25, 0xa7, 0x17, 0x17, 0x02, 0xa5, 0xe3, 0x3d,
0x7a, 0x06, 0xf7, 0x57, 0xc4, 0x46, 0x76, 0xa0, 0x6e, 0xd0, 0x73, 0x74, 0x6a, 0xca, 0xf4, 0x6d,
0x28, 0x72, 0xc0, 0xea, 0xfd, 0x69, 0x43, 0x3d, 0xb5, 0x9d, 0x85, 0x2e, 0xe9, 0xc3, 0x56, 0xb6,
0x1a, 0x49, 0xbb, 0x72, 0x5f, 0xea, 0xc9, 0xdf, 0x7e, 0x50, 0xbd, 0x4b, 0xd3, 0x89, 0xff, 0xca,
0x30, 0xaa, 0xfd, 0x41, 0x1e, 0x2c, 0x4d, 0xfb, 0x62, 0x79, 0xb5, 0xf7, 0xab, 0x85, 0x4b, 0x3c,
0x41, 0x50, 0xc5, 0x93, 0x2f, 0xa2, 0x2a, 0x9e, 0xd2, 0x06, 0x62, 0xe0, 0x14, 0x3b, 0x7d, 0x28,
0x13, 0xe4, 0x13, 0xb2, 0xbf, 0xf4, 0x86, 0x4b, 0x0b, 0xbf, 0xfd, 0x41, 0xe9, 0x81, 0xf5, 0xc4,
0x7a, 0xf9, 0xe9, 0x5f, 0xd7, 0x1d, 0xeb, 0xfd, 0x75, 0xc7, 0xfa, 0xf7, 0xba, 0x63, 0xfd, 0x76,
0xd3, 0xa9, 0xbd, 0xbf, 0xe9, 0xd4, 0xfe, 0xb9, 0xe9, 0xd4, 0xbe, 0x6f, 0xaf, 0xfe, 0x57, 0xf1,
0x7c, 0x43, 0xff, 0x39, 0xfa, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x13, 0xff, 0xe7, 0x17, 0x4f, 0x0a,
0x00, 0x00,
}
func (m *HeadSyncRange) Marshal() (dAtA []byte, err error) {
@ -1582,6 +1600,13 @@ func (m *SpacePushRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.Credential) > 0 {
i -= len(m.Credential)
copy(dAtA[i:], m.Credential)
i = encodeVarintSpacesync(dAtA, i, uint64(len(m.Credential)))
i--
dAtA[i] = 0x12
}
if m.Payload != nil {
{
size, err := m.Payload.MarshalToSizedBuffer(dAtA[:i])
@ -1768,6 +1793,13 @@ func (m *SpaceHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.SpaceHeaderPayload) > 0 {
i -= len(m.SpaceHeaderPayload)
copy(dAtA[i:], m.SpaceHeaderPayload)
i = encodeVarintSpacesync(dAtA, i, uint64(len(m.SpaceHeaderPayload)))
i--
dAtA[i] = 0x32
}
if len(m.Seed) > 0 {
i -= len(m.Seed)
copy(dAtA[i:], m.Seed)
@ -2276,6 +2308,10 @@ func (m *SpacePushRequest) Size() (n int) {
l = m.Payload.Size()
n += 1 + l + sovSpacesync(uint64(l))
}
l = len(m.Credential)
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
return n
}
@ -2367,6 +2403,10 @@ func (m *SpaceHeader) Size() (n int) {
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
l = len(m.SpaceHeaderPayload)
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
return n
}
@ -3363,6 +3403,40 @@ func (m *SpacePushRequest) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Credential", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpacesync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthSpacesync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthSpacesync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Credential = append(m.Credential[:0], dAtA[iNdEx:postIndex]...)
if m.Credential == nil {
m.Credential = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipSpacesync(dAtA[iNdEx:])
@ -3987,6 +4061,40 @@ func (m *SpaceHeader) Unmarshal(dAtA []byte) error {
m.Seed = []byte{}
}
iNdEx = postIndex
case 6:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SpaceHeaderPayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpacesync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthSpacesync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthSpacesync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.SpaceHeaderPayload = append(m.SpaceHeaderPayload[:0], dAtA[iNdEx:postIndex]...)
if m.SpaceHeaderPayload == nil {
m.SpaceHeaderPayload = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipSpacesync(dAtA[iNdEx:])

View File

@ -3,15 +3,16 @@ package syncstatus
import (
"context"
"fmt"
"sync"
"time"
"github.com/anytypeio/any-sync/app/logger"
treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/nodeconf"
"github.com/anytypeio/any-sync/util/periodicsync"
"github.com/anytypeio/any-sync/util/slice"
"golang.org/x/exp/slices"
"sync"
"time"
)
const (
@ -19,7 +20,7 @@ const (
syncTimeout = time.Second
)
var log = logger.NewNamed("commonspace.syncstatus")
var log = logger.NewNamed("common.commonspace.syncstatus")
type UpdateReceiver interface {
UpdateTree(ctx context.Context, treeId string, status SyncStatus) (err error)

View File

@ -4,8 +4,10 @@ package coordinatorclient
import (
"context"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/coordinator/coordinatorproto"
"github.com/anytypeio/any-sync/net/pool"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
"github.com/anytypeio/any-sync/nodeconf"
)
@ -16,7 +18,9 @@ func New() CoordinatorClient {
}
type CoordinatorClient interface {
SpaceSign(ctx context.Context, spaceId string) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error)
ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error)
StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error)
SpaceSign(ctx context.Context, spaceId string, spaceHeader []byte) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error)
FileLimitCheck(ctx context.Context, spaceId string, identity []byte) (limit uint64, err error)
app.Component
}
@ -26,6 +30,40 @@ type coordinatorClient struct {
nodeConf nodeconf.Service
}
func (c *coordinatorClient) ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error) {
cl, err := c.client(ctx)
if err != nil {
return
}
resp, err := cl.SpaceStatusChange(ctx, &coordinatorproto.SpaceStatusChangeRequest{
SpaceId: spaceId,
DeletionChangeId: deleteRaw.GetId(),
DeletionChangePayload: deleteRaw.GetRawChange(),
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
status = resp.Payload
return
}
func (c *coordinatorClient) StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error) {
cl, err := c.client(ctx)
if err != nil {
return
}
resp, err := cl.SpaceStatusCheck(ctx, &coordinatorproto.SpaceStatusCheckRequest{
SpaceId: spaceId,
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
status = resp.Payload
return
}
func (c *coordinatorClient) Init(a *app.App) (err error) {
c.pool = a.MustComponent(pool.CName).(pool.Service).NewPool(CName)
c.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service)
@ -36,15 +74,17 @@ func (c *coordinatorClient) Name() (name string) {
return CName
}
func (c *coordinatorClient) SpaceSign(ctx context.Context, spaceId string) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) {
func (c *coordinatorClient) SpaceSign(ctx context.Context, spaceId string, spaceHeader []byte) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) {
cl, err := c.client(ctx)
if err != nil {
return
}
resp, err := cl.SpaceSign(ctx, &coordinatorproto.SpaceSignRequest{
SpaceId: spaceId,
Header: spaceHeader,
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
return resp.Receipt, nil
@ -60,6 +100,7 @@ func (c *coordinatorClient) FileLimitCheck(ctx context.Context, spaceId string,
SpaceId: spaceId,
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
return resp.Limit, nil

View File

@ -9,6 +9,7 @@ import (
reflect "reflect"
app "github.com/anytypeio/any-sync/app"
treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
coordinatorproto "github.com/anytypeio/any-sync/coordinator/coordinatorproto"
gomock "github.com/golang/mock/gomock"
)
@ -36,6 +37,21 @@ func (m *MockCoordinatorClient) EXPECT() *MockCoordinatorClientMockRecorder {
return m.recorder
}
// ChangeStatus mocks base method.
func (m *MockCoordinatorClient) ChangeStatus(arg0 context.Context, arg1 string, arg2 *treechangeproto.RawTreeChangeWithId) (*coordinatorproto.SpaceStatusPayload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeStatus", arg0, arg1, arg2)
ret0, _ := ret[0].(*coordinatorproto.SpaceStatusPayload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChangeStatus indicates an expected call of ChangeStatus.
func (mr *MockCoordinatorClientMockRecorder) ChangeStatus(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeStatus", reflect.TypeOf((*MockCoordinatorClient)(nil).ChangeStatus), arg0, arg1, arg2)
}
// FileLimitCheck mocks base method.
func (m *MockCoordinatorClient) FileLimitCheck(arg0 context.Context, arg1 string, arg2 []byte) (uint64, error) {
m.ctrl.T.Helper()
@ -80,16 +96,31 @@ func (mr *MockCoordinatorClientMockRecorder) Name() *gomock.Call {
}
// SpaceSign mocks base method.
func (m *MockCoordinatorClient) SpaceSign(arg0 context.Context, arg1 string) (*coordinatorproto.SpaceReceiptWithSignature, error) {
func (m *MockCoordinatorClient) SpaceSign(arg0 context.Context, arg1 string, arg2 []byte) (*coordinatorproto.SpaceReceiptWithSignature, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SpaceSign", arg0, arg1)
ret := m.ctrl.Call(m, "SpaceSign", arg0, arg1, arg2)
ret0, _ := ret[0].(*coordinatorproto.SpaceReceiptWithSignature)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// SpaceSign indicates an expected call of SpaceSign.
func (mr *MockCoordinatorClientMockRecorder) SpaceSign(arg0, arg1 interface{}) *gomock.Call {
func (mr *MockCoordinatorClientMockRecorder) SpaceSign(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceSign", reflect.TypeOf((*MockCoordinatorClient)(nil).SpaceSign), arg0, arg1)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceSign", reflect.TypeOf((*MockCoordinatorClient)(nil).SpaceSign), arg0, arg1, arg2)
}
// StatusCheck mocks base method.
func (m *MockCoordinatorClient) StatusCheck(arg0 context.Context, arg1 string) (*coordinatorproto.SpaceStatusPayload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StatusCheck", arg0, arg1)
ret0, _ := ret[0].(*coordinatorproto.SpaceStatusPayload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StatusCheck indicates an expected call of StatusCheck.
func (mr *MockCoordinatorClientMockRecorder) StatusCheck(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatusCheck", reflect.TypeOf((*MockCoordinatorClient)(nil).StatusCheck), arg0, arg1)
}

File diff suppressed because it is too large Load Diff

View File

@ -42,6 +42,8 @@ type DRPCCoordinatorClient interface {
SpaceSign(ctx context.Context, in *SpaceSignRequest) (*SpaceSignResponse, error)
FileLimitCheck(ctx context.Context, in *FileLimitCheckRequest) (*FileLimitCheckResponse, error)
SpaceStatusCheck(ctx context.Context, in *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error)
SpaceStatusChange(ctx context.Context, in *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error)
}
type drpcCoordinatorClient struct {
@ -72,9 +74,29 @@ func (c *drpcCoordinatorClient) FileLimitCheck(ctx context.Context, in *FileLimi
return out, nil
}
func (c *drpcCoordinatorClient) SpaceStatusCheck(ctx context.Context, in *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) {
out := new(SpaceStatusCheckResponse)
err := c.cc.Invoke(ctx, "/coordinator.Coordinator/SpaceStatusCheck", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
func (c *drpcCoordinatorClient) SpaceStatusChange(ctx context.Context, in *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error) {
out := new(SpaceStatusChangeResponse)
err := c.cc.Invoke(ctx, "/coordinator.Coordinator/SpaceStatusChange", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
type DRPCCoordinatorServer interface {
SpaceSign(context.Context, *SpaceSignRequest) (*SpaceSignResponse, error)
FileLimitCheck(context.Context, *FileLimitCheckRequest) (*FileLimitCheckResponse, error)
SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error)
SpaceStatusChange(context.Context, *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error)
}
type DRPCCoordinatorUnimplementedServer struct{}
@ -87,9 +109,17 @@ func (s *DRPCCoordinatorUnimplementedServer) FileLimitCheck(context.Context, *Fi
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCCoordinatorUnimplementedServer) SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCCoordinatorUnimplementedServer) SpaceStatusChange(context.Context, *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
type DRPCCoordinatorDescription struct{}
func (DRPCCoordinatorDescription) NumMethods() int { return 2 }
func (DRPCCoordinatorDescription) NumMethods() int { return 4 }
func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
switch n {
@ -111,6 +141,24 @@ func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Rec
in1.(*FileLimitCheckRequest),
)
}, DRPCCoordinatorServer.FileLimitCheck, true
case 2:
return "/coordinator.Coordinator/SpaceStatusCheck", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCCoordinatorServer).
SpaceStatusCheck(
ctx,
in1.(*SpaceStatusCheckRequest),
)
}, DRPCCoordinatorServer.SpaceStatusCheck, true
case 3:
return "/coordinator.Coordinator/SpaceStatusChange", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCCoordinatorServer).
SpaceStatusChange(
ctx,
in1.(*SpaceStatusChangeRequest),
)
}, DRPCCoordinatorServer.SpaceStatusChange, true
default:
return "", nil, nil, nil, false
}
@ -151,3 +199,35 @@ func (x *drpcCoordinator_FileLimitCheckStream) SendAndClose(m *FileLimitCheckRes
}
return x.CloseSend()
}
type DRPCCoordinator_SpaceStatusCheckStream interface {
drpc.Stream
SendAndClose(*SpaceStatusCheckResponse) error
}
type drpcCoordinator_SpaceStatusCheckStream struct {
drpc.Stream
}
func (x *drpcCoordinator_SpaceStatusCheckStream) SendAndClose(m *SpaceStatusCheckResponse) error {
if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil {
return err
}
return x.CloseSend()
}
type DRPCCoordinator_SpaceStatusChangeStream interface {
drpc.Stream
SendAndClose(*SpaceStatusChangeResponse) error
}
type drpcCoordinator_SpaceStatusChangeStream struct {
drpc.Stream
}
func (x *drpcCoordinator_SpaceStatusChangeStream) SendAndClose(m *SpaceStatusChangeResponse) error {
if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil {
return err
}
return x.CloseSend()
}

View File

@ -0,0 +1,16 @@
package coordinatorproto
import (
"errors"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
)
var (
errGroup = rpcerr.ErrGroup(ErrorCodes_ErrorOffset)
ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrorCodes_Unexpected))
ErrSpaceIsCreated = errGroup.Register(errors.New("space is missing"), uint64(ErrorCodes_SpaceCreated))
ErrSpaceIsDeleted = errGroup.Register(errors.New("space is deleted"), uint64(ErrorCodes_SpaceDeleted))
ErrSpaceDeletionPending = errGroup.Register(errors.New("space is set out for deletion"), uint64(ErrorCodes_SpaceDeletionPending))
ErrSpaceNotExists = errGroup.Register(errors.New("space not exists"), uint64(ErrorCodes_SpaceNotExists))
)

View File

@ -12,10 +12,38 @@ service Coordinator {
// - if a handshake identity matches a given identity
// - if a requester contains in nodeconf list
rpc FileLimitCheck(FileLimitCheckRequest) returns (FileLimitCheckResponse);
// SpaceStatusCheck checks the status of space and tells if it is deleted or not
rpc SpaceStatusCheck(SpaceStatusCheckRequest) returns (SpaceStatusCheckResponse);
// SpaceStatusChange changes the status of space
rpc SpaceStatusChange(SpaceStatusChangeRequest) returns (SpaceStatusChangeResponse);
}
message SpaceSignRequest {
string spaceId = 1;
bytes header = 2;
}
enum ErrorCodes {
Unexpected = 0;
SpaceDeleted = 1;
SpaceDeletionPending = 2;
SpaceCreated = 3;
SpaceNotExists = 4;
ErrorOffset = 300;
}
enum SpaceStatus {
SpaceStatusCreated = 0;
SpaceStatusPendingDeletion = 1;
SpaceStatusDeletionStarted = 2;
SpaceStatusDeleted = 3;
}
message SpaceStatusPayload {
SpaceStatus status = 1;
int64 deletionTimestamp = 2;
}
message SpaceSignResponse {
@ -53,3 +81,25 @@ message FileLimitCheckRequest {
message FileLimitCheckResponse {
uint64 limit = 1;
}
// SpaceStatusCheckRequest contains the spaceId of requested space
message SpaceStatusCheckRequest {
string spaceId = 1;
}
// SpaceStatusCheckResponse contains the current status of space
message SpaceStatusCheckResponse {
SpaceStatusPayload payload = 1;
}
// SpaceStatusChangeRequest contains the deletionChange if we want to delete space, or it is empty otherwise
message SpaceStatusChangeRequest {
string spaceId = 1;
string deletionChangeId = 2;
bytes deletionChangePayload = 3;
}
// SpaceStatusChangeResponse contains changed status of space
message SpaceStatusChangeResponse {
SpaceStatusPayload payload = 1;
}

9
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/awalterschulze/gographviz v2.0.3+incompatible
github.com/cespare/xxhash v1.1.0
github.com/cheggaaa/mb/v3 v3.0.1
github.com/gobwas/glob v0.2.3
github.com/goccy/go-graphviz v0.1.0
github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.6.0
@ -19,9 +20,10 @@ require (
github.com/ipfs/go-ipfs-exchange-interface v0.2.0
github.com/ipfs/go-ipld-format v0.4.0
github.com/ipfs/go-merkledag v0.10.0
github.com/ipfs/go-unixfs v0.4.3
github.com/ipfs/go-unixfs v0.4.4
github.com/libp2p/go-libp2p v0.24.1
github.com/minio/sha256-simd v1.0.0
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multibase v0.1.1
github.com/multiformats/go-multihash v0.2.1
github.com/prometheus/client_golang v1.14.0
@ -30,7 +32,7 @@ require (
github.com/zeebo/errs v1.3.0
go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3
golang.org/x/net v0.7.0
golang.org/x/net v0.8.0
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
gopkg.in/yaml.v3 v3.0.1
storj.io/drpc v0.0.32
@ -74,7 +76,6 @@ require (
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.8.0 // indirect
@ -99,7 +100,7 @@ require (
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect

14
go.sum
View File

@ -106,6 +106,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/goccy/go-graphviz v0.1.0 h1:6OqQoQ5PeAiHYe/YcusyeulqBrOkUb16HQ4ctRdyVUU=
github.com/goccy/go-graphviz v0.1.0/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -253,8 +255,8 @@ github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2m
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU=
github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ=
github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY=
github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
@ -576,8 +578,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -647,8 +649,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

View File

@ -2,15 +2,16 @@ package peer
import (
"context"
"sync/atomic"
"time"
"github.com/anytypeio/any-sync/app/logger"
"github.com/libp2p/go-libp2p/core/sec"
"go.uber.org/zap"
"storj.io/drpc"
"sync/atomic"
"time"
)
var log = logger.NewNamed("peer")
var log = logger.NewNamed("common.net.peer")
func NewPeer(sc sec.SecureConn, conn drpc.Conn) Peer {
return &peer{
@ -25,11 +26,13 @@ type Peer interface {
Id() string
LastUsage() time.Time
UpdateLastUsage()
TryClose(objectTTL time.Duration) (res bool, err error)
drpc.Conn
}
type peer struct {
id string
ttl time.Duration
lastUsage int64
sc sec.SecureConn
drpc.Conn
@ -76,6 +79,13 @@ func (p *peer) UpdateLastUsage() {
atomic.StoreInt64(&p.lastUsage, time.Now().Unix())
}
func (p *peer) TryClose(objectTTL time.Duration) (res bool, err error) {
if time.Now().Sub(p.LastUsage()) < objectTTL {
return false, nil
}
return true, p.Close()
}
func (p *peer) Close() (err error) {
log.Debug("peer close", zap.String("peerId", p.id))
return p.Conn.Close()

View File

@ -49,7 +49,7 @@ func (p *pool) Get(ctx context.Context, id string) (peer.Peer, error) {
default:
return pr, nil
}
_, _ = p.cache.Remove(id)
_, _ = p.cache.Remove(ctx, id)
return p.Get(ctx, id)
}
@ -67,6 +67,7 @@ func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error
default:
return pr, nil
}
_, _ = p.cache.Remove(ctx, peerId)
}
}
// shuffle ids for better consistency

View File

@ -194,6 +194,10 @@ func (t *testPeer) LastUsage() time.Time {
func (t *testPeer) UpdateLastUsage() {}
func (t *testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, t.Close()
}
func (t *testPeer) Close() error {
select {
case <-t.closed:

View File

@ -103,6 +103,10 @@ type testPeer struct {
drpc.Conn
}
func (t testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, t.Close()
}
func (t testPeer) Id() string {
return t.id
}

View File

@ -2,14 +2,15 @@ package timeoutconn
import (
"errors"
"github.com/anytypeio/any-sync/app/logger"
"go.uber.org/zap"
"net"
"os"
"time"
"github.com/anytypeio/any-sync/app/logger"
"go.uber.org/zap"
)
var log = logger.NewNamed("net.timeoutconn")
var log = logger.NewNamed("common.net.timeoutconn")
type Conn struct {
net.Conn

View File

@ -157,6 +157,20 @@ func (mr *MockConfigurationMockRecorder) ConsensusPeers() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsensusPeers", reflect.TypeOf((*MockConfiguration)(nil).ConsensusPeers))
}
// CoordinatorPeers mocks base method.
func (m *MockConfiguration) CoordinatorPeers() []string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CoordinatorPeers")
ret0, _ := ret[0].([]string)
return ret0
}
// CoordinatorPeers indicates an expected call of CoordinatorPeers.
func (mr *MockConfigurationMockRecorder) CoordinatorPeers() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CoordinatorPeers", reflect.TypeOf((*MockConfiguration)(nil).CoordinatorPeers))
}
// FilePeers mocks base method.
func (m *MockConfiguration) FilePeers() []string {
m.ctrl.T.Helper()

122
util/crc16/crc16.go Normal file
View File

@ -0,0 +1,122 @@
// Package crc16 is implementation according to CCITT standards.
//
// Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the
// following parameters:
//
// Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN"
// Width : 16 bit
// Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1)
// Initialization : 0000
// Reflect Input byte : False
// Reflect Output CRC : False
// Xor constant to output CRC : 0000
// Output for "123456789" : 31C3
//
// ported from the c++ code in the stellar-core codebase
// (https://github.com/stellar/stellar-core). The code is licensed
// as:
/*
* Copyright 2001-2010 Georges Menie (www.menie.org)
* Copyright 2010-2012 Salvatore Sanfilippo (adapted to Redis coding style)
* Copyright 2015 Stellar Development Foundation (ported to go)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the University of California, Berkeley nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package crc16
import (
"bytes"
"encoding/binary"
"fmt"
)
// ErrInvalidChecksum is returned when Validate determines either the checksum
// or the payload has been corrupted
var ErrInvalidChecksum = fmt.Errorf("invalid checksum")
var crc16tab = [256]uint16{
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
}
// Checksum returns the 2-byte checksum for the provided data
func Checksum(data []byte) []byte {
var crc uint16
var out bytes.Buffer
for _, b := range data {
crc = ((crc << 8) & 0xffff) ^ crc16tab[((crc>>8)^uint16(b))&0x00FF]
}
err := binary.Write(&out, binary.LittleEndian, crc)
if err != nil {
panic(err)
}
return out.Bytes()
}
// Validate returns an error if the provided checksum does not match
// the calculated checksum of the provided data
func Validate(data []byte, expected []byte) error {
actual := Checksum(data)
// validate the provided checksum against the calculated
if !bytes.Equal(actual, expected) {
return ErrInvalidChecksum
}
return nil
}

117
util/strkey/strkey.go Normal file
View File

@ -0,0 +1,117 @@
package strkey
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/anytypeio/any-sync/util/crc16"
"github.com/mr-tron/base58/base58"
)
// ErrInvalidVersionByte is returned when the version byte from a provided
// strkey-encoded string is not one of the valid values.
var ErrInvalidVersionByte = fmt.Errorf("invalid version byte")
// VersionByte represents one of the possible prefix values for a StrKey base
// string--the string the when encoded using base58 yields a final StrKey.
type VersionByte byte
// Decode decodes the provided StrKey into a raw value, checking the checksum
// and ensuring the expected VersionByte (the version parameter) is the value
// actually encoded into the provided src string.
func Decode(expected VersionByte, src string) ([]byte, error) {
raw, err := decodeString(src)
if err != nil {
return nil, err
}
// decode into components
version := VersionByte(raw[0])
vp := raw[0 : len(raw)-2]
payload := raw[1 : len(raw)-2]
checksum := raw[len(raw)-2:]
// ensure version byte is expected
if version != expected {
return nil, ErrInvalidVersionByte
}
// ensure checksum is valid
if err := crc16.Validate(vp, checksum); err != nil {
return nil, err
}
// if we made it through the gaunlet, return the decoded value
return payload, nil
}
// MustDecode is like Decode, but panics on error
func MustDecode(expected VersionByte, src string) []byte {
d, err := Decode(expected, src)
if err != nil {
panic(err)
}
return d
}
// Encode encodes the provided data to a StrKey, using the provided version
// byte.
func Encode(version VersionByte, src []byte) (string, error) {
var raw bytes.Buffer
// write version byte
if err := binary.Write(&raw, binary.LittleEndian, version); err != nil {
return "", err
}
// write payload
if _, err := raw.Write(src); err != nil {
return "", err
}
// calculate and write checksum
checksum := crc16.Checksum(raw.Bytes())
if _, err := raw.Write(checksum); err != nil {
return "", err
}
result := base58.FastBase58Encoding(raw.Bytes())
return result, nil
}
// MustEncode is like Encode, but panics on error
func MustEncode(version VersionByte, src []byte) string {
e, err := Encode(version, src)
if err != nil {
panic(err)
}
return e
}
// Version extracts and returns the version byte from the provided source
// string.
func Version(src string) (VersionByte, error) {
raw, err := decodeString(src)
if err != nil {
return VersionByte(0), err
}
return VersionByte(raw[0]), nil
}
// decodeString decodes a base58 string into the raw bytes, and ensures it could
// potentially be strkey encoded (i.e. it has both a version byte and a
// checksum, neither of which are explicitly checked by this func)
func decodeString(src string) ([]byte, error) {
raw, err := base58.FastBase58Decoding(src)
if err != nil {
return nil, fmt.Errorf("base58 decode failed: %s", err)
}
if len(raw) < 3 {
return nil, fmt.Errorf("encoded value is %d bytes; minimum valid length is 3", len(raw))
}
return raw, nil
}

7
util/strkey/versions.go Normal file
View File

@ -0,0 +1,7 @@
package strkey
const (
AccountAddressVersionByte VersionByte = 0x5b // Base58-encodes to 'A...'
AccountSeedVersionByte VersionByte = 0xff // Base58-encodes to 'S...'
DeviceSeedVersionByte VersionByte = 0x7d // Base58-encodes to 'D...'
)