Merge branch 'main' of github.com:anytypeio/any-sync into update-fileproto

This commit is contained in:
Sergey Cherepanov 2023-03-19 19:21:20 +03:00
commit 22a597389d
No known key found for this signature in database
GPG Key ID: 87F8EDE8FBDF637C
57 changed files with 3429 additions and 586 deletions

View File

@ -172,7 +172,7 @@ func (app *App) Start(ctx context.Context) (err error) {
for i, s := range app.components { for i, s := range app.components {
if err = s.Init(app); err != nil { if err = s.Init(app); err != nil {
closeServices(i) closeServices(i)
return fmt.Errorf("can't init service '%s': %v", s.Name(), err) return fmt.Errorf("can't init service '%s': %w", s.Name(), err)
} }
} }
@ -181,7 +181,7 @@ func (app *App) Start(ctx context.Context) (err error) {
start := time.Now() start := time.Now()
if err = serviceRun.Run(ctx); err != nil { if err = serviceRun.Run(ctx); err != nil {
closeServices(i) closeServices(i)
return fmt.Errorf("can't run service '%s': %v", serviceRun.Name(), err) return fmt.Errorf("can't run service '%s': %w", serviceRun.Name(), err)
} }
spent := time.Since(start).Milliseconds() spent := time.Since(start).Milliseconds()
app.startStat.SpentMsTotal += spent app.startStat.SpentMsTotal += spent

View File

@ -1,33 +1,87 @@
package logger package logger
import "go.uber.org/zap" import (
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"github.com/anytypeio/any-sync/util/slice"
)
type LogFormat int
const (
ColorizedOutput LogFormat = iota
PlaintextOutput
JSONOutput
)
type Config struct { type Config struct {
Production bool `yaml:"production"` Production bool `yaml:"production"`
DefaultLevel string `yaml:"defaultLevel"` DefaultLevel string `yaml:"defaultLevel"`
NamedLevels map[string]string `yaml:"namedLevels"` NamedLevels map[string]string `yaml:"namedLevels"`
AddOutputPaths []string `yaml:"outputPaths"`
DisableStdErr bool `yaml:"disableStdErr"`
Format LogFormat `yaml:"format"`
ZapConfig *zap.Config `yaml:"-"` // optional, if set it will be used instead of other config options
} }
func (l Config) ApplyGlobal() { func (l Config) ApplyGlobal() {
var conf zap.Config var conf zap.Config
if l.Production { if l.ZapConfig != nil {
conf = zap.NewProductionConfig() conf = *l.ZapConfig
} else { } else {
conf = zap.NewDevelopmentConfig() if l.Production {
} conf = zap.NewProductionConfig()
if defaultLevel, err := zap.ParseAtomicLevel(l.DefaultLevel); err == nil { } else {
conf.Level = defaultLevel conf = zap.NewDevelopmentConfig()
} }
var levels = make(map[string]zap.AtomicLevel) encConfig := conf.EncoderConfig
for k, v := range l.NamedLevels { switch l.Format {
if lev, err := zap.ParseAtomicLevel(v); err != nil { case PlaintextOutput:
levels[k] = lev encConfig.EncodeLevel = zapcore.CapitalLevelEncoder
conf.Encoding = "console"
case JSONOutput:
encConfig.MessageKey = "msg"
encConfig.TimeKey = "ts"
encConfig.LevelKey = "level"
encConfig.NameKey = "logger"
encConfig.CallerKey = "caller"
encConfig.EncodeTime = zapcore.ISO8601TimeEncoder
conf.Encoding = "json"
default:
// default is ColorizedOutput
conf.Encoding = "console"
encConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
}
conf.EncoderConfig = encConfig
if len(l.AddOutputPaths) > 0 {
conf.OutputPaths = append(conf.OutputPaths, l.AddOutputPaths...)
}
if l.DisableStdErr {
conf.OutputPaths = slice.Filter(conf.OutputPaths, func(path string) bool {
return path != "stderr"
})
}
if defaultLevel, err := zap.ParseAtomicLevel(l.DefaultLevel); err == nil {
conf.Level = defaultLevel
} }
} }
defaultLogger, err := conf.Build() var lvl = make(map[string]zap.AtomicLevel)
for k, v := range l.NamedLevels {
if lev, err := zap.ParseAtomicLevel(v); err == nil {
lvl[k] = lev
// we need to have a minimum level of all named loggers for the main logger
if lev.Level() < conf.Level.Level() {
conf.Level.SetLevel(lev.Level())
}
}
}
lg, err := conf.Build()
if err != nil { if err != nil {
Default().Fatal("can't build logger", zap.Error(err)) Default().Fatal("can't build logger", zap.Error(err))
} }
SetDefault(defaultLogger) SetDefault(lg)
SetNamedLevels(levels) SetNamedLevels(lvl)
} }

View File

@ -32,6 +32,7 @@ func CtxGetFields(ctx context.Context) (fields []zap.Field) {
type CtxLogger struct { type CtxLogger struct {
*zap.Logger *zap.Logger
name string
} }
func (cl CtxLogger) DebugCtx(ctx context.Context, msg string, fields ...zap.Field) { func (cl CtxLogger) DebugCtx(ctx context.Context, msg string, fields ...zap.Field) {
@ -51,5 +52,9 @@ func (cl CtxLogger) ErrorCtx(ctx context.Context, msg string, fields ...zap.Fiel
} }
func (cl CtxLogger) With(fields ...zap.Field) CtxLogger { func (cl CtxLogger) With(fields ...zap.Field) CtxLogger {
return CtxLogger{cl.Logger.With(fields...)} return CtxLogger{cl.Logger.With(fields...), cl.name}
}
func (cl CtxLogger) Sugar() *zap.SugaredLogger {
return NewNamedSugared(cl.name)
} }

View File

@ -1,51 +1,132 @@
package logger package logger
import ( import (
"go.uber.org/zap"
"sync" "sync"
"github.com/gobwas/glob"
"go.uber.org/zap"
) )
var ( var (
mu sync.Mutex mu sync.Mutex
defaultLogger *zap.Logger logger *zap.Logger
levels = make(map[string]zap.AtomicLevel) loggerConfig zap.Config
loggers = make(map[string]CtxLogger) namedLevels = make(map[string]zap.AtomicLevel)
namedGlobs = make(map[string]glob.Glob)
namedLoggers = make(map[string]CtxLogger)
namedSugarLoggers = make(map[string]*zap.SugaredLogger)
) )
func init() { func init() {
defaultLogger, _ = zap.NewDevelopment() loggerConfig = zap.NewDevelopmentConfig()
zap.NewProduction() logger, _ = loggerConfig.Build()
} }
// SetDefault replaces the default logger
// you need to call SetNamedLevels after in case you have named loggers,
// otherwise they will use the old logger
func SetDefault(l *zap.Logger) { func SetDefault(l *zap.Logger) {
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
*defaultLogger = *l *logger = *l
for name, l := range loggers {
*l.Logger = *defaultLogger.Named(name)
}
} }
// SetNamedLevels sets the namedLevels for named loggers
// it also supports glob patterns for names, like "app*"
// can be racy in case there are existing named loggers
// so consider to call only once at the beginning
func SetNamedLevels(l map[string]zap.AtomicLevel) { func SetNamedLevels(l map[string]zap.AtomicLevel) {
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
levels = l namedLevels = l
var minLevel = logger.Level()
for k, l := range namedLevels {
g, err := glob.Compile(k)
if err == nil {
namedGlobs[k] = g
}
namedLevels[k] = l
if l.Level() < minLevel {
minLevel = l.Level()
}
}
if minLevel < logger.Level() {
// recreate logger if the min level is lower than the current min one
loggerConfig.Level = zap.NewAtomicLevelAt(minLevel)
logger, _ = loggerConfig.Build()
}
for name, nl := range namedLoggers {
level := getLevel(name)
newCore := zap.New(logger.Core()).Named(name).WithOptions(
zap.IncreaseLevel(level),
)
*(nl.Logger) = *newCore
}
for name, nl := range namedSugarLoggers {
level := getLevel(name)
newCore := zap.New(logger.Core()).Named(name).WithOptions(
zap.IncreaseLevel(level),
).Sugar()
*(nl) = *newCore
}
} }
func Default() *zap.Logger { func Default() *zap.Logger {
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
return defaultLogger return logger
}
func getLevel(name string) zap.AtomicLevel {
level, ok := namedLevels[name]
if !ok {
var found bool
for globName, glob := range namedGlobs {
if glob.Match(name) {
found = true
level, _ = namedLevels[globName]
// no need to check ok, because we know that globName exists
break
}
}
if !found {
level = loggerConfig.Level
}
}
return level
} }
func NewNamed(name string, fields ...zap.Field) CtxLogger { func NewNamed(name string, fields ...zap.Field) CtxLogger {
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
l := defaultLogger.Named(name)
if len(fields) > 0 { if l, nameExists := namedLoggers[name]; nameExists {
l = l.With(fields...) return l
} }
ctxL := CtxLogger{l}
loggers[name] = ctxL level := getLevel(name)
l := zap.New(logger.Core()).Named(name).WithOptions(zap.IncreaseLevel(level),
zap.Fields(fields...))
ctxL := CtxLogger{Logger: l, name: name}
namedLoggers[name] = ctxL
return ctxL return ctxL
} }
func NewNamedSugared(name string) *zap.SugaredLogger {
mu.Lock()
defer mu.Unlock()
if l, nameExists := namedSugarLoggers[name]; nameExists {
return l
}
level := getLevel(name)
l := zap.New(logger.Core()).Named(name).Sugar().WithOptions(zap.IncreaseLevel(level))
namedSugarLoggers[name] = l
return l
}

120
app/ocache/entry.go Normal file
View File

@ -0,0 +1,120 @@
package ocache
import (
"context"
"go.uber.org/zap"
"sync"
"time"
)
type entryState int
const (
entryStateLoading = iota
entryStateActive
entryStateClosing
entryStateClosed
)
type entry struct {
id string
state entryState
lastUsage time.Time
load chan struct{}
loadErr error
value Object
close chan struct{}
mx sync.Mutex
}
func newEntry(id string, value Object, state entryState) *entry {
return &entry{
id: id,
load: make(chan struct{}),
lastUsage: time.Now(),
state: state,
value: value,
}
}
func (e *entry) isActive() bool {
e.mx.Lock()
defer e.mx.Unlock()
return e.state == entryStateActive
}
func (e *entry) isClosing() bool {
e.mx.Lock()
defer e.mx.Unlock()
return e.state == entryStateClosed || e.state == entryStateClosing
}
func (e *entry) waitLoad(ctx context.Context, id string) (value Object, err error) {
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object load", zap.String("id", id))
return nil, ctx.Err()
case <-e.load:
return e.value, e.loadErr
}
}
func (e *entry) waitClose(ctx context.Context, id string) (res bool, err error) {
e.mx.Lock()
switch e.state {
case entryStateClosing:
waitCh := e.close
e.mx.Unlock()
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object close", zap.String("id", id))
return false, ctx.Err()
case <-waitCh:
return true, nil
}
case entryStateClosed:
e.mx.Unlock()
return true, nil
default:
e.mx.Unlock()
return false, nil
}
}
func (e *entry) setClosing(wait bool) (prevState, curState entryState) {
e.mx.Lock()
prevState = e.state
curState = e.state
if e.state == entryStateClosing {
waitCh := e.close
e.mx.Unlock()
if !wait {
return
}
<-waitCh
e.mx.Lock()
}
if e.state != entryStateClosed {
e.state = entryStateClosing
e.close = make(chan struct{})
}
curState = e.state
e.mx.Unlock()
return
}
func (e *entry) setActive(chClose bool) {
e.mx.Lock()
defer e.mx.Unlock()
if chClose {
close(e.close)
}
e.state = entryStateActive
}
func (e *entry) setClosed() {
e.mx.Lock()
defer e.mx.Unlock()
close(e.close)
e.state = entryStateClosed
}

View File

@ -1,11 +1,18 @@
package ocache package ocache
import "github.com/prometheus/client_golang/prometheus" import (
"github.com/prometheus/client_golang/prometheus"
"strings"
)
func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Option { func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Option {
if subsystem == "" { if subsystem == "" {
subsystem = "cache" subsystem = "cache"
} }
nameSplit := strings.Split(namespace, ".")
subSplit := strings.Split(subsystem, ".")
namespace = strings.Join(nameSplit, "_")
subsystem = strings.Join(subSplit, "_")
if reg == nil { if reg == nil {
return nil return nil
} }

View File

@ -0,0 +1,19 @@
package ocache
import (
"context"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"strings"
"testing"
)
func TestWithPrometheus_MetricsConvertsDots(t *testing.T) {
opt := WithPrometheus(prometheus.NewRegistry(), "some.name", "some.system")
cache := New(func(ctx context.Context, id string) (value Object, err error) {
return &testObject{}, nil
}, opt).(*oCache)
_, err := cache.Get(context.Background(), "id")
require.NoError(t, err)
require.True(t, strings.Contains(cache.metrics.hit.Desc().String(), "some_name_some_system_hit"))
}

View File

@ -44,12 +44,6 @@ var WithGCPeriod = func(gcPeriod time.Duration) Option {
} }
} }
var WithRefCounter = func(enable bool) Option {
return func(cache *oCache) {
cache.refCounter = enable
}
}
func New(loadFunc LoadFunc, opts ...Option) OCache { func New(loadFunc LoadFunc, opts ...Option) OCache {
c := &oCache{ c := &oCache{
data: make(map[string]*entry), data: make(map[string]*entry),
@ -73,33 +67,7 @@ func New(loadFunc LoadFunc, opts ...Option) OCache {
type Object interface { type Object interface {
Close() (err error) Close() (err error)
} TryClose(objectTTL time.Duration) (res bool, err error)
type ObjectLocker interface {
Object
Locked() bool
}
type ObjectLastUsage interface {
LastUsage() time.Time
}
type entry struct {
id string
lastUsage time.Time
refCount uint32
isClosing bool
load chan struct{}
loadErr error
value Object
close chan struct{}
}
func (e *entry) locked() bool {
if locker, ok := e.value.(ObjectLocker); ok {
return locker.Locked()
}
return false
} }
type OCache interface { type OCache interface {
@ -116,12 +84,8 @@ type OCache interface {
// Add adds new object to cache // Add adds new object to cache
// Returns error when object exists // Returns error when object exists
Add(id string, value Object) (err error) Add(id string, value Object) (err error)
// Release decreases the refs counter
Release(id string) bool
// Reset sets refs counter to 0
Reset(id string) bool
// Remove closes and removes object // Remove closes and removes object
Remove(id string) (ok bool, err error) Remove(ctx context.Context, id string) (ok bool, err error)
// ForEach iterates over all loaded objects, breaks when callback returns false // ForEach iterates over all loaded objects, breaks when callback returns false
ForEach(f func(v Object) (isContinue bool)) ForEach(f func(v Object) (isContinue bool))
// GC frees not used and expired objects // GC frees not used and expired objects
@ -134,17 +98,16 @@ type OCache interface {
} }
type oCache struct { type oCache struct {
mu sync.Mutex mu sync.Mutex
data map[string]*entry data map[string]*entry
loadFunc LoadFunc loadFunc LoadFunc
timeNow func() time.Time timeNow func() time.Time
ttl time.Duration ttl time.Duration
gc time.Duration gc time.Duration
closed bool closed bool
closeCh chan struct{} closeCh chan struct{}
log *zap.SugaredLogger log *zap.SugaredLogger
metrics *metrics metrics *metrics
refCounter bool
} }
func (c *oCache) Get(ctx context.Context, id string) (value Object, err error) { func (c *oCache) Get(ctx context.Context, id string) (value Object, err error) {
@ -160,69 +123,36 @@ Load:
return nil, ErrClosed return nil, ErrClosed
} }
if e, ok = c.data[id]; !ok { if e, ok = c.data[id]; !ok {
e = newEntry(id, nil, entryStateLoading)
load = true load = true
e = &entry{
id: id,
load: make(chan struct{}),
}
c.data[id] = e c.data[id] = e
} }
closing := e.isClosing e.lastUsage = time.Now()
if !e.isClosing {
e.lastUsage = c.timeNow()
if c.refCounter {
e.refCount++
}
}
c.mu.Unlock() c.mu.Unlock()
if closing { reload, err := e.waitClose(ctx, id)
select { if err != nil {
case <-ctx.Done(): return nil, err
log.DebugCtx(ctx, "ctx done while waiting on object close", zap.String("id", id)) }
return nil, ctx.Err() if reload {
case <-e.close: goto Load
goto Load
}
} }
if load { if load {
go c.load(ctx, id, e) go c.load(ctx, id, e)
} }
if c.metrics != nil { c.metricsGet(!load)
if load { return e.waitLoad(ctx, id)
c.metrics.miss.Inc()
} else {
c.metrics.hit.Inc()
}
}
select {
case <-ctx.Done():
log.DebugCtx(ctx, "ctx done while waiting on object load", zap.String("id", id))
return nil, ctx.Err()
case <-e.load:
}
return e.value, e.loadErr
} }
func (c *oCache) Pick(ctx context.Context, id string) (value Object, err error) { func (c *oCache) Pick(ctx context.Context, id string) (value Object, err error) {
c.mu.Lock() c.mu.Lock()
val, ok := c.data[id] val, ok := c.data[id]
if !ok || val.isClosing { if !ok || val.isClosing() {
c.mu.Unlock() c.mu.Unlock()
return nil, ErrNotExists return nil, ErrNotExists
} }
c.mu.Unlock() c.mu.Unlock()
c.metricsGet(true)
if c.metrics != nil { return val.waitLoad(ctx, id)
c.metrics.hit.Inc()
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-val.load:
return val.value, val.loadErr
}
} }
func (c *oCache) load(ctx context.Context, id string, e *entry) { func (c *oCache) load(ctx context.Context, id string, e *entry) {
@ -236,63 +166,39 @@ func (c *oCache) load(ctx context.Context, id string, e *entry) {
delete(c.data, id) delete(c.data, id)
} else { } else {
e.value = value e.value = value
e.setActive(false)
} }
} }
func (c *oCache) Release(id string) bool { func (c *oCache) Remove(ctx context.Context, id string) (ok bool, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return false
}
if e, ok := c.data[id]; ok {
if c.refCounter && e.refCount > 0 {
e.refCount--
return true
}
}
return false
}
func (c *oCache) Reset(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return false
}
if e, ok := c.data[id]; ok {
e.refCount = 0
return true
}
return false
}
func (c *oCache) Remove(id string) (ok bool, err error) {
c.mu.Lock() c.mu.Lock()
if c.closed { if c.closed {
c.mu.Unlock() c.mu.Unlock()
err = ErrClosed err = ErrClosed
return return
} }
var e *entry e, ok := c.data[id]
e, ok = c.data[id] if !ok {
if !ok || e.isClosing {
c.mu.Unlock() c.mu.Unlock()
return return false, ErrNotExists
} }
e.isClosing = true
e.close = make(chan struct{})
c.mu.Unlock() c.mu.Unlock()
return c.remove(ctx, e)
}
<-e.load func (c *oCache) remove(ctx context.Context, e *entry) (ok bool, err error) {
if e.value != nil { if _, err = e.waitLoad(ctx, e.id); err != nil {
return false, err
}
_, curState := e.setClosing(true)
if curState == entryStateClosing {
ok = true
err = e.value.Close() err = e.value.Close()
c.mu.Lock()
e.setClosed()
delete(c.data, e.id)
c.mu.Unlock()
} }
c.mu.Lock()
close(e.close)
delete(c.data, e.id)
c.mu.Unlock()
return return
} }
@ -314,13 +220,7 @@ func (c *oCache) Add(id string, value Object) (err error) {
if _, ok := c.data[id]; ok { if _, ok := c.data[id]; ok {
return ErrExists return ErrExists
} }
e := &entry{ e := newEntry(id, value, entryStateActive)
id: id,
lastUsage: time.Now(),
refCount: 0,
load: make(chan struct{}),
value: value,
}
close(e.load) close(e.load)
c.data[id] = e c.data[id] = e
return return
@ -332,7 +232,7 @@ func (c *oCache) ForEach(f func(obj Object) (isContinue bool)) {
for _, v := range c.data { for _, v := range c.data {
select { select {
case <-v.load: case <-v.load:
if v.value != nil && !v.isClosing { if v.value != nil && !v.isClosing() {
objects = append(objects, v.value) objects = append(objects, v.value)
} }
default: default:
@ -368,40 +268,35 @@ func (c *oCache) GC() {
deadline := c.timeNow().Add(-c.ttl) deadline := c.timeNow().Add(-c.ttl)
var toClose []*entry var toClose []*entry
for _, e := range c.data { for _, e := range c.data {
if e.isClosing { if e.isActive() && e.lastUsage.Before(deadline) {
continue
}
lu := e.lastUsage
if lug, ok := e.value.(ObjectLastUsage); ok {
lu = lug.LastUsage()
}
if !e.locked() && e.refCount <= 0 && lu.Before(deadline) {
e.isClosing = true
e.close = make(chan struct{}) e.close = make(chan struct{})
toClose = append(toClose, e) toClose = append(toClose, e)
} }
} }
size := len(c.data) size := len(c.data)
c.mu.Unlock() c.mu.Unlock()
closedNum := 0
for _, e := range toClose { for _, e := range toClose {
<-e.load prevState, _ := e.setClosing(false)
if e.value != nil { if prevState == entryStateClosing || prevState == entryStateClosed {
if err := e.value.Close(); err != nil { continue
c.log.With("object_id", e.id).Warnf("GC: object close error: %v", err) }
} closed, err := e.value.TryClose(c.ttl)
if err != nil {
c.log.With("object_id", e.id).Warnf("GC: object close error: %v", err)
}
if !closed {
e.setActive(true)
continue
} else {
closedNum++
c.mu.Lock()
e.setClosed()
delete(c.data, e.id)
c.mu.Unlock()
} }
} }
c.log.Infof("GC: removed %d; cache size: %d", len(toClose), size) c.metricsClosed(closedNum, size)
if len(toClose) > 0 && c.metrics != nil {
c.metrics.gc.Add(float64(len(toClose)))
}
c.mu.Lock()
for _, e := range toClose {
close(e.close)
delete(c.data, e.id)
}
c.mu.Unlock()
} }
func (c *oCache) Len() int { func (c *oCache) Len() int {
@ -418,25 +313,34 @@ func (c *oCache) Close() (err error) {
} }
c.closed = true c.closed = true
close(c.closeCh) close(c.closeCh)
var toClose, alreadyClosing []*entry var toClose []*entry
for _, e := range c.data { for _, e := range c.data {
if e.isClosing { toClose = append(toClose, e)
alreadyClosing = append(alreadyClosing, e)
} else {
toClose = append(toClose, e)
}
} }
c.mu.Unlock() c.mu.Unlock()
for _, e := range toClose { for _, e := range toClose {
<-e.load if _, err := c.remove(context.Background(), e); err != nil && err != ErrNotExists {
if e.value != nil { c.log.With("object_id", e.id).Warnf("cache close: object close error: %v", err)
if clErr := e.value.Close(); clErr != nil {
c.log.With("object_id", e.id).Warnf("cache close: object close error: %v", clErr)
}
} }
} }
for _, e := range alreadyClosing {
<-e.close
}
return nil return nil
} }
func (c *oCache) metricsGet(hit bool) {
if c.metrics == nil {
return
}
if hit {
c.metrics.hit.Inc()
} else {
c.metrics.miss.Inc()
}
}
func (c *oCache) metricsClosed(closedLen, size int) {
c.log.Infof("GC: removed %d; cache size: %d", closedLen, size)
if c.metrics == nil || closedLen == 0 {
return
}
c.metrics.gc.Add(float64(closedLen))
}

View File

@ -3,6 +3,8 @@ package ocache
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"sync"
"sync/atomic" "sync/atomic"
"testing" "testing"
"time" "time"
@ -11,26 +13,48 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
var ctx = context.Background()
type testObject struct { type testObject struct {
name string name string
closeErr error closeErr error
closeCh chan struct{} closeCh chan struct{}
tryReturn bool
closeCalled bool
tryCloseCalled bool
} }
func NewTestObject(name string, closeCh chan struct{}) *testObject { func NewTestObject(name string, tryReturn bool, closeCh chan struct{}) *testObject {
return &testObject{ return &testObject{
name: name, name: name,
closeCh: closeCh, closeCh: closeCh,
tryReturn: tryReturn,
} }
} }
func (t *testObject) Close() (err error) { func (t *testObject) Close() (err error) {
if t.closeCalled || (t.tryCloseCalled && t.tryReturn) {
panic("close called twice")
}
t.closeCalled = true
if t.closeCh != nil { if t.closeCh != nil {
<-t.closeCh <-t.closeCh
} }
return t.closeErr return t.closeErr
} }
func (t *testObject) TryClose(objectTTL time.Duration) (res bool, err error) {
if t.closeCalled || (t.tryCloseCalled && t.tryReturn) {
panic("close called twice")
}
t.tryCloseCalled = true
if t.closeCh != nil {
<-t.closeCh
return t.tryReturn, t.closeErr
}
return t.tryReturn, nil
}
func TestOCache_Get(t *testing.T) { func TestOCache_Get(t *testing.T) {
t.Run("successful", func(t *testing.T) { t.Run("successful", func(t *testing.T) {
c := New(func(ctx context.Context, id string) (value Object, err error) { c := New(func(ctx context.Context, id string) (value Object, err error) {
@ -116,42 +140,37 @@ func TestOCache_Get(t *testing.T) {
} }
func TestOCache_GC(t *testing.T) { func TestOCache_GC(t *testing.T) {
t.Run("test without close wait", func(t *testing.T) { t.Run("test gc expired object", func(t *testing.T) {
c := New(func(ctx context.Context, id string) (value Object, err error) { c := New(func(ctx context.Context, id string) (value Object, err error) {
return &testObject{name: id}, nil return NewTestObject(id, true, nil), nil
}, WithTTL(time.Millisecond*10), WithRefCounter(true)) }, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id") val, err := c.Get(context.TODO(), "id")
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, val) require.NotNil(t, val)
assert.Equal(t, 1, c.Len()) assert.Equal(t, 1, c.Len())
c.GC() c.GC()
assert.Equal(t, 1, c.Len()) assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 30) time.Sleep(time.Millisecond * 20)
c.GC()
assert.Equal(t, 1, c.Len())
assert.True(t, c.Release("id"))
c.GC() c.GC()
assert.Equal(t, 0, c.Len()) assert.Equal(t, 0, c.Len())
assert.False(t, c.Release("id"))
}) })
t.Run("test with close wait", func(t *testing.T) { t.Run("test gc tryClose true, close before get", func(t *testing.T) {
closeCh := make(chan struct{}) closeCh := make(chan struct{})
getCh := make(chan struct{}) getCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) { c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, closeCh), nil return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10), WithRefCounter(true)) }, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id") val, err := c.Get(context.TODO(), "id")
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, val) require.NotNil(t, val)
assert.Equal(t, 1, c.Len()) assert.Equal(t, 1, c.Len())
assert.True(t, c.Release("id"))
// making ttl pass // making ttl pass
time.Sleep(time.Millisecond * 40) time.Sleep(time.Millisecond * 20)
// first gc will be run after 20 secs, so calling it manually // first gc will be run after 20 secs, so calling it manually
go c.GC() go c.GC()
// waiting until all objects are marked as closing // waiting until all objects are marked as closing
time.Sleep(time.Millisecond * 40) time.Sleep(time.Millisecond * 20)
var events []string var events []string
go func() { go func() {
_, err := c.Get(context.TODO(), "id") _, err := c.Get(context.TODO(), "id")
@ -160,47 +179,313 @@ func TestOCache_GC(t *testing.T) {
events = append(events, "get") events = append(events, "get")
close(getCh) close(getCh)
}() }()
events = append(events, "close")
// sleeping to make sure that Get is called // sleeping to make sure that Get is called
time.Sleep(time.Millisecond * 40) time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh) close(closeCh)
<-getCh <-getCh
require.Equal(t, []string{"close", "get"}, events) require.Equal(t, []string{"close", "get"}, events)
}) })
t.Run("test gc tryClose false, many parallel get", func(t *testing.T) {
timesCalled := &atomic.Int32{}
obj := NewTestObject("id", false, nil)
c := New(func(ctx context.Context, id string) (value Object, err error) {
timesCalled.Add(1)
return obj, nil
}, WithTTL(0))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
begin := make(chan struct{})
wg := sync.WaitGroup{}
once := sync.Once{}
wg.Add(1)
go func() {
<-begin
c.GC()
wg.Done()
}()
for i := 0; i < 50; i++ {
wg.Add(1)
go func(i int) {
once.Do(func() {
close(begin)
})
if i%2 != 0 {
time.Sleep(time.Millisecond)
}
_, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
wg.Done()
}(i)
}
require.NoError(t, err)
wg.Wait()
require.Equal(t, timesCalled.Load(), int32(1))
require.True(t, obj.tryCloseCalled)
})
t.Run("test gc tryClose different, many objects", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
called := make(map[string]int)
max := 1000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
called[id] = called[id] + 1
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Millisecond*10))
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
assert.Equal(t, max, c.Len())
time.Sleep(time.Millisecond * 20)
c.GC()
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
require.Equal(t, called[getId(i)], i%2+1)
}
})
} }
func Test_OCache_Remove(t *testing.T) { func Test_OCache_Remove(t *testing.T) {
closeCh := make(chan struct{}) t.Run("remove simple", func(t *testing.T) {
getCh := make(chan struct{}) closeCh := make(chan struct{})
getCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, false, closeCh), nil
}, WithTTL(time.Millisecond*10))
c := New(func(ctx context.Context, id string) (value Object, err error) { val, err := c.Get(context.TODO(), "id")
return NewTestObject(id, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
// removing the object, so we will wait on closing
go func() {
_, err := c.Remove("id")
require.NoError(t, err)
}()
time.Sleep(time.Millisecond * 40)
var events []string
go func() {
_, err := c.Get(context.TODO(), "id")
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, val) require.NotNil(t, val)
events = append(events, "get") assert.Equal(t, 1, c.Len())
close(getCh) // removing the object, so we will wait on closing
}() go func() {
events = append(events, "close") _, err := c.Remove(ctx, "id")
// sleeping to make sure that Get is called require.NoError(t, err)
time.Sleep(time.Millisecond * 40) }()
close(closeCh) time.Sleep(time.Millisecond * 20)
<-getCh var events []string
require.Equal(t, []string{"close", "get"}, events) go func() {
_, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
events = append(events, "get")
close(getCh)
}()
// sleeping to make sure that Get is called
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-getCh
require.Equal(t, []string{"close", "get"}, events)
})
t.Run("test remove while gc, tryClose false", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, false, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 20)
go c.GC()
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.True(t, ok)
events = append(events, "remove")
close(removeCh)
}()
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-removeCh
require.Equal(t, []string{"close", "remove"}, events)
})
t.Run("test remove while gc, tryClose true", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
time.Sleep(time.Millisecond * 20)
go c.GC()
time.Sleep(time.Millisecond * 20)
var events []string
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.False(t, ok)
events = append(events, "remove")
close(removeCh)
}()
time.Sleep(time.Millisecond * 20)
events = append(events, "close")
close(closeCh)
<-removeCh
require.Equal(t, []string{"close", "remove"}, events)
})
t.Run("test gc while remove, tryClose true", func(t *testing.T) {
closeCh := make(chan struct{})
removeCh := make(chan struct{})
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, true, closeCh), nil
}, WithTTL(time.Millisecond*10))
val, err := c.Get(context.TODO(), "id")
require.NoError(t, err)
require.NotNil(t, val)
assert.Equal(t, 1, c.Len())
go func() {
ok, err := c.Remove(ctx, "id")
require.NoError(t, err)
require.True(t, ok)
close(removeCh)
}()
time.Sleep(20 * time.Millisecond)
c.GC()
close(closeCh)
<-removeCh
})
}
func TestOCacheFuzzy(t *testing.T) {
t.Run("test many objects gc, get and remove simultaneously, close after", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
max := 2000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Nanosecond))
stopGC := make(chan struct{})
wg := sync.WaitGroup{}
go func() {
for {
select {
case <-stopGC:
return
default:
c.GC()
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
require.NoError(t, err)
require.NotNil(t, val)
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
c.Remove(ctx, getId(i))
}
}
}()
wg.Wait()
close(stopGC)
err := c.Close()
require.NoError(t, err)
require.Equal(t, 0, c.Len())
})
t.Run("test many objects gc, get, remove and close simultaneously", func(t *testing.T) {
tryCloseIds := make(map[string]bool)
max := 2000
getId := func(i int) string {
return fmt.Sprintf("id%d", i)
}
for i := 0; i < max; i++ {
if i%2 == 1 {
tryCloseIds[getId(i)] = true
} else {
tryCloseIds[getId(i)] = false
}
}
c := New(func(ctx context.Context, id string) (value Object, err error) {
return NewTestObject(id, tryCloseIds[id], nil), nil
}, WithTTL(time.Nanosecond))
go func() {
for {
c.GC()
}
}()
go func() {
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
val, err := c.Get(context.TODO(), getId(i))
if err == ErrClosed {
return
}
require.NoError(t, err)
require.NotNil(t, val)
}
}
}()
go func() {
for j := 0; j < 10; j++ {
for i := 0; i < max; i++ {
c.Remove(ctx, getId(i))
}
}
}()
time.Sleep(time.Millisecond)
err := c.Close()
require.NoError(t, err)
require.Equal(t, 0, c.Len())
})
} }

View File

@ -0,0 +1,52 @@
//go:generate mockgen -destination mock_credentialprovider/mock_credentialprovider.go github.com/anytypeio/any-sync/commonspace/credentialprovider CredentialProvider
package credentialprovider
import (
"context"
"github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/coordinator/coordinatorclient"
"github.com/gogo/protobuf/proto"
)
const CName = "common.commonspace.credentialprovider"
func New() app.Component {
return &credentialProvider{}
}
func NewNoOp() CredentialProvider {
return &noOpProvider{}
}
type CredentialProvider interface {
GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error)
}
type noOpProvider struct {
}
func (n noOpProvider) GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
return nil, nil
}
type credentialProvider struct {
client coordinatorclient.CoordinatorClient
}
func (c *credentialProvider) Init(a *app.App) (err error) {
c.client = a.MustComponent(coordinatorclient.CName).(coordinatorclient.CoordinatorClient)
return
}
func (c *credentialProvider) Name() (name string) {
return CName
}
func (c *credentialProvider) GetCredential(ctx context.Context, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
receipt, err := c.client.SpaceSign(ctx, spaceHeader.Id, spaceHeader.RawHeader)
if err != nil {
return nil, err
}
return proto.Marshal(receipt)
}

View File

@ -0,0 +1,51 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/anytypeio/any-sync/commonspace/credentialprovider (interfaces: CredentialProvider)
// Package mock_credentialprovider is a generated GoMock package.
package mock_credentialprovider
import (
context "context"
reflect "reflect"
spacesyncproto "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
gomock "github.com/golang/mock/gomock"
)
// MockCredentialProvider is a mock of CredentialProvider interface.
type MockCredentialProvider struct {
ctrl *gomock.Controller
recorder *MockCredentialProviderMockRecorder
}
// MockCredentialProviderMockRecorder is the mock recorder for MockCredentialProvider.
type MockCredentialProviderMockRecorder struct {
mock *MockCredentialProvider
}
// NewMockCredentialProvider creates a new mock instance.
func NewMockCredentialProvider(ctrl *gomock.Controller) *MockCredentialProvider {
mock := &MockCredentialProvider{ctrl: ctrl}
mock.recorder = &MockCredentialProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockCredentialProvider) EXPECT() *MockCredentialProviderMockRecorder {
return m.recorder
}
// GetCredential mocks base method.
func (m *MockCredentialProvider) GetCredential(arg0 context.Context, arg1 *spacesyncproto.RawSpaceHeaderWithId) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetCredential", arg0, arg1)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetCredential indicates an expected call of GetCredential.
func (mr *MockCredentialProviderMockRecorder) GetCredential(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredential", reflect.TypeOf((*MockCredentialProvider)(nil).GetCredential), arg0, arg1)
}

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/treegetter" "github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/peermanager"
@ -33,29 +34,32 @@ func newDiffSyncer(
storage spacestorage.SpaceStorage, storage spacestorage.SpaceStorage,
clientFactory spacesyncproto.ClientFactory, clientFactory spacesyncproto.ClientFactory,
syncStatus syncstatus.StatusUpdater, syncStatus syncstatus.StatusUpdater,
credentialProvider credentialprovider.CredentialProvider,
log logger.CtxLogger) DiffSyncer { log logger.CtxLogger) DiffSyncer {
return &diffSyncer{ return &diffSyncer{
diff: diff, diff: diff,
spaceId: spaceId, spaceId: spaceId,
cache: cache, cache: cache,
storage: storage, storage: storage,
peerManager: peerManager, peerManager: peerManager,
clientFactory: clientFactory, clientFactory: clientFactory,
log: log, credentialProvider: credentialProvider,
syncStatus: syncStatus, log: log,
syncStatus: syncStatus,
} }
} }
type diffSyncer struct { type diffSyncer struct {
spaceId string spaceId string
diff ldiff.Diff diff ldiff.Diff
peerManager peermanager.PeerManager peerManager peermanager.PeerManager
cache treegetter.TreeGetter cache treegetter.TreeGetter
storage spacestorage.SpaceStorage storage spacestorage.SpaceStorage
clientFactory spacesyncproto.ClientFactory clientFactory spacesyncproto.ClientFactory
log logger.CtxLogger log logger.CtxLogger
deletionState settingsstate.ObjectDeletionState deletionState settingsstate.ObjectDeletionState
syncStatus syncstatus.StatusUpdater credentialProvider credentialprovider.CredentialProvider
syncStatus syncstatus.StatusUpdater
} }
func (d *diffSyncer) Init(deletionState settingsstate.ObjectDeletionState) { func (d *diffSyncer) Init(deletionState settingsstate.ObjectDeletionState) {
@ -86,6 +90,7 @@ func (d *diffSyncer) UpdateHeads(id string, heads []string) {
} }
func (d *diffSyncer) Sync(ctx context.Context) error { func (d *diffSyncer) Sync(ctx context.Context) error {
// TODO: split diffsyncer into components
st := time.Now() st := time.Now()
// diffing with responsible peers according to configuration // diffing with responsible peers according to configuration
peers, err := d.peerManager.GetResponsiblePeers(ctx) peers, err := d.peerManager.GetResponsiblePeers(ctx)
@ -117,6 +122,10 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error)
newIds, changedIds, removedIds, err := d.diff.Diff(ctx, rdiff) newIds, changedIds, removedIds, err := d.diff.Diff(ctx, rdiff)
err = rpcerr.Unwrap(err) err = rpcerr.Unwrap(err)
if err != nil && err != spacesyncproto.ErrSpaceMissing { if err != nil && err != spacesyncproto.ErrSpaceMissing {
if err == spacesyncproto.ErrSpaceIsDeleted {
d.log.Debug("got space deleted while syncing")
d.syncTrees(ctx, p.Id(), []string{d.storage.SpaceSettingsId()})
}
d.syncStatus.SetNodesOnline(p.Id(), false) d.syncStatus.SetNodesOnline(p.Id(), false)
return fmt.Errorf("diff error: %v", err) return fmt.Errorf("diff error: %v", err)
} }
@ -192,6 +201,10 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, peerId string, cl
return return
} }
cred, err := d.credentialProvider.GetCredential(ctx, header)
if err != nil {
return
}
spacePayload := &spacesyncproto.SpacePayload{ spacePayload := &spacesyncproto.SpacePayload{
SpaceHeader: header, SpaceHeader: header,
AclPayload: root.Payload, AclPayload: root.Payload,
@ -200,7 +213,8 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, peerId string, cl
SpaceSettingsPayloadId: spaceSettingsRoot.Id, SpaceSettingsPayloadId: spaceSettingsRoot.Id,
} }
_, err = cl.SpacePush(ctx, &spacesyncproto.SpacePushRequest{ _, err = cl.SpacePush(ctx, &spacesyncproto.SpacePushRequest{
Payload: spacePayload, Payload: spacePayload,
Credential: cred,
}) })
if err != nil { if err != nil {
return return

View File

@ -1,11 +1,13 @@
package headsync package headsync
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/ldiff/mock_ldiff" "github.com/anytypeio/any-sync/app/ldiff/mock_ldiff"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider/mock_credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage" "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@ -30,6 +32,7 @@ type pushSpaceRequestMatcher struct {
spaceId string spaceId string
aclRootId string aclRootId string
settingsId string settingsId string
credential []byte
spaceHeader *spacesyncproto.RawSpaceHeaderWithId spaceHeader *spacesyncproto.RawSpaceHeaderWithId
} }
@ -39,7 +42,7 @@ func (p pushSpaceRequestMatcher) Matches(x interface{}) bool {
return false return false
} }
return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader && res.Payload.SpaceSettingsPayloadId == p.settingsId return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader && res.Payload.SpaceSettingsPayloadId == p.settingsId && bytes.Equal(p.credential, res.Credential)
} }
func (p pushSpaceRequestMatcher) String() string { func (p pushSpaceRequestMatcher) String() string {
@ -48,6 +51,10 @@ func (p pushSpaceRequestMatcher) String() string {
type mockPeer struct{} type mockPeer struct{}
func (m mockPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, m.Close()
}
func (m mockPeer) Id() string { func (m mockPeer) Id() string {
return "mockId" return "mockId"
} }
@ -83,11 +90,13 @@ func newPushSpaceRequestMatcher(
spaceId string, spaceId string,
aclRootId string, aclRootId string,
settingsId string, settingsId string,
credential []byte,
spaceHeader *spacesyncproto.RawSpaceHeaderWithId) *pushSpaceRequestMatcher { spaceHeader *spacesyncproto.RawSpaceHeaderWithId) *pushSpaceRequestMatcher {
return &pushSpaceRequestMatcher{ return &pushSpaceRequestMatcher{
spaceId: spaceId, spaceId: spaceId,
aclRootId: aclRootId, aclRootId: aclRootId,
settingsId: settingsId, settingsId: settingsId,
credential: credential,
spaceHeader: spaceHeader, spaceHeader: spaceHeader,
} }
} }
@ -106,11 +115,12 @@ func TestDiffSyncer_Sync(t *testing.T) {
factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient { factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient {
return clientMock return clientMock
}) })
credentialProvider := mock_credentialprovider.NewMockCredentialProvider(ctrl)
delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl)
spaceId := "spaceId" spaceId := "spaceId"
aclRootId := "aclRootId" aclRootId := "aclRootId"
l := logger.NewNamed(spaceId) l := logger.NewNamed(spaceId)
diffSyncer := newDiffSyncer(spaceId, diffMock, peerManagerMock, cacheMock, stMock, factory, syncstatus.NewNoOpSyncStatus(), l) diffSyncer := newDiffSyncer(spaceId, diffMock, peerManagerMock, cacheMock, stMock, factory, syncstatus.NewNoOpSyncStatus(), credentialProvider, l)
delState.EXPECT().AddObserver(gomock.Any()) delState.EXPECT().AddObserver(gomock.Any())
diffSyncer.Init(delState) diffSyncer.Init(delState)
@ -172,6 +182,7 @@ func TestDiffSyncer_Sync(t *testing.T) {
} }
spaceHeader := &spacesyncproto.RawSpaceHeaderWithId{} spaceHeader := &spacesyncproto.RawSpaceHeaderWithId{}
spaceSettingsId := "spaceSettingsId" spaceSettingsId := "spaceSettingsId"
credential := []byte("credential")
peerManagerMock.EXPECT(). peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()). GetResponsiblePeers(gomock.Any()).
@ -189,15 +200,18 @@ func TestDiffSyncer_Sync(t *testing.T) {
aclStorageMock.EXPECT(). aclStorageMock.EXPECT().
Root(). Root().
Return(aclRoot, nil) Return(aclRoot, nil)
credentialProvider.EXPECT().
GetCredential(gomock.Any(), spaceHeader).
Return(credential, nil)
clientMock.EXPECT(). clientMock.EXPECT().
SpacePush(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, settingsId, spaceHeader)). SpacePush(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, settingsId, credential, spaceHeader)).
Return(nil, nil) Return(nil, nil)
peerManagerMock.EXPECT().SendPeer(gomock.Any(), "mockId", gomock.Any()) peerManagerMock.EXPECT().SendPeer(gomock.Any(), "mockId", gomock.Any())
require.NoError(t, diffSyncer.Sync(ctx)) require.NoError(t, diffSyncer.Sync(ctx))
}) })
t.Run("diff syncer sync other error", func(t *testing.T) { t.Run("diff syncer sync unexpected", func(t *testing.T) {
peerManagerMock.EXPECT(). peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()). GetResponsiblePeers(gomock.Any()).
Return([]peer.Peer{mockPeer{}}, nil) Return([]peer.Peer{mockPeer{}}, nil)
@ -207,4 +221,19 @@ func TestDiffSyncer_Sync(t *testing.T) {
require.NoError(t, diffSyncer.Sync(ctx)) require.NoError(t, diffSyncer.Sync(ctx))
}) })
t.Run("diff syncer sync space is deleted error", func(t *testing.T) {
peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()).
Return([]peer.Peer{mockPeer{}}, nil)
diffMock.EXPECT().
Diff(gomock.Any(), gomock.Eq(NewRemoteDiff(spaceId, clientMock))).
Return(nil, nil, nil, spacesyncproto.ErrSpaceIsDeleted)
stMock.EXPECT().SpaceSettingsId().Return("settingsId")
cacheMock.EXPECT().
GetTree(gomock.Any(), spaceId, "settingsId").
Return(nil, nil)
require.NoError(t, diffSyncer.Sync(ctx))
})
} }

View File

@ -5,6 +5,7 @@ import (
"context" "context"
"github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/ldiff"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/object/treegetter" "github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
@ -60,12 +61,13 @@ func NewHeadSync(
peerManager peermanager.PeerManager, peerManager peermanager.PeerManager,
cache treegetter.TreeGetter, cache treegetter.TreeGetter,
syncStatus syncstatus.StatusUpdater, syncStatus syncstatus.StatusUpdater,
credentialProvider credentialprovider.CredentialProvider,
log logger.CtxLogger) HeadSync { log logger.CtxLogger) HeadSync {
diff := ldiff.New(16, 16) diff := ldiff.New(16, 16)
l := log.With(zap.String("spaceId", spaceId)) l := log.With(zap.String("spaceId", spaceId))
factory := spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient) factory := spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient)
syncer := newDiffSyncer(spaceId, diff, peerManager, cache, storage, factory, syncStatus, l) syncer := newDiffSyncer(spaceId, diff, peerManager, cache, storage, factory, syncStatus, credentialProvider, l)
sync := func(ctx context.Context) (err error) { sync := func(ctx context.Context) (err error) {
// for clients cancelling the sync process // for clients cancelling the sync process
if spaceIsDeleted.Load() && !configuration.IsResponsible(spaceId) { if spaceIsDeleted.Load() && !configuration.IsResponsible(spaceId) {

View File

@ -88,7 +88,7 @@ func (a *aclRecordBuilder) BuildUserJoin(acceptPrivKeyBytes []byte, encSymKeyByt
Identity: state.Identity(), Identity: state.Identity(),
Data: marshalledJoin, Data: marshalledJoin,
CurrentReadKeyHash: state.CurrentReadKeyHash(), CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
} }
marshalledRecord, err := aclRecord.Marshal() marshalledRecord, err := aclRecord.Marshal()
if err != nil { if err != nil {

View File

@ -4,8 +4,10 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"hash/fnv"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
aclrecordproto "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/keychain" "github.com/anytypeio/any-sync/commonspace/object/keychain"
"github.com/anytypeio/any-sync/util/keys" "github.com/anytypeio/any-sync/util/keys"
"github.com/anytypeio/any-sync/util/keys/asymmetric/encryptionkey" "github.com/anytypeio/any-sync/util/keys/asymmetric/encryptionkey"
@ -13,10 +15,9 @@ import (
"github.com/anytypeio/any-sync/util/keys/symmetric" "github.com/anytypeio/any-sync/util/keys/symmetric"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
"go.uber.org/zap" "go.uber.org/zap"
"hash/fnv"
) )
var log = logger.NewNamed("acllist").Sugar() var log = logger.NewNamedSugared("common.commonspace.acllist")
var ( var (
ErrNoSuchUser = errors.New("no such user") ErrNoSuchUser = errors.New("no such user")

View File

@ -126,7 +126,7 @@ func (t *AclListStorageBuilder) parseRecord(rec *Record, prevId string) *aclreco
Identity: []byte(t.keychain.GetIdentity(rec.Identity)), Identity: []byte(t.keychain.GetIdentity(rec.Identity)),
Data: bytes, Data: bytes,
CurrentReadKeyHash: k.Hash, CurrentReadKeyHash: k.Hash,
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
} }
} }

View File

@ -3,6 +3,7 @@ package objecttree
import ( import (
"errors" "errors"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/gogo/protobuf/proto"
) )
var ( var (
@ -48,6 +49,11 @@ func NewChange(id string, ch *treechangeproto.TreeChange, signature []byte) *Cha
} }
func NewChangeFromRoot(id string, ch *treechangeproto.RootChange, signature []byte) *Change { func NewChangeFromRoot(id string, ch *treechangeproto.RootChange, signature []byte) *Change {
changeInfo := &treechangeproto.TreeChangeInfo{
ChangeType: ch.ChangeType,
ChangePayload: ch.ChangePayload,
}
data, _ := proto.Marshal(changeInfo)
return &Change{ return &Change{
Next: nil, Next: nil,
AclHeadId: ch.AclHeadId, AclHeadId: ch.AclHeadId,
@ -56,7 +62,8 @@ func NewChangeFromRoot(id string, ch *treechangeproto.RootChange, signature []by
Timestamp: ch.Timestamp, Timestamp: ch.Timestamp,
Identity: string(ch.Identity), Identity: string(ch.Identity),
Signature: signature, Signature: signature,
Data: []byte(ch.ChangeType), Data: data,
Model: changeInfo,
} }
} }

View File

@ -26,13 +26,14 @@ type BuilderContent struct {
} }
type InitialContent struct { type InitialContent struct {
AclHeadId string AclHeadId string
Identity []byte Identity []byte
SigningKey signingkey.PrivKey SigningKey signingkey.PrivKey
SpaceId string SpaceId string
Seed []byte Seed []byte
ChangeType string ChangeType string
Timestamp int64 ChangePayload []byte
Timestamp int64
} }
type nonVerifiableChangeBuilder struct { type nonVerifiableChangeBuilder struct {
@ -122,41 +123,35 @@ func (c *changeBuilder) SetRootRawChange(rawIdChange *treechangeproto.RawTreeCha
func (c *changeBuilder) BuildRoot(payload InitialContent) (ch *Change, rawIdChange *treechangeproto.RawTreeChangeWithId, err error) { func (c *changeBuilder) BuildRoot(payload InitialContent) (ch *Change, rawIdChange *treechangeproto.RawTreeChangeWithId, err error) {
change := &treechangeproto.RootChange{ change := &treechangeproto.RootChange{
AclHeadId: payload.AclHeadId, AclHeadId: payload.AclHeadId,
Timestamp: payload.Timestamp, Timestamp: payload.Timestamp,
Identity: payload.Identity, Identity: payload.Identity,
ChangeType: payload.ChangeType, ChangeType: payload.ChangeType,
SpaceId: payload.SpaceId, ChangePayload: payload.ChangePayload,
Seed: payload.Seed, SpaceId: payload.SpaceId,
Seed: payload.Seed,
} }
marshalledChange, err := proto.Marshal(change) marshalledChange, err := proto.Marshal(change)
if err != nil { if err != nil {
return return
} }
signature, err := payload.SigningKey.Sign(marshalledChange) signature, err := payload.SigningKey.Sign(marshalledChange)
if err != nil { if err != nil {
return return
} }
raw := &treechangeproto.RawTreeChange{ raw := &treechangeproto.RawTreeChange{
Payload: marshalledChange, Payload: marshalledChange,
Signature: signature, Signature: signature,
} }
marshalledRawChange, err := proto.Marshal(raw) marshalledRawChange, err := proto.Marshal(raw)
if err != nil { if err != nil {
return return
} }
id, err := cidutil.NewCidFromBytes(marshalledRawChange) id, err := cidutil.NewCidFromBytes(marshalledRawChange)
if err != nil { if err != nil {
return return
} }
ch = NewChangeFromRoot(id, change, signature) ch = NewChangeFromRoot(id, change, signature)
rawIdChange = &treechangeproto.RawTreeChangeWithId{ rawIdChange = &treechangeproto.RawTreeChangeWithId{
RawChange: marshalledRawChange, RawChange: marshalledRawChange,
Id: id, Id: id,
@ -170,7 +165,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
AclHeadId: payload.AclHeadId, AclHeadId: payload.AclHeadId,
SnapshotBaseId: payload.SnapshotBaseId, SnapshotBaseId: payload.SnapshotBaseId,
CurrentReadKeyHash: payload.CurrentReadKeyHash, CurrentReadKeyHash: payload.CurrentReadKeyHash,
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
Identity: payload.Identity, Identity: payload.Identity,
IsSnapshot: payload.IsSnapshot, IsSnapshot: payload.IsSnapshot,
} }
@ -184,34 +179,27 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
} else { } else {
change.ChangesData = payload.Content change.ChangesData = payload.Content
} }
marshalledChange, err := proto.Marshal(change) marshalledChange, err := proto.Marshal(change)
if err != nil { if err != nil {
return return
} }
signature, err := payload.SigningKey.Sign(marshalledChange) signature, err := payload.SigningKey.Sign(marshalledChange)
if err != nil { if err != nil {
return return
} }
raw := &treechangeproto.RawTreeChange{ raw := &treechangeproto.RawTreeChange{
Payload: marshalledChange, Payload: marshalledChange,
Signature: signature, Signature: signature,
} }
marshalledRawChange, err := proto.Marshal(raw) marshalledRawChange, err := proto.Marshal(raw)
if err != nil { if err != nil {
return return
} }
id, err := cidutil.NewCidFromBytes(marshalledRawChange) id, err := cidutil.NewCidFromBytes(marshalledRawChange)
if err != nil { if err != nil {
return return
} }
ch = NewChange(id, change, signature) ch = NewChange(id, change, signature)
rawIdChange = &treechangeproto.RawTreeChangeWithId{ rawIdChange = &treechangeproto.RawTreeChangeWithId{
RawChange: marshalledRawChange, RawChange: marshalledRawChange,
Id: id, Id: id,
@ -220,7 +208,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange *
} }
func (c *changeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) { func (c *changeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) {
if ch.Id == c.rootChange.Id { if c.isRoot(ch.Id) {
return c.rootChange, nil return c.rootChange, nil
} }
treeChange := &treechangeproto.TreeChange{ treeChange := &treechangeproto.TreeChange{
@ -255,7 +243,7 @@ func (c *changeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChange
} }
func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, id string) (ch *Change, err error) { func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, id string) (ch *Change, err error) {
if c.rootChange.Id == id { if c.isRoot(id) {
unmarshalled := &treechangeproto.RootChange{} unmarshalled := &treechangeproto.RootChange{}
err = proto.Unmarshal(raw.Payload, unmarshalled) err = proto.Unmarshal(raw.Payload, unmarshalled)
if err != nil { if err != nil {
@ -264,7 +252,6 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
ch = NewChangeFromRoot(id, unmarshalled, raw.Signature) ch = NewChangeFromRoot(id, unmarshalled, raw.Signature)
return return
} }
unmarshalled := &treechangeproto.TreeChange{} unmarshalled := &treechangeproto.TreeChange{}
err = proto.Unmarshal(raw.Payload, unmarshalled) err = proto.Unmarshal(raw.Payload, unmarshalled)
if err != nil { if err != nil {
@ -274,3 +261,10 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange,
ch = NewChange(id, unmarshalled, raw.Signature) ch = NewChange(id, unmarshalled, raw.Signature)
return return
} }
func (c *changeBuilder) isRoot(id string) bool {
if c.rootChange != nil {
return c.rootChange.Id == id
}
return false
}

View File

@ -7,6 +7,7 @@ package mock_objecttree
import ( import (
context "context" context "context"
reflect "reflect" reflect "reflect"
time "time"
list "github.com/anytypeio/any-sync/commonspace/object/acl/list" list "github.com/anytypeio/any-sync/commonspace/object/acl/list"
objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -82,6 +83,20 @@ func (mr *MockObjectTreeMockRecorder) AddRawChanges(arg0, arg1 interface{}) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockObjectTree)(nil).AddRawChanges), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockObjectTree)(nil).AddRawChanges), arg0, arg1)
} }
// ChangeInfo mocks base method.
func (m *MockObjectTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeInfo")
ret0, _ := ret[0].(*treechangeproto.TreeChangeInfo)
return ret0
}
// ChangeInfo indicates an expected call of ChangeInfo.
func (mr *MockObjectTreeMockRecorder) ChangeInfo() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeInfo", reflect.TypeOf((*MockObjectTree)(nil).ChangeInfo))
}
// ChangesAfterCommonSnapshot mocks base method. // ChangesAfterCommonSnapshot mocks base method.
func (m *MockObjectTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) { func (m *MockObjectTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
@ -336,6 +351,21 @@ func (mr *MockObjectTreeMockRecorder) Storage() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Storage", reflect.TypeOf((*MockObjectTree)(nil).Storage)) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Storage", reflect.TypeOf((*MockObjectTree)(nil).Storage))
} }
// TryClose mocks base method.
func (m *MockObjectTree) TryClose(arg0 time.Duration) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TryClose", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// TryClose indicates an expected call of TryClose.
func (mr *MockObjectTreeMockRecorder) TryClose(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryClose", reflect.TypeOf((*MockObjectTree)(nil).TryClose), arg0)
}
// TryLock mocks base method. // TryLock mocks base method.
func (m *MockObjectTree) TryLock() bool { func (m *MockObjectTree) TryLock() bool {
m.ctrl.T.Helper() m.ctrl.T.Helper()

View File

@ -5,6 +5,7 @@ import (
"context" "context"
"errors" "errors"
"sync" "sync"
"time"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/list"
@ -52,6 +53,7 @@ type ReadableObjectTree interface {
Id() string Id() string
Header() *treechangeproto.RawTreeChangeWithId Header() *treechangeproto.RawTreeChangeWithId
UnmarshalledHeader() *Change UnmarshalledHeader() *Change
ChangeInfo() *treechangeproto.TreeChangeInfo
Heads() []string Heads() []string
Root() *Change Root() *Change
@ -81,6 +83,7 @@ type ObjectTree interface {
Delete() error Delete() error
Close() error Close() error
TryClose(objectTTL time.Duration) (bool, error)
} }
type objectTree struct { type objectTree struct {
@ -142,6 +145,10 @@ func (ot *objectTree) UnmarshalledHeader() *Change {
return ot.root return ot.root
} }
func (ot *objectTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
return ot.root.Model.(*treechangeproto.TreeChangeInfo)
}
func (ot *objectTree) Storage() treestorage.TreeStorage { func (ot *objectTree) Storage() treestorage.TreeStorage {
return ot.treeStorage return ot.treeStorage
} }
@ -555,6 +562,10 @@ func (ot *objectTree) Root() *Change {
return ot.tree.Root() return ot.tree.Root()
} }
func (ot *objectTree) TryClose(objectTTL time.Duration) (bool, error) {
return true, ot.Close()
}
func (ot *objectTree) Close() error { func (ot *objectTree) Close() error {
return nil return nil
} }

View File

@ -12,11 +12,12 @@ import (
) )
type ObjectTreeCreatePayload struct { type ObjectTreeCreatePayload struct {
SignKey signingkey.PrivKey SignKey signingkey.PrivKey
ChangeType string ChangeType string
SpaceId string ChangePayload []byte
Identity []byte SpaceId string
IsEncrypted bool Identity []byte
IsEncrypted bool
} }
type HistoryTreeParams struct { type HistoryTreeParams struct {
@ -75,7 +76,7 @@ func CreateObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList)
if err != nil { if err != nil {
return return
} }
return createObjectTreeRoot(payload, time.Now().UnixNano(), bytes, aclList) return createObjectTreeRoot(payload, time.Now().Unix(), bytes, aclList)
} }
func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) (root *treechangeproto.RawTreeChangeWithId, err error) { func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) (root *treechangeproto.RawTreeChangeWithId, err error) {
@ -125,7 +126,7 @@ func CreateObjectTree(
if err != nil { if err != nil {
return return
} }
return createObjectTree(payload, time.Now().UnixNano(), bytes, aclList, createStorage) return createObjectTree(payload, time.Now().Unix(), bytes, aclList, createStorage)
} }
func createObjectTree( func createObjectTree(
@ -165,13 +166,14 @@ func createObjectTreeRoot(
return return
} }
cnt := InitialContent{ cnt := InitialContent{
AclHeadId: aclHeadId, AclHeadId: aclHeadId,
Identity: payload.Identity, Identity: payload.Identity,
SigningKey: payload.SignKey, SigningKey: payload.SignKey,
SpaceId: payload.SpaceId, SpaceId: payload.SpaceId,
ChangeType: payload.ChangeType, ChangeType: payload.ChangeType,
Timestamp: timestamp, ChangePayload: payload.ChangePayload,
Seed: seed, Timestamp: timestamp,
Seed: seed,
} }
_, root, err = NewChangeBuilder(keychain.NewKeychain(), nil).BuildRoot(cnt) _, root, err = NewChangeBuilder(keychain.NewKeychain(), nil).BuildRoot(cnt)

View File

@ -125,7 +125,7 @@ func TestTree_Hash(t *testing.T) {
} }
func TestTree_AddFuzzy(t *testing.T) { func TestTree_AddFuzzy(t *testing.T) {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().Unix())
getChanges := func() []*Change { getChanges := func() []*Change {
changes := []*Change{ changes := []*Change{
newChange("1", "root", "root"), newChange("1", "root", "root"),

View File

@ -4,15 +4,16 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"time"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/util/slice" "github.com/anytypeio/any-sync/util/slice"
"go.uber.org/zap" "go.uber.org/zap"
"time"
) )
var ( var (
log = logger.NewNamed("acltree").Sugar() log = logger.NewNamedSugared("common.commonspace.objecttree")
ErrEmpty = errors.New("logs empty") ErrEmpty = errors.New("logs empty")
) )

View File

@ -7,6 +7,7 @@ package mock_synctree
import ( import (
context "context" context "context"
reflect "reflect" reflect "reflect"
time "time"
list "github.com/anytypeio/any-sync/commonspace/object/acl/list" list "github.com/anytypeio/any-sync/commonspace/object/acl/list"
objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -193,6 +194,20 @@ func (mr *MockSyncTreeMockRecorder) AddRawChanges(arg0, arg1 interface{}) *gomoc
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockSyncTree)(nil).AddRawChanges), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockSyncTree)(nil).AddRawChanges), arg0, arg1)
} }
// ChangeInfo mocks base method.
func (m *MockSyncTree) ChangeInfo() *treechangeproto.TreeChangeInfo {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeInfo")
ret0, _ := ret[0].(*treechangeproto.TreeChangeInfo)
return ret0
}
// ChangeInfo indicates an expected call of ChangeInfo.
func (mr *MockSyncTreeMockRecorder) ChangeInfo() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeInfo", reflect.TypeOf((*MockSyncTree)(nil).ChangeInfo))
}
// ChangesAfterCommonSnapshot mocks base method. // ChangesAfterCommonSnapshot mocks base method.
func (m *MockSyncTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) { func (m *MockSyncTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
@ -487,6 +502,21 @@ func (mr *MockSyncTreeMockRecorder) SyncWithPeer(arg0, arg1 interface{}) *gomock
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncWithPeer", reflect.TypeOf((*MockSyncTree)(nil).SyncWithPeer), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncWithPeer", reflect.TypeOf((*MockSyncTree)(nil).SyncWithPeer), arg0, arg1)
} }
// TryClose mocks base method.
func (m *MockSyncTree) TryClose(arg0 time.Duration) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TryClose", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// TryClose indicates an expected call of TryClose.
func (mr *MockSyncTreeMockRecorder) TryClose(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryClose", reflect.TypeOf((*MockSyncTree)(nil).TryClose), arg0)
}
// TryLock mocks base method. // TryLock mocks base method.
func (m *MockSyncTree) TryLock() bool { func (m *MockSyncTree) TryLock() bool {
m.ctrl.T.Helper() m.ctrl.T.Helper()

View File

@ -3,6 +3,8 @@ package synctree
import ( import (
"context" "context"
"errors" "errors"
"time"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
@ -10,7 +12,7 @@ import (
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/commonspace/syncstatus"
"github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/peer"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
@ -50,7 +52,7 @@ type syncTree struct {
isDeleted bool isDeleted bool
} }
var log = logger.NewNamed("commonspace.synctree") var log = logger.NewNamed("common.commonspace.synctree")
var buildObjectTree = objecttree.BuildObjectTree var buildObjectTree = objecttree.BuildObjectTree
var createSyncClient = newSyncClient var createSyncClient = newSyncClient
@ -208,6 +210,10 @@ func (s *syncTree) Delete() (err error) {
return return
} }
func (s *syncTree) TryClose(objectTTL time.Duration) (bool, error) {
return true, s.Close()
}
func (s *syncTree) Close() (err error) { func (s *syncTree) Close() (err error) {
log.Debug("closing sync tree", zap.String("id", s.Id())) log.Debug("closing sync tree", zap.String("id", s.Id()))
defer func() { defer func() {

View File

@ -16,6 +16,8 @@ message RootChange {
bytes seed = 5; bytes seed = 5;
// Identity is a public key of the tree's creator // Identity is a public key of the tree's creator
bytes identity = 6; bytes identity = 6;
// ChangePayload is a payload related to ChangeType
bytes changePayload = 7;
} }
// TreeChange is a change of a tree // TreeChange is a change of a tree
@ -94,3 +96,9 @@ message TreeFullSyncResponse {
message TreeErrorResponse { message TreeErrorResponse {
string error = 1; string error = 1;
} }
// TreeChangeInfo is used internally in Tree implementation for ease of marshalling
message TreeChangeInfo {
string changeType = 1;
bytes changePayload = 2;
}

View File

@ -36,6 +36,8 @@ type RootChange struct {
Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"` Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"`
// Identity is a public key of the tree's creator // Identity is a public key of the tree's creator
Identity []byte `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"` Identity []byte `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"`
// ChangePayload is a payload related to ChangeType
ChangePayload []byte `protobuf:"bytes,7,opt,name=changePayload,proto3" json:"changePayload,omitempty"`
} }
func (m *RootChange) Reset() { *m = RootChange{} } func (m *RootChange) Reset() { *m = RootChange{} }
@ -113,6 +115,13 @@ func (m *RootChange) GetIdentity() []byte {
return nil return nil
} }
func (m *RootChange) GetChangePayload() []byte {
if m != nil {
return m.ChangePayload
}
return nil
}
// TreeChange is a change of a tree // TreeChange is a change of a tree
type TreeChange struct { type TreeChange struct {
// TreeHeadIds are previous ids for this TreeChange // TreeHeadIds are previous ids for this TreeChange
@ -725,6 +734,59 @@ func (m *TreeErrorResponse) GetError() string {
return "" return ""
} }
// TreeChangeInfo is used internally in Tree implementation for ease of marshalling
type TreeChangeInfo struct {
ChangeType string `protobuf:"bytes,1,opt,name=changeType,proto3" json:"changeType,omitempty"`
ChangePayload []byte `protobuf:"bytes,2,opt,name=changePayload,proto3" json:"changePayload,omitempty"`
}
func (m *TreeChangeInfo) Reset() { *m = TreeChangeInfo{} }
func (m *TreeChangeInfo) String() string { return proto.CompactTextString(m) }
func (*TreeChangeInfo) ProtoMessage() {}
func (*TreeChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_5033f0301ef9b772, []int{10}
}
func (m *TreeChangeInfo) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *TreeChangeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_TreeChangeInfo.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *TreeChangeInfo) XXX_Merge(src proto.Message) {
xxx_messageInfo_TreeChangeInfo.Merge(m, src)
}
func (m *TreeChangeInfo) XXX_Size() int {
return m.Size()
}
func (m *TreeChangeInfo) XXX_DiscardUnknown() {
xxx_messageInfo_TreeChangeInfo.DiscardUnknown(m)
}
var xxx_messageInfo_TreeChangeInfo proto.InternalMessageInfo
func (m *TreeChangeInfo) GetChangeType() string {
if m != nil {
return m.ChangeType
}
return ""
}
func (m *TreeChangeInfo) GetChangePayload() []byte {
if m != nil {
return m.ChangePayload
}
return nil
}
func init() { func init() {
proto.RegisterType((*RootChange)(nil), "treechange.RootChange") proto.RegisterType((*RootChange)(nil), "treechange.RootChange")
proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange") proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange")
@ -736,6 +798,7 @@ func init() {
proto.RegisterType((*TreeFullSyncRequest)(nil), "treechange.TreeFullSyncRequest") proto.RegisterType((*TreeFullSyncRequest)(nil), "treechange.TreeFullSyncRequest")
proto.RegisterType((*TreeFullSyncResponse)(nil), "treechange.TreeFullSyncResponse") proto.RegisterType((*TreeFullSyncResponse)(nil), "treechange.TreeFullSyncResponse")
proto.RegisterType((*TreeErrorResponse)(nil), "treechange.TreeErrorResponse") proto.RegisterType((*TreeErrorResponse)(nil), "treechange.TreeErrorResponse")
proto.RegisterType((*TreeChangeInfo)(nil), "treechange.TreeChangeInfo")
} }
func init() { func init() {
@ -743,48 +806,51 @@ func init() {
} }
var fileDescriptor_5033f0301ef9b772 = []byte{ var fileDescriptor_5033f0301ef9b772 = []byte{
// 656 bytes of a gzipped FileDescriptorProto // 690 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xc1, 0x6e, 0xd3, 0x40, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xc1, 0x4e, 0xdb, 0x4a,
0x10, 0xf5, 0x3a, 0x69, 0xd3, 0x4e, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x07, 0x08, 0x14, 0xf5, 0x38, 0x01, 0xc3, 0x25, 0xc0, 0x7b, 0x03, 0x0b, 0x0b, 0xbd, 0xe7, 0x5a, 0x56, 0xd5,
0x97, 0x56, 0x2a, 0x27, 0x10, 0x52, 0x45, 0x4b, 0x8b, 0xab, 0x0a, 0x84, 0xb6, 0x05, 0x24, 0x6e, 0xa6, 0x1b, 0x90, 0xe8, 0xaa, 0x55, 0x25, 0x54, 0x28, 0xd4, 0x11, 0x6a, 0x85, 0x06, 0x4a, 0xa5,
0x5b, 0x7b, 0x68, 0x8c, 0x12, 0xdb, 0x78, 0x37, 0x54, 0xf9, 0x00, 0x2e, 0x20, 0x21, 0x3e, 0x81, 0xee, 0x06, 0xfb, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0x99, 0x14, 0xe5, 0x03, 0xba, 0x69, 0xa5, 0xaa,
0x6f, 0xe0, 0x0f, 0xb8, 0x71, 0xec, 0x91, 0x23, 0x6a, 0x7e, 0x04, 0xed, 0x3a, 0x4e, 0xd6, 0x6e, 0x9f, 0xd0, 0x5f, 0xe9, 0xae, 0x4b, 0x96, 0x2c, 0x2b, 0xf8, 0x91, 0xca, 0x63, 0x3b, 0xb1, 0x1d,
0x0e, 0xbd, 0xf5, 0xb2, 0xc9, 0xbc, 0x9d, 0x79, 0xfb, 0xe6, 0xcd, 0x6e, 0x02, 0x3b, 0x61, 0xda, 0x2f, 0xd8, 0xb1, 0x71, 0x72, 0x8f, 0xef, 0x3d, 0x73, 0xee, 0xb9, 0x33, 0x63, 0xd8, 0xf1, 0xa2,
0xef, 0xa7, 0x89, 0xc8, 0x78, 0x88, 0x5b, 0xe9, 0xe9, 0x47, 0x0c, 0xe5, 0x96, 0xcc, 0x11, 0xf5, 0xe1, 0x30, 0x0a, 0x45, 0xcc, 0x3d, 0xdc, 0x8a, 0xce, 0x3e, 0xa2, 0x27, 0xb7, 0x64, 0x82, 0xa8,
0x12, 0x76, 0x79, 0x72, 0x86, 0x59, 0x9e, 0xca, 0x74, 0x4b, 0xaf, 0xc2, 0x80, 0x37, 0x35, 0x42, 0x1e, 0x5e, 0x9f, 0x87, 0x17, 0x18, 0x27, 0x91, 0x8c, 0xb6, 0xd4, 0x53, 0x94, 0xe0, 0x4d, 0x85,
0x61, 0x8a, 0xf8, 0xbf, 0x08, 0x00, 0x4b, 0x53, 0xb9, 0xa7, 0x43, 0x7a, 0x07, 0x16, 0x79, 0xd8, 0x50, 0x98, 0x22, 0xce, 0x35, 0x01, 0x60, 0x51, 0x24, 0xf7, 0x54, 0x48, 0xff, 0x83, 0x45, 0xee,
0x0b, 0x90, 0x47, 0x87, 0x91, 0x43, 0x3c, 0xd2, 0x59, 0x64, 0x53, 0x80, 0x3a, 0xd0, 0xd2, 0xa7, 0x0d, 0x5c, 0xe4, 0x7e, 0xcf, 0x37, 0x89, 0x4d, 0xba, 0x8b, 0x6c, 0x0a, 0x50, 0x13, 0x0c, 0xb5,
0x1e, 0x46, 0x8e, 0xad, 0xf7, 0xca, 0x90, 0xba, 0x00, 0x05, 0xe1, 0xc9, 0x30, 0x43, 0xa7, 0xa1, 0x6a, 0xcf, 0x37, 0x75, 0xf5, 0xae, 0x08, 0xa9, 0x05, 0x90, 0x11, 0x9e, 0x8c, 0x63, 0x34, 0x5b,
0x37, 0x0d, 0x44, 0xf1, 0xca, 0xb8, 0x8f, 0x42, 0xf2, 0x7e, 0xe6, 0x34, 0x3d, 0xd2, 0x69, 0xb0, 0xea, 0x65, 0x09, 0x49, 0x79, 0x65, 0x30, 0x44, 0x21, 0xf9, 0x30, 0x36, 0xdb, 0x36, 0xe9, 0xb6,
0x29, 0x40, 0x29, 0x34, 0x05, 0x62, 0xe4, 0xcc, 0x79, 0xa4, 0xd3, 0x66, 0xfa, 0x3b, 0xdd, 0x80, 0xd8, 0x14, 0xa0, 0x14, 0xda, 0x02, 0xd1, 0x37, 0xe7, 0x6c, 0xd2, 0xed, 0x30, 0xf5, 0x9f, 0x6e,
0x85, 0x38, 0xc2, 0x44, 0xc6, 0x72, 0xe8, 0xcc, 0x6b, 0x7c, 0x12, 0xfb, 0x3f, 0x6d, 0x80, 0x93, 0xc0, 0x42, 0xe0, 0x63, 0x28, 0x03, 0x39, 0x36, 0xe7, 0x15, 0x3e, 0x89, 0xe9, 0x43, 0x58, 0xce,
0x1c, 0x71, 0x2c, 0xda, 0x83, 0x25, 0xd5, 0x51, 0x21, 0x52, 0x38, 0xc4, 0x6b, 0x74, 0x16, 0x99, 0xb8, 0x8f, 0xf8, 0x78, 0x10, 0x71, 0xdf, 0x34, 0x54, 0x42, 0x15, 0x74, 0x7e, 0xea, 0x00, 0x27,
0x09, 0x55, 0xdb, 0xb2, 0xeb, 0x6d, 0xdd, 0x87, 0x15, 0x91, 0xf0, 0x4c, 0x74, 0x53, 0xb9, 0xcb, 0x09, 0x62, 0xde, 0x9a, 0x0d, 0x4b, 0x69, 0xdf, 0x59, 0x2b, 0xc2, 0x24, 0x76, 0xab, 0xbb, 0xc8,
0x85, 0xea, 0xae, 0x68, 0xa0, 0x86, 0xaa, 0x73, 0x8a, 0x96, 0xc4, 0x73, 0x2e, 0xb9, 0x6e, 0xa3, 0xca, 0x50, 0xb5, 0x79, 0xbd, 0xde, 0xfc, 0x23, 0x58, 0x11, 0x21, 0x8f, 0x45, 0x3f, 0x92, 0xbb,
0xcd, 0x4c, 0x88, 0x6e, 0x02, 0x0d, 0x07, 0x79, 0x8e, 0x89, 0x64, 0xc8, 0xa3, 0x23, 0x1c, 0x06, 0x5c, 0xa4, 0x1e, 0x64, 0x6d, 0xd6, 0xd0, 0x74, 0x9d, 0x4c, 0x87, 0x78, 0xc5, 0x25, 0x57, 0xcd,
0x5c, 0x74, 0x75, 0x5b, 0x4d, 0x36, 0x63, 0xa7, 0x6a, 0xcb, 0x7c, 0xdd, 0x16, 0xd3, 0x82, 0x56, 0x76, 0x58, 0x19, 0xa2, 0x9b, 0x40, 0xbd, 0x51, 0x92, 0x60, 0x28, 0x19, 0x72, 0xff, 0x10, 0xc7,
0xd5, 0x02, 0x65, 0x78, 0x2c, 0x8e, 0xc7, 0xfa, 0x9c, 0x05, 0x8f, 0x74, 0x16, 0x98, 0x81, 0xf8, 0x2e, 0x17, 0x7d, 0xd5, 0x7c, 0x9b, 0x35, 0xbc, 0xa9, 0x9a, 0x37, 0x5f, 0x37, 0xaf, 0x6c, 0x94,
0x2f, 0x60, 0x99, 0xf1, 0x73, 0xc3, 0x24, 0x07, 0x5a, 0x19, 0x1f, 0xf6, 0x52, 0x5e, 0xcc, 0xb5, 0x51, 0x33, 0xca, 0x02, 0x08, 0xc4, 0x71, 0xae, 0xcf, 0x5c, 0xb0, 0x49, 0x77, 0x81, 0x95, 0x10,
0xcd, 0xca, 0x50, 0x89, 0x10, 0xf1, 0x59, 0xc2, 0xe5, 0x20, 0x47, 0x6d, 0x4e, 0x9b, 0x4d, 0x01, 0xe7, 0x35, 0x2c, 0x33, 0x7e, 0x59, 0x32, 0xc9, 0x04, 0x23, 0xce, 0x3d, 0x25, 0x8a, 0xab, 0x08,
0x7f, 0x0f, 0xd6, 0x2a, 0x44, 0xef, 0x62, 0xd9, 0x3d, 0xd4, 0x45, 0x39, 0x3f, 0x2f, 0xa0, 0x31, 0x53, 0x11, 0x22, 0xb8, 0x08, 0xb9, 0x1c, 0x25, 0xa8, 0xcc, 0xe9, 0xb0, 0x29, 0xe0, 0xec, 0xc1,
0xe1, 0x14, 0xa0, 0x2b, 0x60, 0xc7, 0xa5, 0xd1, 0x76, 0x1c, 0xf9, 0xdf, 0x09, 0xac, 0x2a, 0x8a, 0x5a, 0x85, 0xe8, 0x7d, 0x20, 0xfb, 0x3d, 0x55, 0x94, 0xf0, 0xcb, 0x0c, 0xca, 0x09, 0xa7, 0x00,
0xe3, 0x61, 0x12, 0xbe, 0x44, 0x21, 0xf8, 0x19, 0xd2, 0x27, 0xd0, 0x0a, 0xd3, 0x44, 0x62, 0x22, 0x5d, 0x01, 0x3d, 0x28, 0x8c, 0xd6, 0x03, 0xdf, 0xf9, 0x4e, 0x60, 0x35, 0xa5, 0x38, 0x1e, 0x87,
0x75, 0xfd, 0xd2, 0xb6, 0xb7, 0x69, 0xdc, 0xd4, 0x32, 0x7b, 0xaf, 0x48, 0x79, 0xcb, 0x7b, 0x03, 0xde, 0x1b, 0x14, 0x82, 0x5f, 0x20, 0x7d, 0x0e, 0x86, 0x17, 0x85, 0x12, 0x43, 0xa9, 0xea, 0x97,
0x64, 0x65, 0x01, 0xdd, 0x01, 0xc8, 0x27, 0x97, 0x56, 0x9f, 0xb3, 0xb4, 0x7d, 0xcf, 0x2c, 0x9f, 0xb6, 0xed, 0xcd, 0xd2, 0x7e, 0x2e, 0xb2, 0xf7, 0xb2, 0x94, 0x53, 0x3e, 0x18, 0x21, 0x2b, 0x0a,
0x21, 0x99, 0x19, 0x25, 0xfe, 0x6f, 0x1b, 0xd6, 0x67, 0x1d, 0x41, 0x9f, 0x02, 0x74, 0x91, 0x47, 0xe8, 0x0e, 0x40, 0x32, 0xd9, 0xda, 0x6a, 0x9d, 0xa5, 0xed, 0x07, 0xe5, 0xf2, 0x06, 0xc9, 0xac,
0x6f, 0xb2, 0x88, 0x4b, 0x1c, 0x0b, 0xdb, 0xa8, 0x0b, 0x0b, 0x26, 0x19, 0x81, 0xc5, 0x8c, 0x7c, 0x54, 0xe2, 0xfc, 0xd2, 0x61, 0xbd, 0x69, 0x09, 0xfa, 0x02, 0xa0, 0x8f, 0xdc, 0x7f, 0x17, 0xfb,
0x7a, 0x04, 0xab, 0x1f, 0x06, 0xbd, 0x9e, 0x62, 0x65, 0xf8, 0x69, 0x80, 0x42, 0xce, 0x12, 0xa7, 0x5c, 0x62, 0x2e, 0x6c, 0xa3, 0x2e, 0xcc, 0x9d, 0x64, 0xb8, 0x1a, 0x2b, 0xe5, 0xd3, 0x43, 0x58,
0x28, 0x0e, 0xaa, 0x69, 0x81, 0xc5, 0xea, 0x95, 0xf4, 0x15, 0xdc, 0x9a, 0x42, 0x22, 0x4b, 0x13, 0x3d, 0x1f, 0x0d, 0x06, 0x29, 0x2b, 0xc3, 0x4f, 0x23, 0x14, 0xb2, 0x49, 0x5c, 0x4a, 0x71, 0x50,
0x51, 0xbc, 0xac, 0x19, 0x4e, 0x1d, 0xd4, 0xf2, 0x02, 0x8b, 0x5d, 0xa9, 0xa5, 0xfb, 0xb0, 0x8c, 0x4d, 0x73, 0x35, 0x56, 0xaf, 0xa4, 0x6f, 0xe1, 0x9f, 0x29, 0x24, 0xe2, 0x28, 0x14, 0xd9, 0xf9,
0x79, 0x9e, 0xe6, 0x13, 0xb2, 0xa6, 0x26, 0xbb, 0x5b, 0x27, 0xdb, 0x37, 0x93, 0x02, 0x8b, 0x55, 0x6b, 0x70, 0xea, 0xa0, 0x96, 0xe7, 0x6a, 0x6c, 0xa6, 0x96, 0xee, 0xc3, 0x32, 0x26, 0x49, 0x94,
0xab, 0x76, 0x5b, 0x30, 0xf7, 0x59, 0x59, 0xe5, 0x7f, 0x21, 0xb0, 0x52, 0x75, 0x83, 0xae, 0xc3, 0x4c, 0xc8, 0xda, 0x8a, 0xec, 0xff, 0x3a, 0xd9, 0x7e, 0x39, 0xc9, 0xd5, 0x58, 0xb5, 0x6a, 0xd7,
0x9c, 0x72, 0xa3, 0x7c, 0x83, 0x45, 0x40, 0x1f, 0x43, 0x6b, 0xfc, 0x48, 0x1c, 0xdb, 0x6b, 0x5c, 0x80, 0xb9, 0xcf, 0xa9, 0x55, 0xce, 0x17, 0x02, 0x2b, 0x55, 0x37, 0xe8, 0x3a, 0xcc, 0xa5, 0x6e,
0x67, 0x54, 0x65, 0x3e, 0xf5, 0xa1, 0x5d, 0x3e, 0xc2, 0xd7, 0x5c, 0x76, 0x9d, 0x86, 0xe6, 0xad, 0x14, 0x67, 0x30, 0x0b, 0xe8, 0x33, 0x30, 0xf2, 0x43, 0x62, 0xea, 0x76, 0xeb, 0x2e, 0xa3, 0x2a,
0x60, 0xfe, 0x57, 0x02, 0x6b, 0x33, 0x2c, 0xbd, 0x19, 0x31, 0xdf, 0x48, 0x71, 0xb1, 0xea, 0x13, 0xf2, 0xa9, 0x03, 0x9d, 0xe2, 0x10, 0x1e, 0x71, 0xd9, 0x37, 0x5b, 0x8a, 0xb7, 0x82, 0x39, 0x5f,
0xb9, 0x19, 0x35, 0x0f, 0xe1, 0xf6, 0x95, 0x89, 0x2a, 0x25, 0x7a, 0xa2, 0xe3, 0xdf, 0xf7, 0x22, 0x09, 0xac, 0x35, 0x58, 0x7a, 0x3f, 0x62, 0xbe, 0x91, 0x6c, 0x63, 0xd5, 0x27, 0x72, 0x3f, 0x6a,
0xd8, 0x7d, 0xf6, 0xe7, 0xd2, 0x25, 0x17, 0x97, 0x2e, 0xf9, 0x77, 0xe9, 0x92, 0x1f, 0x23, 0xd7, 0x9e, 0xc0, 0xbf, 0x33, 0x13, 0x4d, 0x95, 0xa8, 0x89, 0xe6, 0x5f, 0x81, 0x2c, 0x70, 0x4e, 0xb3,
0xba, 0x18, 0xb9, 0xd6, 0xdf, 0x91, 0x6b, 0xbd, 0x7f, 0x70, 0xcd, 0xff, 0x9b, 0xd3, 0x79, 0xfd, 0x61, 0x66, 0x6b, 0xf5, 0xc2, 0xf3, 0xa8, 0x76, 0xf3, 0x93, 0x99, 0x9b, 0x7f, 0xe6, 0xae, 0xd6,
0xf1, 0xe8, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4e, 0xa5, 0xe2, 0x7c, 0xa1, 0x06, 0x00, 0x00, 0x1b, 0xee, 0xea, 0xdd, 0x97, 0xbf, 0x6f, 0x2c, 0x72, 0x75, 0x63, 0x91, 0x3f, 0x37, 0x16, 0xf9,
0x71, 0x6b, 0x69, 0x57, 0xb7, 0x96, 0x76, 0x7d, 0x6b, 0x69, 0x1f, 0x1e, 0xdf, 0xf1, 0x6b, 0x77,
0x36, 0xaf, 0x7e, 0x9e, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x24, 0x93, 0x3b, 0x00, 0x1f, 0x07,
0x00, 0x00,
} }
func (m *RootChange) Marshal() (dAtA []byte, err error) { func (m *RootChange) Marshal() (dAtA []byte, err error) {
@ -807,6 +873,13 @@ func (m *RootChange) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if len(m.ChangePayload) > 0 {
i -= len(m.ChangePayload)
copy(dAtA[i:], m.ChangePayload)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangePayload)))
i--
dAtA[i] = 0x3a
}
if len(m.Identity) > 0 { if len(m.Identity) > 0 {
i -= len(m.Identity) i -= len(m.Identity)
copy(dAtA[i:], m.Identity) copy(dAtA[i:], m.Identity)
@ -1362,6 +1435,43 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *TreeChangeInfo) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *TreeChangeInfo) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *TreeChangeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.ChangePayload) > 0 {
i -= len(m.ChangePayload)
copy(dAtA[i:], m.ChangePayload)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangePayload)))
i--
dAtA[i] = 0x12
}
if len(m.ChangeType) > 0 {
i -= len(m.ChangeType)
copy(dAtA[i:], m.ChangeType)
i = encodeVarintTreechange(dAtA, i, uint64(len(m.ChangeType)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintTreechange(dAtA []byte, offset int, v uint64) int { func encodeVarintTreechange(dAtA []byte, offset int, v uint64) int {
offset -= sovTreechange(v) offset -= sovTreechange(v)
base := offset base := offset
@ -1402,6 +1512,10 @@ func (m *RootChange) Size() (n int) {
if l > 0 { if l > 0 {
n += 1 + l + sovTreechange(uint64(l)) n += 1 + l + sovTreechange(uint64(l))
} }
l = len(m.ChangePayload)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
return n return n
} }
@ -1650,6 +1764,23 @@ func (m *TreeErrorResponse) Size() (n int) {
return n return n
} }
func (m *TreeChangeInfo) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.ChangeType)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
l = len(m.ChangePayload)
if l > 0 {
n += 1 + l + sovTreechange(uint64(l))
}
return n
}
func sovTreechange(x uint64) (n int) { func sovTreechange(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7 return (math_bits.Len64(x|1) + 6) / 7
} }
@ -1868,6 +1999,40 @@ func (m *RootChange) Unmarshal(dAtA []byte) error {
m.Identity = []byte{} m.Identity = []byte{}
} }
iNdEx = postIndex iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangePayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangePayload = append(m.ChangePayload[:0], dAtA[iNdEx:postIndex]...)
if m.ChangePayload == nil {
m.ChangePayload = []byte{}
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipTreechange(dAtA[iNdEx:]) skippy, err := skipTreechange(dAtA[iNdEx:])
@ -3233,6 +3398,122 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *TreeChangeInfo) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: TreeChangeInfo: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: TreeChangeInfo: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangeType", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangeType = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ChangePayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTreechange
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthTreechange
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthTreechange
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ChangePayload = append(m.ChangePayload[:0], dAtA[iNdEx:postIndex]...)
if m.ChangePayload == nil {
m.ChangePayload = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTreechange(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTreechange
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipTreechange(dAtA []byte) (n int, err error) { func skipTreechange(dAtA []byte) (n int, err error) {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0

View File

@ -3,7 +3,6 @@ package objectsync
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
"github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
@ -15,9 +14,13 @@ import (
"time" "time"
) )
type LastUsage interface {
LastUsage() time.Time
}
// MessagePool can be made generic to work with different streams // MessagePool can be made generic to work with different streams
type MessagePool interface { type MessagePool interface {
ocache.ObjectLastUsage LastUsage
synchandler.SyncHandler synchandler.SyncHandler
peermanager.PeerManager peermanager.PeerManager
SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error)

View File

@ -2,23 +2,24 @@ package objectsync
import ( import (
"context" "context"
"sync/atomic"
"time"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter" "github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter"
"github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler"
"github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/peermanager"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"sync/atomic"
"time"
) )
var log = logger.NewNamed("commonspace.objectsync") var log = logger.NewNamed("common.commonspace.objectsync")
type ObjectSync interface { type ObjectSync interface {
ocache.ObjectLastUsage LastUsage
synchandler.SyncHandler synchandler.SyncHandler
MessagePool() MessagePool MessagePool() MessagePool
@ -32,6 +33,7 @@ type objectSync struct {
messagePool MessagePool messagePool MessagePool
objectGetter syncobjectgetter.SyncObjectGetter objectGetter syncobjectgetter.SyncObjectGetter
configuration nodeconf.Configuration configuration nodeconf.Configuration
spaceStorage spacestorage.SpaceStorage
syncCtx context.Context syncCtx context.Context
cancelSync context.CancelFunc cancelSync context.CancelFunc
@ -43,13 +45,15 @@ func NewObjectSync(
spaceIsDeleted *atomic.Bool, spaceIsDeleted *atomic.Bool,
configuration nodeconf.Configuration, configuration nodeconf.Configuration,
peerManager peermanager.PeerManager, peerManager peermanager.PeerManager,
objectGetter syncobjectgetter.SyncObjectGetter) ObjectSync { objectGetter syncobjectgetter.SyncObjectGetter,
storage spacestorage.SpaceStorage) ObjectSync {
syncCtx, cancel := context.WithCancel(context.Background()) syncCtx, cancel := context.WithCancel(context.Background())
os := newObjectSync( os := newObjectSync(
spaceId, spaceId,
spaceIsDeleted, spaceIsDeleted,
configuration, configuration,
objectGetter, objectGetter,
storage,
syncCtx, syncCtx,
cancel) cancel)
msgPool := newMessagePool(peerManager, os.handleMessage) msgPool := newMessagePool(peerManager, os.handleMessage)
@ -62,11 +66,13 @@ func newObjectSync(
spaceIsDeleted *atomic.Bool, spaceIsDeleted *atomic.Bool,
configuration nodeconf.Configuration, configuration nodeconf.Configuration,
objectGetter syncobjectgetter.SyncObjectGetter, objectGetter syncobjectgetter.SyncObjectGetter,
spaceStorage spacestorage.SpaceStorage,
syncCtx context.Context, syncCtx context.Context,
cancel context.CancelFunc, cancel context.CancelFunc,
) *objectSync { ) *objectSync {
return &objectSync{ return &objectSync{
objectGetter: objectGetter, objectGetter: objectGetter,
spaceStorage: spaceStorage,
spaceId: spaceId, spaceId: spaceId,
syncCtx: syncCtx, syncCtx: syncCtx,
cancelSync: cancel, cancelSync: cancel,
@ -95,8 +101,8 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp
log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)) log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId))
if s.spaceIsDeleted.Load() { if s.spaceIsDeleted.Load() {
log = log.With(zap.Bool("isDeleted", true)) log = log.With(zap.Bool("isDeleted", true))
// preventing sync with other clients // preventing sync with other clients if they are not just syncing the settings tree
if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) { if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() {
return spacesyncproto.ErrSpaceIsDeleted return spacesyncproto.ErrSpaceIsDeleted
} }
} }

View File

@ -14,8 +14,8 @@ import (
) )
const ( const (
SpaceSettingsChangeType = "reserved.spacesettings" SpaceReserved = "any-sync.space"
SpaceDerivationScheme = "derivation.standard" SpaceDerivationScheme = "derivation.standard"
) )
func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) { func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) {
@ -36,11 +36,12 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
return return
} }
header := &spacesyncproto.SpaceHeader{ header := &spacesyncproto.SpaceHeader{
Identity: identity, Identity: identity,
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
SpaceType: payload.SpaceType, SpaceType: payload.SpaceType,
ReplicationKey: payload.ReplicationKey, SpaceHeaderPayload: payload.SpacePayload,
Seed: spaceHeaderSeed, ReplicationKey: payload.ReplicationKey,
Seed: spaceHeaderSeed,
} }
marshalled, err := header.Marshal() marshalled, err := header.Marshal()
if err != nil { if err != nil {
@ -81,7 +82,7 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
SpaceId: spaceId, SpaceId: spaceId,
EncryptedReadKey: encReadKey, EncryptedReadKey: encReadKey,
CurrentReadKeyHash: readKeyHash, CurrentReadKeyHash: readKeyHash,
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
} }
rawWithId, err := marshalAclRoot(aclRoot, payload.SigningKey) rawWithId, err := marshalAclRoot(aclRoot, payload.SigningKey)
if err != nil { if err != nil {
@ -101,8 +102,8 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
SigningKey: payload.SigningKey, SigningKey: payload.SigningKey,
SpaceId: spaceId, SpaceId: spaceId,
Seed: spaceSettingsSeed, Seed: spaceSettingsSeed,
ChangeType: SpaceSettingsChangeType, ChangeType: SpaceReserved,
Timestamp: time.Now().UnixNano(), Timestamp: time.Now().Unix(),
}) })
if err != nil { if err != nil {
return return
@ -146,9 +147,10 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp
// preparing header and space id // preparing header and space id
header := &spacesyncproto.SpaceHeader{ header := &spacesyncproto.SpaceHeader{
Identity: identity, Identity: identity,
SpaceType: SpaceTypeDerived, SpaceType: payload.SpaceType,
ReplicationKey: repKey, SpaceHeaderPayload: payload.SpacePayload,
ReplicationKey: repKey,
} }
marshalled, err := header.Marshal() marshalled, err := header.Marshal()
if err != nil { if err != nil {
@ -201,7 +203,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp
Identity: aclRoot.Identity, Identity: aclRoot.Identity,
SigningKey: payload.SigningKey, SigningKey: payload.SigningKey,
SpaceId: spaceId, SpaceId: spaceId,
ChangeType: SpaceSettingsChangeType, ChangeType: SpaceReserved,
}) })
if err != nil { if err != nil {
return return

View File

@ -5,15 +5,17 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/object/keychain"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/treegetter" "github.com/anytypeio/any-sync/commonspace/object/treegetter"
"github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate"
spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
@ -242,17 +244,7 @@ func (s *settingsObject) verifyDeleteSpace(raw *treechangeproto.RawTreeChangeWit
if err != nil { if err != nil {
return return
} }
content := &spacesyncproto.SettingsData{} return verifyDeleteContent(data, "")
err = proto.Unmarshal(data, content)
if err != nil {
return
}
if len(content.GetContent()) != 1 ||
content.GetContent()[0].GetSpaceDelete() == nil ||
content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() == "" {
return fmt.Errorf("incorrect delete change payload")
}
return
} }
func (s *settingsObject) addContent(data []byte) (err error) { func (s *settingsObject) addContent(data []byte) (err error) {
@ -271,3 +263,30 @@ func (s *settingsObject) addContent(data []byte) (err error) {
s.Update(s) s.Update(s)
return return
} }
func VerifyDeleteChange(raw *treechangeproto.RawTreeChangeWithId, identity []byte, peerId string) (err error) {
changeBuilder := objecttree.NewChangeBuilder(keychain.NewKeychain(), nil)
res, err := changeBuilder.Unmarshall(raw, true)
if err != nil {
return
}
if res.Identity != string(identity) {
return fmt.Errorf("incorrect identity")
}
return verifyDeleteContent(res.Data, peerId)
}
func verifyDeleteContent(data []byte, peerId string) (err error) {
content := &spacesyncproto.SettingsData{}
err = proto.Unmarshal(data, content)
if err != nil {
return
}
if len(content.GetContent()) != 1 ||
content.GetContent()[0].GetSpaceDelete() == nil ||
(peerId == "" && content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() == "") ||
(peerId != "" && content.GetContent()[0].GetSpaceDelete().GetDeleterPeerId() != peerId) {
return fmt.Errorf("incorrect delete change payload")
}
return
}

View File

@ -48,7 +48,7 @@ func (s *stateBuilder) Build(tr objecttree.ObjectTree, oldState *State) (state *
func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, state *State) *State { func (s *stateBuilder) processChange(change *objecttree.Change, rootId string, state *State) *State {
// ignoring root change which has empty model or startId change // ignoring root change which has empty model or startId change
if change.Model == nil || state.LastIteratedId == change.Id { if len(change.PreviousIds) == 0 || state.LastIteratedId == change.Id {
return state return state
} }

View File

@ -42,6 +42,7 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
t.Run("changeId is equal to rootId", func(t *testing.T) { t.Run("changeId is equal to rootId", func(t *testing.T) {
ch := &objecttree.Change{} ch := &objecttree.Change{}
ch.PreviousIds = []string{"someId"}
ch.Model = &spacesyncproto.SettingsData{ ch.Model = &spacesyncproto.SettingsData{
Snapshot: &spacesyncproto.SpaceSettingsSnapshot{ Snapshot: &spacesyncproto.SpaceSettingsSnapshot{
DeletedIds: []string{"id1", "id2"}, DeletedIds: []string{"id1", "id2"},
@ -56,6 +57,7 @@ func TestStateBuilder_ProcessChange(t *testing.T) {
t.Run("changeId is not equal to lastIteratedId or rootId", func(t *testing.T) { t.Run("changeId is not equal to lastIteratedId or rootId", func(t *testing.T) {
ch := &objecttree.Change{} ch := &objecttree.Change{}
ch.PreviousIds = []string{"someId"}
ch.Model = &spacesyncproto.SettingsData{ ch.Model = &spacesyncproto.SettingsData{
Content: []*spacesyncproto.SpaceSettingsContent{ Content: []*spacesyncproto.SpaceSettingsContent{
{Value: &spacesyncproto.SpaceSettingsContent_ObjectDelete{ {Value: &spacesyncproto.SpaceSettingsContent_ObjectDelete{

View File

@ -6,7 +6,6 @@ import (
"fmt" "fmt"
"github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/app/ocache"
"github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/headsync"
"github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/list"
"github.com/anytypeio/any-sync/commonspace/object/acl/syncacl" "github.com/anytypeio/any-sync/commonspace/object/acl/syncacl"
@ -52,6 +51,8 @@ type SpaceCreatePayload struct {
ReadKey []byte ReadKey []byte
// ReplicationKey is a key which is to be used to determine the node where the space should be held // ReplicationKey is a key which is to be used to determine the node where the space should be held
ReplicationKey uint64 ReplicationKey uint64
// SpacePayload is an arbitrary payload related to space type
SpacePayload []byte
} }
type HandleMessage struct { type HandleMessage struct {
@ -61,11 +62,11 @@ type HandleMessage struct {
Message *spacesyncproto.ObjectSyncMessage Message *spacesyncproto.ObjectSyncMessage
} }
const SpaceTypeDerived = "derived.space"
type SpaceDerivePayload struct { type SpaceDerivePayload struct {
SigningKey signingkey.PrivKey SigningKey signingkey.PrivKey
EncryptionKey encryptionkey.PrivKey EncryptionKey encryptionkey.PrivKey
SpaceType string
SpacePayload []byte
} }
type SpaceDescription struct { type SpaceDescription struct {
@ -81,9 +82,6 @@ func NewSpaceId(id string, repKey uint64) string {
} }
type Space interface { type Space interface {
ocache.ObjectLocker
ocache.ObjectLastUsage
Id() string Id() string
Init(ctx context.Context) error Init(ctx context.Context) error
@ -108,6 +106,7 @@ type Space interface {
HandleMessage(ctx context.Context, msg HandleMessage) (err error) HandleMessage(ctx context.Context, msg HandleMessage) (err error)
TryClose(objectTTL time.Duration) (close bool, err error)
Close() error Close() error
} }
@ -134,16 +133,6 @@ type space struct {
treesUsed *atomic.Int32 treesUsed *atomic.Int32
} }
func (s *space) LastUsage() time.Time {
return s.objectSync.LastUsage()
}
func (s *space) Locked() bool {
locked := s.treesUsed.Load() > 1
log.With(zap.Int32("trees used", s.treesUsed.Load()), zap.Bool("locked", locked), zap.String("spaceId", s.id)).Debug("space lock status check")
return locked
}
func (s *space) Id() string { func (s *space) Id() string {
return s.id return s.id
} }
@ -306,7 +295,13 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea
SyncStatus: s.syncStatus, SyncStatus: s.syncStatus,
PeerGetter: s.peerManager, PeerGetter: s.peerManager,
} }
return synctree.PutSyncTree(ctx, payload, deps) t, err = synctree.PutSyncTree(ctx, payload, deps)
if err != nil {
return
}
s.treesUsed.Add(1)
log.Debug("incrementing counter", zap.String("id", payload.RootRawChange.Id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
return
} }
type BuildTreeOpts struct { type BuildTreeOpts struct {
@ -341,8 +336,8 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t
if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil {
return nil, err return nil, err
} }
log.Debug("incrementing counter", zap.String("id", id), zap.String("spaceId", s.id))
s.treesUsed.Add(1) s.treesUsed.Add(1)
log.Debug("incrementing counter", zap.String("id", id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
return return
} }
@ -417,8 +412,8 @@ func (s *space) handleMessage(msg HandleMessage) {
} }
func (s *space) onObjectClose(id string) { func (s *space) onObjectClose(id string) {
log.Debug("decrementing counter", zap.String("id", id), zap.String("spaceId", s.id))
s.treesUsed.Add(-1) s.treesUsed.Add(-1)
log.Debug("decrementing counter", zap.String("id", id), zap.Int32("trees", s.treesUsed.Load()), zap.String("spaceId", s.id))
_ = s.handleQueue.CloseThread(id) _ = s.handleQueue.CloseThread(id)
} }
@ -462,3 +457,15 @@ func (s *space) Close() error {
log.With(zap.String("id", s.id)).Debug("space closed") log.With(zap.String("id", s.id)).Debug("space closed")
return mError.Err() return mError.Err()
} }
func (s *space) TryClose(objectTTL time.Duration) (close bool, err error) {
if time.Now().Sub(s.objectSync.LastUsage()) < objectTTL {
return false, nil
}
locked := s.treesUsed.Load() > 1
log.With(zap.Int32("trees used", s.treesUsed.Load()), zap.Bool("locked", locked), zap.String("spaceId", s.id)).Debug("space lock status check")
if locked {
return false, nil
}
return true, s.Close()
}

View File

@ -5,6 +5,7 @@ import (
"github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/accountservice"
"github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/credentialprovider"
"github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/headsync"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
@ -16,6 +17,7 @@ import (
"github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/commonspace/syncstatus"
"github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/peer"
"github.com/anytypeio/any-sync/net/pool" "github.com/anytypeio/any-sync/net/pool"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
"sync/atomic" "sync/atomic"
) )
@ -34,6 +36,7 @@ const AddSpaceCtxKey ctxKey = 0
type SpaceService interface { type SpaceService interface {
DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (string, error) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (string, error)
DeriveId(ctx context.Context, payload SpaceDerivePayload) (string, error)
CreateSpace(ctx context.Context, payload SpaceCreatePayload) (string, error) CreateSpace(ctx context.Context, payload SpaceCreatePayload) (string, error)
NewSpace(ctx context.Context, id string) (sp Space, err error) NewSpace(ctx context.Context, id string) (sp Space, err error)
app.Component app.Component
@ -45,6 +48,7 @@ type spaceService struct {
configurationService nodeconf.Service configurationService nodeconf.Service
storageProvider spacestorage.SpaceStorageProvider storageProvider spacestorage.SpaceStorageProvider
peermanagerProvider peermanager.PeerManagerProvider peermanagerProvider peermanager.PeerManagerProvider
credentialProvider credentialprovider.CredentialProvider
treeGetter treegetter.TreeGetter treeGetter treegetter.TreeGetter
pool pool.Pool pool pool.Pool
} }
@ -56,6 +60,12 @@ func (s *spaceService) Init(a *app.App) (err error) {
s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service) s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service)
s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter) s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter)
s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider) s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider)
credProvider := a.Component(credentialprovider.CName)
if credProvider != nil {
s.credentialProvider = credProvider.(credentialprovider.CredentialProvider)
} else {
s.credentialProvider = credentialprovider.NewNoOp()
}
s.pool = a.MustComponent(pool.CName).(pool.Pool) s.pool = a.MustComponent(pool.CName).(pool.Pool)
return nil return nil
} }
@ -80,6 +90,15 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload SpaceCreatePaylo
return store.Id(), nil return store.Id(), nil
} }
func (s *spaceService) DeriveId(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload)
if err != nil {
return
}
id = storageCreate.SpaceHeaderWithId.Id
return
}
func (s *spaceService) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (id string, err error) { func (s *spaceService) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload) storageCreate, err := storagePayloadForSpaceDerive(payload)
if err != nil { if err != nil {
@ -139,8 +158,8 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) {
return nil, err return nil, err
} }
headSync := headsync.NewHeadSync(id, spaceIsDeleted, s.config.SyncPeriod, lastConfiguration, st, peerManager, getter, syncStatus, log) headSync := headsync.NewHeadSync(id, spaceIsDeleted, s.config.SyncPeriod, lastConfiguration, st, peerManager, getter, syncStatus, s.credentialProvider, log)
objectSync := objectsync.NewObjectSync(id, spaceIsDeleted, lastConfiguration, peerManager, getter) objectSync := objectsync.NewObjectSync(id, spaceIsDeleted, lastConfiguration, peerManager, getter, st)
sp := &space{ sp := &space{
id: id, id: id,
objectSync: objectSync, objectSync: objectSync,
@ -198,6 +217,7 @@ func (s *spaceService) getSpaceStorageFromRemote(ctx context.Context, id string)
cl := spacesyncproto.NewDRPCSpaceSyncClient(p) cl := spacesyncproto.NewDRPCSpaceSyncClient(p)
res, err := cl.SpacePull(ctx, &spacesyncproto.SpacePullRequest{Id: id}) res, err := cl.SpacePull(ctx, &spacesyncproto.SpacePullRequest{Id: id})
if err != nil { if err != nil {
err = rpcerr.Unwrap(err)
return return
} }

View File

@ -2,21 +2,28 @@
package spacestorage package spacestorage
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anytypeio/any-sync/commonspace/object/acl/liststorage" "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/spacesyncproto"
"github.com/anytypeio/any-sync/util/cidutil"
"github.com/anytypeio/any-sync/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
"strings"
) )
const CName = "common.commonspace.spacestorage" const CName = "common.commonspace.spacestorage"
var ( var (
ErrSpaceStorageExists = errors.New("space storage exists") ErrSpaceStorageExists = errors.New("space storage exists")
ErrSpaceStorageMissing = errors.New("space storage missing") ErrSpaceStorageMissing = errors.New("space storage missing")
ErrIncorrectSpaceHeader = errors.New("incorrect space header")
ErrTreeStorageAlreadyDeleted = errors.New("tree storage already deleted") ErrTreeStorageAlreadyDeleted = errors.New("tree storage already deleted")
) )
@ -63,3 +70,38 @@ func ValidateSpaceStorageCreatePayload(payload SpaceStorageCreatePayload) (err e
// TODO: add proper validation // TODO: add proper validation
return nil return nil
} }
func ValidateSpaceHeader(spaceId string, header, identity []byte) (err error) {
split := strings.Split(spaceId, ".")
if len(split) != 2 {
return ErrIncorrectSpaceHeader
}
if !cidutil.VerifyCid(header, split[0]) {
err = objecttree.ErrIncorrectCid
return
}
raw := &spacesyncproto.RawSpaceHeader{}
err = proto.Unmarshal(header, raw)
if err != nil {
return
}
payload := &spacesyncproto.SpaceHeader{}
err = proto.Unmarshal(raw.SpaceHeader, payload)
if err != nil {
return
}
if identity != nil && !bytes.Equal(identity, payload.Identity) {
err = ErrIncorrectSpaceHeader
return
}
key, err := signingkey.NewSigningEd25519PubKeyFromBytes(payload.Identity)
if err != nil {
return
}
res, err := key.Verify(raw.SpaceHeader, raw.Signature)
if err != nil || !res {
err = ErrIncorrectSpaceHeader
return
}
return
}

View File

@ -66,6 +66,7 @@ message ObjectSyncMessage {
// SpacePushRequest is a request to add space on a node containing only one acl record // SpacePushRequest is a request to add space on a node containing only one acl record
message SpacePushRequest { message SpacePushRequest {
SpacePayload payload = 1; SpacePayload payload = 1;
bytes Credential = 2;
} }
// SpacePushResponse is an empty response // SpacePushResponse is an empty response
@ -97,6 +98,7 @@ message SpaceHeader {
string spaceType = 3; string spaceType = 3;
uint64 replicationKey = 4; uint64 replicationKey = 4;
bytes seed = 5; bytes seed = 5;
bytes spaceHeaderPayload = 6;
} }
// RawSpaceHeader is raw header for SpaceHeader // RawSpaceHeader is raw header for SpaceHeader

View File

@ -437,7 +437,8 @@ func (m *ObjectSyncMessage) GetObjectId() string {
// SpacePushRequest is a request to add space on a node containing only one acl record // SpacePushRequest is a request to add space on a node containing only one acl record
type SpacePushRequest struct { type SpacePushRequest struct {
Payload *SpacePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` Payload *SpacePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
Credential []byte `protobuf:"bytes,2,opt,name=Credential,proto3" json:"Credential,omitempty"`
} }
func (m *SpacePushRequest) Reset() { *m = SpacePushRequest{} } func (m *SpacePushRequest) Reset() { *m = SpacePushRequest{} }
@ -480,6 +481,13 @@ func (m *SpacePushRequest) GetPayload() *SpacePayload {
return nil return nil
} }
func (m *SpacePushRequest) GetCredential() []byte {
if m != nil {
return m.Credential
}
return nil
}
// SpacePushResponse is an empty response // SpacePushResponse is an empty response
type SpacePushResponse struct { type SpacePushResponse struct {
} }
@ -686,11 +694,12 @@ func (m *SpacePayload) GetSpaceSettingsPayloadId() string {
// SpaceHeader is a header for a space // SpaceHeader is a header for a space
type SpaceHeader struct { type SpaceHeader struct {
Identity []byte `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"` Identity []byte `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"`
Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
SpaceType string `protobuf:"bytes,3,opt,name=spaceType,proto3" json:"spaceType,omitempty"` SpaceType string `protobuf:"bytes,3,opt,name=spaceType,proto3" json:"spaceType,omitempty"`
ReplicationKey uint64 `protobuf:"varint,4,opt,name=replicationKey,proto3" json:"replicationKey,omitempty"` ReplicationKey uint64 `protobuf:"varint,4,opt,name=replicationKey,proto3" json:"replicationKey,omitempty"`
Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"` Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"`
SpaceHeaderPayload []byte `protobuf:"bytes,6,opt,name=spaceHeaderPayload,proto3" json:"spaceHeaderPayload,omitempty"`
} }
func (m *SpaceHeader) Reset() { *m = SpaceHeader{} } func (m *SpaceHeader) Reset() { *m = SpaceHeader{} }
@ -761,6 +770,13 @@ func (m *SpaceHeader) GetSeed() []byte {
return nil return nil
} }
func (m *SpaceHeader) GetSpaceHeaderPayload() []byte {
if m != nil {
return m.SpaceHeaderPayload
}
return nil
}
// RawSpaceHeader is raw header for SpaceHeader // RawSpaceHeader is raw header for SpaceHeader
type RawSpaceHeader struct { type RawSpaceHeader struct {
SpaceHeader []byte `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"` SpaceHeader []byte `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"`
@ -1232,71 +1248,73 @@ func init() {
} }
var fileDescriptor_80e49f1f4ac27799 = []byte{ var fileDescriptor_80e49f1f4ac27799 = []byte{
// 1019 bytes of a gzipped FileDescriptorProto // 1042 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0x4b, 0x6f, 0xdb, 0x46, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0x4f, 0x6f, 0x1b, 0x45,
0x10, 0x16, 0xe9, 0xa7, 0xc6, 0xb2, 0xcc, 0x6c, 0x9c, 0x44, 0x55, 0x0c, 0x45, 0x58, 0x14, 0x85, 0x14, 0xf7, 0x6e, 0xd2, 0x24, 0x7e, 0x71, 0x9c, 0xed, 0x34, 0x6d, 0x8d, 0x1b, 0xb9, 0xd6, 0x08,
0x91, 0x43, 0x1e, 0x72, 0x51, 0x20, 0x69, 0x7b, 0x48, 0x6c, 0xa5, 0x11, 0x8a, 0xd4, 0xc6, 0xaa, 0xa1, 0xa8, 0x87, 0xb4, 0x75, 0x10, 0x52, 0x0b, 0x1c, 0x5a, 0x27, 0xa5, 0x16, 0x2a, 0x89, 0xc6,
0x41, 0x81, 0x02, 0x39, 0xac, 0xc9, 0xb1, 0xc4, 0x96, 0x22, 0x59, 0xee, 0xaa, 0xb6, 0x8e, 0x3d, 0x54, 0x48, 0x48, 0x3d, 0x4c, 0x76, 0x5f, 0xec, 0x85, 0xf5, 0xee, 0xb2, 0x33, 0x26, 0xf1, 0x91,
0xf5, 0xda, 0x73, 0x7b, 0xea, 0x7f, 0xe8, 0x8f, 0xe8, 0x31, 0xc7, 0x1e, 0x0b, 0xfb, 0x8f, 0x14, 0x13, 0x57, 0xce, 0xf0, 0x35, 0xf8, 0x10, 0x1c, 0xcb, 0x8d, 0x23, 0x4a, 0xbe, 0x08, 0x9a, 0xd9,
0xbb, 0x5c, 0x3e, 0x24, 0x51, 0x39, 0xf4, 0x22, 0x73, 0xbf, 0x99, 0xf9, 0xe6, 0xb5, 0x3b, 0x63, 0xd9, 0x3f, 0xb6, 0xd7, 0x95, 0xb8, 0x38, 0x3b, 0xbf, 0xf7, 0xde, 0xef, 0xfd, 0x9b, 0x79, 0x2f,
0x78, 0xea, 0x46, 0x93, 0x49, 0x14, 0x8a, 0x98, 0xbb, 0xf8, 0x58, 0xff, 0x8a, 0x59, 0xe8, 0xc6, 0xf0, 0xd4, 0x8d, 0x26, 0x93, 0x28, 0x14, 0x31, 0x77, 0xf1, 0xb1, 0xfe, 0x15, 0xb3, 0xd0, 0x8d,
0x49, 0x24, 0xa3, 0xc7, 0xfa, 0x57, 0x14, 0xe8, 0x23, 0x0d, 0x90, 0x7a, 0x0e, 0xd0, 0x01, 0xec, 0x93, 0x48, 0x46, 0x8f, 0xf5, 0xaf, 0x28, 0xd0, 0x43, 0x0d, 0x90, 0x7a, 0x0e, 0xd0, 0x01, 0xec,
0xbe, 0x46, 0xee, 0x0d, 0x67, 0xa1, 0xcb, 0x78, 0x38, 0x42, 0x42, 0x60, 0xfd, 0x22, 0x89, 0x26, 0xbc, 0x46, 0xee, 0x0d, 0x67, 0xa1, 0xcb, 0x78, 0x38, 0x42, 0x42, 0x60, 0xfd, 0x22, 0x89, 0x26,
0x2d, 0xab, 0x6b, 0x1d, 0xae, 0x33, 0xfd, 0x4d, 0x9a, 0x60, 0xcb, 0xa8, 0x65, 0x6b, 0xc4, 0x96, 0x2d, 0xab, 0x6b, 0x1d, 0xac, 0x33, 0xfd, 0x4d, 0x9a, 0x60, 0xcb, 0xa8, 0x65, 0x6b, 0xc4, 0x96,
0x11, 0xd9, 0x87, 0x8d, 0xc0, 0x9f, 0xf8, 0xb2, 0xb5, 0xd6, 0xb5, 0x0e, 0x77, 0x59, 0x7a, 0xa0, 0x11, 0xd9, 0x83, 0x5b, 0x81, 0x3f, 0xf1, 0x65, 0x6b, 0xad, 0x6b, 0x1d, 0xec, 0xb0, 0xf4, 0x40,
0x57, 0xd0, 0xcc, 0xa9, 0x50, 0x4c, 0x03, 0xa9, 0xb8, 0xc6, 0x5c, 0x8c, 0x35, 0x57, 0x83, 0xe9, 0xaf, 0xa0, 0x99, 0x53, 0xa1, 0x98, 0x06, 0x52, 0x71, 0x8d, 0xb9, 0x18, 0x6b, 0xae, 0x06, 0xd3,
0x6f, 0xf2, 0x05, 0x6c, 0x63, 0x80, 0x13, 0x0c, 0xa5, 0x68, 0xd9, 0xdd, 0xb5, 0xc3, 0x9d, 0x5e, 0xdf, 0xe4, 0x0b, 0xd8, 0xc2, 0x00, 0x27, 0x18, 0x4a, 0xd1, 0xb2, 0xbb, 0x6b, 0x07, 0xdb, 0xbd,
0xf7, 0x51, 0x11, 0xdf, 0x3c, 0x41, 0x3f, 0x55, 0x64, 0xb9, 0x85, 0xf2, 0xec, 0x46, 0xd3, 0x30, 0xee, 0x61, 0x11, 0xdf, 0x3c, 0xc1, 0x49, 0xaa, 0xc8, 0x72, 0x0b, 0xe5, 0xd9, 0x8d, 0xa6, 0x61,
0xf7, 0xac, 0x0f, 0xf4, 0x73, 0xb8, 0x53, 0x69, 0xa8, 0x02, 0xf7, 0x3d, 0xed, 0xbe, 0xce, 0x6c, 0xee, 0x59, 0x1f, 0xe8, 0xe7, 0x70, 0xb7, 0xd2, 0x50, 0x05, 0xee, 0x7b, 0xda, 0x7d, 0x9d, 0xd9,
0xdf, 0xd3, 0x01, 0x21, 0xf7, 0x74, 0x2a, 0x75, 0xa6, 0xbf, 0xe9, 0x3b, 0xd8, 0x2b, 0x8c, 0x7f, 0xbe, 0xa7, 0x03, 0x42, 0xee, 0xe9, 0x54, 0xea, 0x4c, 0x7f, 0xd3, 0x77, 0xb0, 0x5b, 0x18, 0xff,
0x9a, 0xa2, 0x90, 0xa4, 0x05, 0x5b, 0x3a, 0xa4, 0x41, 0x66, 0x9b, 0x1d, 0xc9, 0x13, 0xd8, 0x4c, 0x34, 0x45, 0x21, 0x49, 0x0b, 0x36, 0x75, 0x48, 0x83, 0xcc, 0x36, 0x3b, 0x92, 0x27, 0xb0, 0x91,
0x54, 0x99, 0xb2, 0xd8, 0x5b, 0x55, 0xb1, 0x2b, 0x05, 0x66, 0xf4, 0xe8, 0x57, 0xe0, 0x94, 0x62, 0xa8, 0x32, 0x65, 0xb1, 0xb7, 0xaa, 0x62, 0x57, 0x0a, 0xcc, 0xe8, 0xd1, 0xaf, 0xc0, 0x29, 0xc5,
0x8b, 0xa3, 0x50, 0x20, 0x39, 0x82, 0xad, 0x44, 0xc7, 0x29, 0x5a, 0x96, 0xa6, 0xf9, 0x68, 0x65, 0x16, 0x47, 0xa1, 0x40, 0x72, 0x04, 0x9b, 0x89, 0x8e, 0x53, 0xb4, 0x2c, 0x4d, 0xf3, 0xd1, 0xca,
0x09, 0x58, 0xa6, 0x49, 0xff, 0xb0, 0xe0, 0xd6, 0xe9, 0xf9, 0x0f, 0xe8, 0x4a, 0x25, 0x7d, 0x83, 0x12, 0xb0, 0x4c, 0x93, 0xfe, 0x61, 0xc1, 0xed, 0xd3, 0xf3, 0x1f, 0xd0, 0x95, 0x4a, 0xfa, 0x06,
0x42, 0xf0, 0x11, 0x7e, 0x20, 0xd4, 0x03, 0xa8, 0x27, 0x69, 0x3e, 0x83, 0x2c, 0xe1, 0x02, 0x50, 0x85, 0xe0, 0x23, 0xfc, 0x40, 0xa8, 0xfb, 0x50, 0x4f, 0xd2, 0x7c, 0x06, 0x59, 0xc2, 0x05, 0xa0,
0x76, 0x09, 0xc6, 0xc1, 0x6c, 0xe0, 0xe9, 0x52, 0xd6, 0x59, 0x76, 0x54, 0x92, 0x98, 0xcf, 0x82, 0xec, 0x12, 0x8c, 0x83, 0xd9, 0xc0, 0xd3, 0xa5, 0xac, 0xb3, 0xec, 0xa8, 0x24, 0x31, 0x9f, 0x05,
0x88, 0x7b, 0xad, 0x75, 0xdd, 0xb7, 0xec, 0x48, 0xda, 0xb0, 0x1d, 0xe9, 0x00, 0x06, 0x5e, 0x6b, 0x11, 0xf7, 0x5a, 0xeb, 0xba, 0x6f, 0xd9, 0x91, 0xb4, 0x61, 0x2b, 0xd2, 0x01, 0x0c, 0xbc, 0xd6,
0x43, 0x1b, 0xe5, 0x67, 0xda, 0x07, 0x67, 0xa8, 0x1c, 0x9f, 0x4d, 0xc5, 0x38, 0x2b, 0xe3, 0xd3, 0x2d, 0x6d, 0x94, 0x9f, 0x29, 0x82, 0x33, 0x54, 0x8e, 0xcf, 0xa6, 0x62, 0x9c, 0x95, 0xf1, 0x69,
0x82, 0x49, 0xc5, 0xb6, 0xd3, 0xbb, 0x57, 0x4a, 0x33, 0xd5, 0x4e, 0xc5, 0xb9, 0x0b, 0x7a, 0x1b, 0xc1, 0xa4, 0x62, 0xdb, 0xee, 0xdd, 0x2f, 0xa5, 0x99, 0x6a, 0xa7, 0xe2, 0xc2, 0x45, 0x07, 0xa0,
0x6e, 0x95, 0x68, 0xd2, 0x72, 0x51, 0x9a, 0x73, 0x07, 0x41, 0xc6, 0xbd, 0xd0, 0x59, 0xfa, 0x2a, 0x9f, 0xa0, 0x87, 0xa1, 0xf4, 0x79, 0xa0, 0xa3, 0x6e, 0xb0, 0x12, 0x42, 0xef, 0xc0, 0xed, 0x92,
0x37, 0x54, 0x3a, 0xa6, 0xce, 0xff, 0x23, 0x80, 0x5f, 0x6c, 0x68, 0x94, 0x25, 0xe4, 0x05, 0xec, 0x9b, 0xb4, 0x9c, 0x94, 0xe6, 0xbe, 0x83, 0x20, 0xf3, 0xbd, 0xd0, 0x79, 0xfa, 0x2a, 0x37, 0x54,
0x68, 0x1b, 0xd5, 0x16, 0x4c, 0x0c, 0xcf, 0x83, 0x12, 0x0f, 0xe3, 0x97, 0xc3, 0x42, 0xe1, 0x3b, 0x3a, 0xa6, 0x0f, 0xff, 0x3f, 0x40, 0xfa, 0x8b, 0x0d, 0x8d, 0xb2, 0x84, 0xbc, 0x80, 0x6d, 0x6d,
0x5f, 0x8e, 0x07, 0x1e, 0x2b, 0xdb, 0x90, 0x0e, 0x00, 0x77, 0x03, 0x43, 0xa8, 0x5b, 0xd1, 0x60, 0xa3, 0xda, 0x86, 0x89, 0xe1, 0x79, 0x58, 0xe2, 0x61, 0xfc, 0x72, 0x58, 0x28, 0x7c, 0xe7, 0xcb,
0x25, 0x84, 0x50, 0x68, 0x14, 0xa7, 0xbc, 0x21, 0x73, 0x18, 0xe9, 0xc1, 0xbe, 0xa6, 0x1c, 0xa2, 0xf1, 0xc0, 0x63, 0x65, 0x1b, 0x95, 0x34, 0x77, 0x03, 0x43, 0x98, 0x25, 0x5d, 0x20, 0x84, 0x42,
0x94, 0x7e, 0x38, 0x12, 0x67, 0x73, 0x2d, 0xaa, 0x94, 0x91, 0xcf, 0xe0, 0x6e, 0x15, 0x9e, 0x77, 0xa3, 0x38, 0xe5, 0x0d, 0x9b, 0xc3, 0x48, 0x0f, 0xf6, 0x34, 0xe5, 0x10, 0xa5, 0xf4, 0xc3, 0x91,
0x6f, 0x85, 0x94, 0xfe, 0x69, 0xc1, 0x4e, 0x29, 0x25, 0xd5, 0x77, 0xdf, 0xc3, 0x50, 0xfa, 0x72, 0x38, 0x9b, 0x6b, 0x61, 0xa5, 0x8c, 0x7c, 0x06, 0xf7, 0xaa, 0xf0, 0xbc, 0xbb, 0x2b, 0xa4, 0xf4,
0x66, 0x9e, 0x72, 0x7e, 0x56, 0xb7, 0x4c, 0xfa, 0x13, 0x14, 0x92, 0x4f, 0x62, 0x9d, 0xda, 0x1a, 0x6f, 0x0b, 0xb6, 0x4b, 0x29, 0xa9, 0x7b, 0xe1, 0xeb, 0x06, 0xc9, 0x99, 0x79, 0xea, 0xf9, 0x59,
0x2b, 0x00, 0x25, 0xd5, 0x3e, 0xbe, 0x9d, 0xc5, 0x68, 0xd2, 0x2a, 0x00, 0xf2, 0x09, 0x34, 0xd5, 0xdd, 0x42, 0xe9, 0x4f, 0x50, 0x48, 0x3e, 0x89, 0x75, 0x6a, 0x6b, 0xac, 0x00, 0x94, 0x54, 0xfb,
0xa5, 0xf3, 0x5d, 0x2e, 0xfd, 0x28, 0xfc, 0x1a, 0x67, 0x3a, 0x9b, 0x75, 0xb6, 0x80, 0xaa, 0x57, 0xf8, 0x76, 0x16, 0xa3, 0x49, 0xab, 0x00, 0xc8, 0x27, 0xd0, 0x54, 0x97, 0xd2, 0x77, 0xb9, 0xf4,
0x2b, 0x10, 0xd3, 0xa8, 0x1b, 0x4c, 0x7f, 0xd3, 0x33, 0x68, 0xce, 0x17, 0x9e, 0x74, 0x97, 0x1b, 0xa3, 0xf0, 0x6b, 0x9c, 0xe9, 0x6c, 0xd6, 0xd9, 0x02, 0xaa, 0x5e, 0xb5, 0x40, 0x4c, 0xa3, 0x6e,
0xd5, 0x98, 0xef, 0x83, 0x8a, 0xc6, 0x1f, 0x85, 0x5c, 0x4e, 0x13, 0x34, 0x6d, 0x28, 0x00, 0x7a, 0x30, 0xfd, 0x4d, 0x0e, 0x81, 0x94, 0x4a, 0x9c, 0x55, 0x63, 0x43, 0x6b, 0x54, 0x48, 0xe8, 0x19,
0x02, 0xfb, 0x55, 0xad, 0xd4, 0xef, 0x88, 0x5f, 0xce, 0xb1, 0x16, 0x80, 0xb9, 0x87, 0x76, 0x7e, 0x34, 0xe7, 0x1b, 0x45, 0xba, 0xcb, 0x8d, 0x6d, 0xcc, 0xf7, 0x4d, 0x45, 0xef, 0x8f, 0x42, 0x2e,
0x0f, 0x7f, 0xb7, 0x60, 0x7f, 0x58, 0x2e, 0xeb, 0x71, 0x14, 0x4a, 0x35, 0x8a, 0xbe, 0x84, 0x46, 0xa7, 0x09, 0x9a, 0xb6, 0x15, 0x00, 0x3d, 0x86, 0xbd, 0xaa, 0xd6, 0xeb, 0x77, 0xc9, 0x2f, 0xe7,
0xfa, 0x58, 0x4e, 0x30, 0x40, 0x89, 0x15, 0x17, 0xf2, 0xb4, 0x24, 0x7e, 0x5d, 0x63, 0x73, 0xea, 0x58, 0x0b, 0xc0, 0xdc, 0x5b, 0x3b, 0xbf, 0xb7, 0xbf, 0x5b, 0xb0, 0x37, 0x2c, 0xb7, 0xa1, 0x1f,
0xe4, 0xb9, 0xc9, 0xce, 0x58, 0xdb, 0xda, 0xfa, 0xee, 0xe2, 0x75, 0xce, 0x8d, 0xcb, 0xca, 0x2f, 0x85, 0x52, 0x8d, 0xb6, 0x2f, 0xa1, 0x91, 0x3e, 0xbe, 0x63, 0x0c, 0x50, 0x62, 0xc5, 0x05, 0x3e,
0xb7, 0x60, 0xe3, 0x67, 0x1e, 0x4c, 0x91, 0x76, 0xa0, 0x51, 0x76, 0xb2, 0xf4, 0x88, 0x8e, 0x4c, 0x2d, 0x89, 0x5f, 0xd7, 0xd8, 0x9c, 0x3a, 0x79, 0x6e, 0xb2, 0x33, 0xd6, 0xb6, 0xb6, 0xbe, 0xb7,
0xdf, 0x8d, 0xf8, 0x63, 0xd8, 0xf5, 0xf4, 0x57, 0x72, 0x86, 0x98, 0xe4, 0x13, 0x66, 0x1e, 0xa4, 0x78, 0xfd, 0x73, 0xe3, 0xb2, 0xf2, 0xcb, 0x4d, 0xb8, 0xf5, 0x33, 0x0f, 0xa6, 0x48, 0x3b, 0xd0,
0xef, 0xe0, 0xce, 0x5c, 0xc2, 0xc3, 0x90, 0xc7, 0x62, 0x1c, 0x49, 0x75, 0xed, 0x53, 0x4d, 0x6f, 0x28, 0x3b, 0x59, 0x7a, 0x74, 0x47, 0xe6, 0x9e, 0x18, 0xf1, 0xc7, 0xb0, 0xe3, 0xe9, 0xaf, 0xe4,
0xe0, 0xa5, 0x83, 0xae, 0xce, 0x4a, 0xc8, 0x32, 0xbd, 0x5d, 0x45, 0xff, 0xab, 0x05, 0x8d, 0x8c, 0x0c, 0x31, 0xc9, 0x27, 0xd6, 0x3c, 0x48, 0xdf, 0xc1, 0xdd, 0xb9, 0x84, 0x87, 0x21, 0x8f, 0xc5,
0xfa, 0x84, 0x4b, 0x4e, 0x9e, 0xc1, 0x96, 0x9b, 0xd6, 0xd4, 0x0c, 0xcf, 0x07, 0x8b, 0x55, 0x58, 0x38, 0x92, 0xea, 0x99, 0xa4, 0x9a, 0xde, 0xc0, 0x4b, 0x07, 0x67, 0x9d, 0x95, 0x90, 0x65, 0x7a,
0x28, 0x3d, 0xcb, 0xf4, 0xd5, 0xee, 0x11, 0x26, 0x3a, 0x53, 0xc1, 0xee, 0x2a, 0xdb, 0x2c, 0x0b, 0xbb, 0x8a, 0xfe, 0x57, 0x0b, 0x1a, 0x19, 0xf5, 0x31, 0x97, 0x9c, 0x3c, 0x83, 0x4d, 0x37, 0xad,
0x96, 0x5b, 0xd0, 0x1f, 0xcd, 0x88, 0x19, 0x4e, 0xcf, 0x85, 0x9b, 0xf8, 0xb1, 0xba, 0x9e, 0xea, 0xa9, 0x19, 0xc6, 0x0f, 0x17, 0xab, 0xb0, 0x50, 0x7a, 0x96, 0xe9, 0xab, 0x5d, 0x26, 0x4c, 0x74,
0x6d, 0x98, 0x81, 0x9b, 0xa5, 0x98, 0x9f, 0xc9, 0x73, 0xd8, 0xe4, 0xae, 0xd2, 0xd2, 0xce, 0x9a, 0xa6, 0x82, 0xdd, 0x55, 0xb6, 0x59, 0x16, 0x2c, 0xb7, 0xa0, 0x3f, 0x9a, 0x91, 0x34, 0x9c, 0x9e,
0x3d, 0xba, 0xe4, 0xac, 0xc4, 0xf4, 0x42, 0x6b, 0x32, 0x63, 0xf1, 0xf0, 0x12, 0xb6, 0xfb, 0x49, 0x0b, 0x37, 0xf1, 0x63, 0x75, 0x9d, 0xd5, 0x5b, 0x32, 0x03, 0x3c, 0x4b, 0x31, 0x3f, 0x93, 0xe7,
0x72, 0x1c, 0x79, 0x28, 0x48, 0x13, 0xe0, 0x6d, 0x88, 0x57, 0x31, 0xba, 0x12, 0x3d, 0xa7, 0x46, 0xb0, 0xc1, 0x5d, 0xa5, 0xa5, 0x9d, 0x35, 0x7b, 0x74, 0xc9, 0x59, 0x89, 0xe9, 0x85, 0xd6, 0x64,
0x1c, 0x33, 0xa2, 0xde, 0xf8, 0x42, 0xf8, 0xe1, 0xc8, 0xb1, 0xc8, 0x9e, 0x69, 0x5c, 0xff, 0xca, 0xc6, 0xe2, 0xd1, 0x25, 0x6c, 0x9d, 0x24, 0x49, 0x3f, 0xf2, 0x50, 0x90, 0x26, 0xc0, 0xdb, 0x10,
0x17, 0x52, 0x38, 0x36, 0xb9, 0x0d, 0x7b, 0x1a, 0xf8, 0x26, 0x92, 0x83, 0xf0, 0x98, 0xbb, 0x63, 0xaf, 0x62, 0x74, 0x25, 0x7a, 0x4e, 0x8d, 0x38, 0x66, 0xa4, 0xbd, 0xf1, 0x85, 0xf0, 0xc3, 0x91,
0x74, 0xd6, 0x08, 0x81, 0xa6, 0x06, 0x07, 0x22, 0x6d, 0xb0, 0xe7, 0xac, 0x2b, 0xcb, 0x7e, 0x92, 0x63, 0x91, 0x5d, 0xd3, 0xb8, 0x93, 0x2b, 0x5f, 0x48, 0xe1, 0xd8, 0xe4, 0x0e, 0xec, 0x6a, 0xe0,
0x44, 0xc9, 0xe9, 0xc5, 0x85, 0x40, 0xe9, 0x78, 0x0f, 0x9f, 0xc1, 0xbd, 0x15, 0xb1, 0x91, 0x5d, 0x9b, 0x48, 0x0e, 0xc2, 0x3e, 0x77, 0xc7, 0xe8, 0xac, 0x11, 0x02, 0x4d, 0x0d, 0x0e, 0x44, 0xda,
0xa8, 0x1b, 0xf4, 0x1c, 0x9d, 0x9a, 0x32, 0x7d, 0x1b, 0x8a, 0x1c, 0xb0, 0x7a, 0x7f, 0xd9, 0x50, 0x60, 0xcf, 0x59, 0x57, 0x96, 0x27, 0x49, 0x12, 0x25, 0xa7, 0x17, 0x17, 0x02, 0xa5, 0xe3, 0x3d,
0x4f, 0x6d, 0x67, 0xa1, 0x4b, 0x8e, 0x61, 0x3b, 0x5b, 0x65, 0xa4, 0x5d, 0xb9, 0xdf, 0xf4, 0x24, 0x7a, 0x06, 0xf7, 0x57, 0xc4, 0x46, 0x76, 0xa0, 0x6e, 0xd0, 0x73, 0x74, 0x6a, 0xca, 0xf4, 0x6d,
0x6f, 0xdf, 0xaf, 0xde, 0x7d, 0xe9, 0x04, 0x7f, 0x65, 0x18, 0xd5, 0x3e, 0x20, 0xf7, 0x97, 0xa6, 0x28, 0x72, 0xc0, 0xea, 0xfd, 0x69, 0x43, 0x3d, 0xb5, 0x9d, 0x85, 0x2e, 0xe9, 0xc3, 0x56, 0xb6,
0x77, 0xb1, 0x6c, 0xda, 0x07, 0xd5, 0xc2, 0x25, 0x9e, 0x20, 0xa8, 0xe2, 0xc9, 0x17, 0x4b, 0x15, 0x1a, 0x49, 0xbb, 0x72, 0x5f, 0xea, 0xc9, 0xdf, 0x7e, 0x50, 0xbd, 0x4b, 0xd3, 0x89, 0xff, 0xca,
0x4f, 0x69, 0xa3, 0x30, 0x70, 0x8a, 0x1d, 0x3c, 0x94, 0x09, 0xf2, 0x09, 0x39, 0x58, 0x7a, 0xc3, 0x30, 0xaa, 0xfd, 0x41, 0x1e, 0x2c, 0x4d, 0xfb, 0x62, 0x79, 0xb5, 0xf7, 0xab, 0x85, 0x4b, 0x3c,
0xa5, 0x05, 0xdd, 0xfe, 0xa0, 0xf4, 0xd0, 0x7a, 0x62, 0xbd, 0xfc, 0xf4, 0xef, 0xeb, 0x8e, 0xf5, 0x41, 0x50, 0xc5, 0x93, 0x2f, 0xa2, 0x2a, 0x9e, 0xd2, 0x06, 0x62, 0xe0, 0x14, 0x3b, 0x7d, 0x28,
0xfe, 0xba, 0x63, 0xfd, 0x7b, 0xdd, 0xb1, 0x7e, 0xbb, 0xe9, 0xd4, 0xde, 0xdf, 0x74, 0x6a, 0xff, 0x13, 0xe4, 0x13, 0xb2, 0xbf, 0xf4, 0x86, 0x4b, 0x0b, 0xbf, 0xfd, 0x41, 0xe9, 0x81, 0xf5, 0xc4,
0xdc, 0x74, 0x6a, 0xdf, 0xb7, 0x57, 0xff, 0x6b, 0x77, 0xbe, 0xa9, 0xff, 0x1c, 0xfd, 0x17, 0x00, 0x7a, 0xf9, 0xe9, 0x5f, 0xd7, 0x1d, 0xeb, 0xfd, 0x75, 0xc7, 0xfa, 0xf7, 0xba, 0x63, 0xfd, 0x76,
0x00, 0xff, 0xff, 0xa1, 0xe4, 0x04, 0x3d, 0xff, 0x09, 0x00, 0x00, 0xd3, 0xa9, 0xbd, 0xbf, 0xe9, 0xd4, 0xfe, 0xb9, 0xe9, 0xd4, 0xbe, 0x6f, 0xaf, 0xfe, 0x57, 0xf1,
0x7c, 0x43, 0xff, 0x39, 0xfa, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x13, 0xff, 0xe7, 0x17, 0x4f, 0x0a,
0x00, 0x00,
} }
func (m *HeadSyncRange) Marshal() (dAtA []byte, err error) { func (m *HeadSyncRange) Marshal() (dAtA []byte, err error) {
@ -1582,6 +1600,13 @@ func (m *SpacePushRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if len(m.Credential) > 0 {
i -= len(m.Credential)
copy(dAtA[i:], m.Credential)
i = encodeVarintSpacesync(dAtA, i, uint64(len(m.Credential)))
i--
dAtA[i] = 0x12
}
if m.Payload != nil { if m.Payload != nil {
{ {
size, err := m.Payload.MarshalToSizedBuffer(dAtA[:i]) size, err := m.Payload.MarshalToSizedBuffer(dAtA[:i])
@ -1768,6 +1793,13 @@ func (m *SpaceHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if len(m.SpaceHeaderPayload) > 0 {
i -= len(m.SpaceHeaderPayload)
copy(dAtA[i:], m.SpaceHeaderPayload)
i = encodeVarintSpacesync(dAtA, i, uint64(len(m.SpaceHeaderPayload)))
i--
dAtA[i] = 0x32
}
if len(m.Seed) > 0 { if len(m.Seed) > 0 {
i -= len(m.Seed) i -= len(m.Seed)
copy(dAtA[i:], m.Seed) copy(dAtA[i:], m.Seed)
@ -2276,6 +2308,10 @@ func (m *SpacePushRequest) Size() (n int) {
l = m.Payload.Size() l = m.Payload.Size()
n += 1 + l + sovSpacesync(uint64(l)) n += 1 + l + sovSpacesync(uint64(l))
} }
l = len(m.Credential)
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
return n return n
} }
@ -2367,6 +2403,10 @@ func (m *SpaceHeader) Size() (n int) {
if l > 0 { if l > 0 {
n += 1 + l + sovSpacesync(uint64(l)) n += 1 + l + sovSpacesync(uint64(l))
} }
l = len(m.SpaceHeaderPayload)
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
return n return n
} }
@ -3363,6 +3403,40 @@ func (m *SpacePushRequest) Unmarshal(dAtA []byte) error {
return err return err
} }
iNdEx = postIndex iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Credential", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpacesync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthSpacesync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthSpacesync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Credential = append(m.Credential[:0], dAtA[iNdEx:postIndex]...)
if m.Credential == nil {
m.Credential = []byte{}
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipSpacesync(dAtA[iNdEx:]) skippy, err := skipSpacesync(dAtA[iNdEx:])
@ -3987,6 +4061,40 @@ func (m *SpaceHeader) Unmarshal(dAtA []byte) error {
m.Seed = []byte{} m.Seed = []byte{}
} }
iNdEx = postIndex iNdEx = postIndex
case 6:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SpaceHeaderPayload", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpacesync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthSpacesync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthSpacesync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.SpaceHeaderPayload = append(m.SpaceHeaderPayload[:0], dAtA[iNdEx:postIndex]...)
if m.SpaceHeaderPayload == nil {
m.SpaceHeaderPayload = []byte{}
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipSpacesync(dAtA[iNdEx:]) skippy, err := skipSpacesync(dAtA[iNdEx:])

View File

@ -3,15 +3,16 @@ package syncstatus
import ( import (
"context" "context"
"fmt" "fmt"
"sync"
"time"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
"github.com/anytypeio/any-sync/util/periodicsync" "github.com/anytypeio/any-sync/util/periodicsync"
"github.com/anytypeio/any-sync/util/slice" "github.com/anytypeio/any-sync/util/slice"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"sync"
"time"
) )
const ( const (
@ -19,7 +20,7 @@ const (
syncTimeout = time.Second syncTimeout = time.Second
) )
var log = logger.NewNamed("commonspace.syncstatus") var log = logger.NewNamed("common.commonspace.syncstatus")
type UpdateReceiver interface { type UpdateReceiver interface {
UpdateTree(ctx context.Context, treeId string, status SyncStatus) (err error) UpdateTree(ctx context.Context, treeId string, status SyncStatus) (err error)

View File

@ -4,8 +4,10 @@ package coordinatorclient
import ( import (
"context" "context"
"github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app"
"github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anytypeio/any-sync/coordinator/coordinatorproto" "github.com/anytypeio/any-sync/coordinator/coordinatorproto"
"github.com/anytypeio/any-sync/net/pool" "github.com/anytypeio/any-sync/net/pool"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
"github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/nodeconf"
) )
@ -16,7 +18,9 @@ func New() CoordinatorClient {
} }
type CoordinatorClient interface { type CoordinatorClient interface {
SpaceSign(ctx context.Context, spaceId string) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error)
StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error)
SpaceSign(ctx context.Context, spaceId string, spaceHeader []byte) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error)
FileLimitCheck(ctx context.Context, spaceId string, identity []byte) (limit uint64, err error) FileLimitCheck(ctx context.Context, spaceId string, identity []byte) (limit uint64, err error)
app.Component app.Component
} }
@ -26,6 +30,40 @@ type coordinatorClient struct {
nodeConf nodeconf.Service nodeConf nodeconf.Service
} }
func (c *coordinatorClient) ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error) {
cl, err := c.client(ctx)
if err != nil {
return
}
resp, err := cl.SpaceStatusChange(ctx, &coordinatorproto.SpaceStatusChangeRequest{
SpaceId: spaceId,
DeletionChangeId: deleteRaw.GetId(),
DeletionChangePayload: deleteRaw.GetRawChange(),
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
status = resp.Payload
return
}
func (c *coordinatorClient) StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error) {
cl, err := c.client(ctx)
if err != nil {
return
}
resp, err := cl.SpaceStatusCheck(ctx, &coordinatorproto.SpaceStatusCheckRequest{
SpaceId: spaceId,
})
if err != nil {
err = rpcerr.Unwrap(err)
return
}
status = resp.Payload
return
}
func (c *coordinatorClient) Init(a *app.App) (err error) { func (c *coordinatorClient) Init(a *app.App) (err error) {
c.pool = a.MustComponent(pool.CName).(pool.Service).NewPool(CName) c.pool = a.MustComponent(pool.CName).(pool.Service).NewPool(CName)
c.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service) c.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service)
@ -36,15 +74,17 @@ func (c *coordinatorClient) Name() (name string) {
return CName return CName
} }
func (c *coordinatorClient) SpaceSign(ctx context.Context, spaceId string) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) { func (c *coordinatorClient) SpaceSign(ctx context.Context, spaceId string, spaceHeader []byte) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) {
cl, err := c.client(ctx) cl, err := c.client(ctx)
if err != nil { if err != nil {
return return
} }
resp, err := cl.SpaceSign(ctx, &coordinatorproto.SpaceSignRequest{ resp, err := cl.SpaceSign(ctx, &coordinatorproto.SpaceSignRequest{
SpaceId: spaceId, SpaceId: spaceId,
Header: spaceHeader,
}) })
if err != nil { if err != nil {
err = rpcerr.Unwrap(err)
return return
} }
return resp.Receipt, nil return resp.Receipt, nil
@ -60,6 +100,7 @@ func (c *coordinatorClient) FileLimitCheck(ctx context.Context, spaceId string,
SpaceId: spaceId, SpaceId: spaceId,
}) })
if err != nil { if err != nil {
err = rpcerr.Unwrap(err)
return return
} }
return resp.Limit, nil return resp.Limit, nil

View File

@ -9,6 +9,7 @@ import (
reflect "reflect" reflect "reflect"
app "github.com/anytypeio/any-sync/app" app "github.com/anytypeio/any-sync/app"
treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto"
coordinatorproto "github.com/anytypeio/any-sync/coordinator/coordinatorproto" coordinatorproto "github.com/anytypeio/any-sync/coordinator/coordinatorproto"
gomock "github.com/golang/mock/gomock" gomock "github.com/golang/mock/gomock"
) )
@ -36,6 +37,21 @@ func (m *MockCoordinatorClient) EXPECT() *MockCoordinatorClientMockRecorder {
return m.recorder return m.recorder
} }
// ChangeStatus mocks base method.
func (m *MockCoordinatorClient) ChangeStatus(arg0 context.Context, arg1 string, arg2 *treechangeproto.RawTreeChangeWithId) (*coordinatorproto.SpaceStatusPayload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChangeStatus", arg0, arg1, arg2)
ret0, _ := ret[0].(*coordinatorproto.SpaceStatusPayload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChangeStatus indicates an expected call of ChangeStatus.
func (mr *MockCoordinatorClientMockRecorder) ChangeStatus(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeStatus", reflect.TypeOf((*MockCoordinatorClient)(nil).ChangeStatus), arg0, arg1, arg2)
}
// FileLimitCheck mocks base method. // FileLimitCheck mocks base method.
func (m *MockCoordinatorClient) FileLimitCheck(arg0 context.Context, arg1 string, arg2 []byte) (uint64, error) { func (m *MockCoordinatorClient) FileLimitCheck(arg0 context.Context, arg1 string, arg2 []byte) (uint64, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
@ -80,16 +96,31 @@ func (mr *MockCoordinatorClientMockRecorder) Name() *gomock.Call {
} }
// SpaceSign mocks base method. // SpaceSign mocks base method.
func (m *MockCoordinatorClient) SpaceSign(arg0 context.Context, arg1 string) (*coordinatorproto.SpaceReceiptWithSignature, error) { func (m *MockCoordinatorClient) SpaceSign(arg0 context.Context, arg1 string, arg2 []byte) (*coordinatorproto.SpaceReceiptWithSignature, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SpaceSign", arg0, arg1) ret := m.ctrl.Call(m, "SpaceSign", arg0, arg1, arg2)
ret0, _ := ret[0].(*coordinatorproto.SpaceReceiptWithSignature) ret0, _ := ret[0].(*coordinatorproto.SpaceReceiptWithSignature)
ret1, _ := ret[1].(error) ret1, _ := ret[1].(error)
return ret0, ret1 return ret0, ret1
} }
// SpaceSign indicates an expected call of SpaceSign. // SpaceSign indicates an expected call of SpaceSign.
func (mr *MockCoordinatorClientMockRecorder) SpaceSign(arg0, arg1 interface{}) *gomock.Call { func (mr *MockCoordinatorClientMockRecorder) SpaceSign(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper() mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceSign", reflect.TypeOf((*MockCoordinatorClient)(nil).SpaceSign), arg0, arg1) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceSign", reflect.TypeOf((*MockCoordinatorClient)(nil).SpaceSign), arg0, arg1, arg2)
}
// StatusCheck mocks base method.
func (m *MockCoordinatorClient) StatusCheck(arg0 context.Context, arg1 string) (*coordinatorproto.SpaceStatusPayload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StatusCheck", arg0, arg1)
ret0, _ := ret[0].(*coordinatorproto.SpaceStatusPayload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StatusCheck indicates an expected call of StatusCheck.
func (mr *MockCoordinatorClientMockRecorder) StatusCheck(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StatusCheck", reflect.TypeOf((*MockCoordinatorClient)(nil).StatusCheck), arg0, arg1)
} }

File diff suppressed because it is too large Load Diff

View File

@ -42,6 +42,8 @@ type DRPCCoordinatorClient interface {
SpaceSign(ctx context.Context, in *SpaceSignRequest) (*SpaceSignResponse, error) SpaceSign(ctx context.Context, in *SpaceSignRequest) (*SpaceSignResponse, error)
FileLimitCheck(ctx context.Context, in *FileLimitCheckRequest) (*FileLimitCheckResponse, error) FileLimitCheck(ctx context.Context, in *FileLimitCheckRequest) (*FileLimitCheckResponse, error)
SpaceStatusCheck(ctx context.Context, in *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error)
SpaceStatusChange(ctx context.Context, in *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error)
} }
type drpcCoordinatorClient struct { type drpcCoordinatorClient struct {
@ -72,9 +74,29 @@ func (c *drpcCoordinatorClient) FileLimitCheck(ctx context.Context, in *FileLimi
return out, nil return out, nil
} }
func (c *drpcCoordinatorClient) SpaceStatusCheck(ctx context.Context, in *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) {
out := new(SpaceStatusCheckResponse)
err := c.cc.Invoke(ctx, "/coordinator.Coordinator/SpaceStatusCheck", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
func (c *drpcCoordinatorClient) SpaceStatusChange(ctx context.Context, in *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error) {
out := new(SpaceStatusChangeResponse)
err := c.cc.Invoke(ctx, "/coordinator.Coordinator/SpaceStatusChange", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
type DRPCCoordinatorServer interface { type DRPCCoordinatorServer interface {
SpaceSign(context.Context, *SpaceSignRequest) (*SpaceSignResponse, error) SpaceSign(context.Context, *SpaceSignRequest) (*SpaceSignResponse, error)
FileLimitCheck(context.Context, *FileLimitCheckRequest) (*FileLimitCheckResponse, error) FileLimitCheck(context.Context, *FileLimitCheckRequest) (*FileLimitCheckResponse, error)
SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error)
SpaceStatusChange(context.Context, *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error)
} }
type DRPCCoordinatorUnimplementedServer struct{} type DRPCCoordinatorUnimplementedServer struct{}
@ -87,9 +109,17 @@ func (s *DRPCCoordinatorUnimplementedServer) FileLimitCheck(context.Context, *Fi
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
} }
func (s *DRPCCoordinatorUnimplementedServer) SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCCoordinatorUnimplementedServer) SpaceStatusChange(context.Context, *SpaceStatusChangeRequest) (*SpaceStatusChangeResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
type DRPCCoordinatorDescription struct{} type DRPCCoordinatorDescription struct{}
func (DRPCCoordinatorDescription) NumMethods() int { return 2 } func (DRPCCoordinatorDescription) NumMethods() int { return 4 }
func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
switch n { switch n {
@ -111,6 +141,24 @@ func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Rec
in1.(*FileLimitCheckRequest), in1.(*FileLimitCheckRequest),
) )
}, DRPCCoordinatorServer.FileLimitCheck, true }, DRPCCoordinatorServer.FileLimitCheck, true
case 2:
return "/coordinator.Coordinator/SpaceStatusCheck", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCCoordinatorServer).
SpaceStatusCheck(
ctx,
in1.(*SpaceStatusCheckRequest),
)
}, DRPCCoordinatorServer.SpaceStatusCheck, true
case 3:
return "/coordinator.Coordinator/SpaceStatusChange", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCCoordinatorServer).
SpaceStatusChange(
ctx,
in1.(*SpaceStatusChangeRequest),
)
}, DRPCCoordinatorServer.SpaceStatusChange, true
default: default:
return "", nil, nil, nil, false return "", nil, nil, nil, false
} }
@ -151,3 +199,35 @@ func (x *drpcCoordinator_FileLimitCheckStream) SendAndClose(m *FileLimitCheckRes
} }
return x.CloseSend() return x.CloseSend()
} }
type DRPCCoordinator_SpaceStatusCheckStream interface {
drpc.Stream
SendAndClose(*SpaceStatusCheckResponse) error
}
type drpcCoordinator_SpaceStatusCheckStream struct {
drpc.Stream
}
func (x *drpcCoordinator_SpaceStatusCheckStream) SendAndClose(m *SpaceStatusCheckResponse) error {
if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil {
return err
}
return x.CloseSend()
}
type DRPCCoordinator_SpaceStatusChangeStream interface {
drpc.Stream
SendAndClose(*SpaceStatusChangeResponse) error
}
type drpcCoordinator_SpaceStatusChangeStream struct {
drpc.Stream
}
func (x *drpcCoordinator_SpaceStatusChangeStream) SendAndClose(m *SpaceStatusChangeResponse) error {
if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil {
return err
}
return x.CloseSend()
}

View File

@ -0,0 +1,16 @@
package coordinatorproto
import (
"errors"
"github.com/anytypeio/any-sync/net/rpc/rpcerr"
)
var (
errGroup = rpcerr.ErrGroup(ErrorCodes_ErrorOffset)
ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrorCodes_Unexpected))
ErrSpaceIsCreated = errGroup.Register(errors.New("space is missing"), uint64(ErrorCodes_SpaceCreated))
ErrSpaceIsDeleted = errGroup.Register(errors.New("space is deleted"), uint64(ErrorCodes_SpaceDeleted))
ErrSpaceDeletionPending = errGroup.Register(errors.New("space is set out for deletion"), uint64(ErrorCodes_SpaceDeletionPending))
ErrSpaceNotExists = errGroup.Register(errors.New("space not exists"), uint64(ErrorCodes_SpaceNotExists))
)

View File

@ -12,10 +12,38 @@ service Coordinator {
// - if a handshake identity matches a given identity // - if a handshake identity matches a given identity
// - if a requester contains in nodeconf list // - if a requester contains in nodeconf list
rpc FileLimitCheck(FileLimitCheckRequest) returns (FileLimitCheckResponse); rpc FileLimitCheck(FileLimitCheckRequest) returns (FileLimitCheckResponse);
// SpaceStatusCheck checks the status of space and tells if it is deleted or not
rpc SpaceStatusCheck(SpaceStatusCheckRequest) returns (SpaceStatusCheckResponse);
// SpaceStatusChange changes the status of space
rpc SpaceStatusChange(SpaceStatusChangeRequest) returns (SpaceStatusChangeResponse);
} }
message SpaceSignRequest { message SpaceSignRequest {
string spaceId = 1; string spaceId = 1;
bytes header = 2;
}
enum ErrorCodes {
Unexpected = 0;
SpaceDeleted = 1;
SpaceDeletionPending = 2;
SpaceCreated = 3;
SpaceNotExists = 4;
ErrorOffset = 300;
}
enum SpaceStatus {
SpaceStatusCreated = 0;
SpaceStatusPendingDeletion = 1;
SpaceStatusDeletionStarted = 2;
SpaceStatusDeleted = 3;
}
message SpaceStatusPayload {
SpaceStatus status = 1;
int64 deletionTimestamp = 2;
} }
message SpaceSignResponse { message SpaceSignResponse {
@ -53,3 +81,25 @@ message FileLimitCheckRequest {
message FileLimitCheckResponse { message FileLimitCheckResponse {
uint64 limit = 1; uint64 limit = 1;
} }
// SpaceStatusCheckRequest contains the spaceId of requested space
message SpaceStatusCheckRequest {
string spaceId = 1;
}
// SpaceStatusCheckResponse contains the current status of space
message SpaceStatusCheckResponse {
SpaceStatusPayload payload = 1;
}
// SpaceStatusChangeRequest contains the deletionChange if we want to delete space, or it is empty otherwise
message SpaceStatusChangeRequest {
string spaceId = 1;
string deletionChangeId = 2;
bytes deletionChangePayload = 3;
}
// SpaceStatusChangeResponse contains changed status of space
message SpaceStatusChangeResponse {
SpaceStatusPayload payload = 1;
}

9
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/awalterschulze/gographviz v2.0.3+incompatible
github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash v1.1.0
github.com/cheggaaa/mb/v3 v3.0.1 github.com/cheggaaa/mb/v3 v3.0.1
github.com/gobwas/glob v0.2.3
github.com/goccy/go-graphviz v0.1.0 github.com/goccy/go-graphviz v0.1.0
github.com/gogo/protobuf v1.3.2 github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.6.0 github.com/golang/mock v1.6.0
@ -19,9 +20,10 @@ require (
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.0
github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipld-format v0.4.0
github.com/ipfs/go-merkledag v0.10.0 github.com/ipfs/go-merkledag v0.10.0
github.com/ipfs/go-unixfs v0.4.3 github.com/ipfs/go-unixfs v0.4.4
github.com/libp2p/go-libp2p v0.24.1 github.com/libp2p/go-libp2p v0.24.1
github.com/minio/sha256-simd v1.0.0 github.com/minio/sha256-simd v1.0.0
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multibase v0.1.1
github.com/multiformats/go-multihash v0.2.1 github.com/multiformats/go-multihash v0.2.1
github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_golang v1.14.0
@ -30,7 +32,7 @@ require (
github.com/zeebo/errs v1.3.0 github.com/zeebo/errs v1.3.0
go.uber.org/zap v1.24.0 go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3
golang.org/x/net v0.7.0 golang.org/x/net v0.8.0
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
storj.io/drpc v0.0.32 storj.io/drpc v0.0.32
@ -74,7 +76,6 @@ require (
github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-pointer v0.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr v0.8.0 // indirect
@ -99,7 +100,7 @@ require (
golang.org/x/crypto v0.4.0 // indirect golang.org/x/crypto v0.4.0 // indirect
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
golang.org/x/sync v0.1.0 // indirect golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect golang.org/x/sys v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect lukechampine.com/blake3 v1.1.7 // indirect

14
go.sum
View File

@ -106,6 +106,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/goccy/go-graphviz v0.1.0 h1:6OqQoQ5PeAiHYe/YcusyeulqBrOkUb16HQ4ctRdyVUU= github.com/goccy/go-graphviz v0.1.0 h1:6OqQoQ5PeAiHYe/YcusyeulqBrOkUb16HQ4ctRdyVUU=
github.com/goccy/go-graphviz v0.1.0/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/goccy/go-graphviz v0.1.0/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -253,8 +255,8 @@ github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2m
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU= github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU=
github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY=
github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
@ -576,8 +578,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -647,8 +649,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

View File

@ -2,15 +2,16 @@ package peer
import ( import (
"context" "context"
"sync/atomic"
"time"
"github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/logger"
"github.com/libp2p/go-libp2p/core/sec" "github.com/libp2p/go-libp2p/core/sec"
"go.uber.org/zap" "go.uber.org/zap"
"storj.io/drpc" "storj.io/drpc"
"sync/atomic"
"time"
) )
var log = logger.NewNamed("peer") var log = logger.NewNamed("common.net.peer")
func NewPeer(sc sec.SecureConn, conn drpc.Conn) Peer { func NewPeer(sc sec.SecureConn, conn drpc.Conn) Peer {
return &peer{ return &peer{
@ -25,11 +26,13 @@ type Peer interface {
Id() string Id() string
LastUsage() time.Time LastUsage() time.Time
UpdateLastUsage() UpdateLastUsage()
TryClose(objectTTL time.Duration) (res bool, err error)
drpc.Conn drpc.Conn
} }
type peer struct { type peer struct {
id string id string
ttl time.Duration
lastUsage int64 lastUsage int64
sc sec.SecureConn sc sec.SecureConn
drpc.Conn drpc.Conn
@ -76,6 +79,13 @@ func (p *peer) UpdateLastUsage() {
atomic.StoreInt64(&p.lastUsage, time.Now().Unix()) atomic.StoreInt64(&p.lastUsage, time.Now().Unix())
} }
func (p *peer) TryClose(objectTTL time.Duration) (res bool, err error) {
if time.Now().Sub(p.LastUsage()) < objectTTL {
return false, nil
}
return true, p.Close()
}
func (p *peer) Close() (err error) { func (p *peer) Close() (err error) {
log.Debug("peer close", zap.String("peerId", p.id)) log.Debug("peer close", zap.String("peerId", p.id))
return p.Conn.Close() return p.Conn.Close()

View File

@ -49,7 +49,7 @@ func (p *pool) Get(ctx context.Context, id string) (peer.Peer, error) {
default: default:
return pr, nil return pr, nil
} }
_, _ = p.cache.Remove(id) _, _ = p.cache.Remove(ctx, id)
return p.Get(ctx, id) return p.Get(ctx, id)
} }
@ -67,6 +67,7 @@ func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error
default: default:
return pr, nil return pr, nil
} }
_, _ = p.cache.Remove(ctx, peerId)
} }
} }
// shuffle ids for better consistency // shuffle ids for better consistency

View File

@ -194,6 +194,10 @@ func (t *testPeer) LastUsage() time.Time {
func (t *testPeer) UpdateLastUsage() {} func (t *testPeer) UpdateLastUsage() {}
func (t *testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, t.Close()
}
func (t *testPeer) Close() error { func (t *testPeer) Close() error {
select { select {
case <-t.closed: case <-t.closed:

View File

@ -103,6 +103,10 @@ type testPeer struct {
drpc.Conn drpc.Conn
} }
func (t testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
return true, t.Close()
}
func (t testPeer) Id() string { func (t testPeer) Id() string {
return t.id return t.id
} }

View File

@ -2,14 +2,15 @@ package timeoutconn
import ( import (
"errors" "errors"
"github.com/anytypeio/any-sync/app/logger"
"go.uber.org/zap"
"net" "net"
"os" "os"
"time" "time"
"github.com/anytypeio/any-sync/app/logger"
"go.uber.org/zap"
) )
var log = logger.NewNamed("net.timeoutconn") var log = logger.NewNamed("common.net.timeoutconn")
type Conn struct { type Conn struct {
net.Conn net.Conn

View File

@ -157,6 +157,20 @@ func (mr *MockConfigurationMockRecorder) ConsensusPeers() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsensusPeers", reflect.TypeOf((*MockConfiguration)(nil).ConsensusPeers)) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConsensusPeers", reflect.TypeOf((*MockConfiguration)(nil).ConsensusPeers))
} }
// CoordinatorPeers mocks base method.
func (m *MockConfiguration) CoordinatorPeers() []string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CoordinatorPeers")
ret0, _ := ret[0].([]string)
return ret0
}
// CoordinatorPeers indicates an expected call of CoordinatorPeers.
func (mr *MockConfigurationMockRecorder) CoordinatorPeers() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CoordinatorPeers", reflect.TypeOf((*MockConfiguration)(nil).CoordinatorPeers))
}
// FilePeers mocks base method. // FilePeers mocks base method.
func (m *MockConfiguration) FilePeers() []string { func (m *MockConfiguration) FilePeers() []string {
m.ctrl.T.Helper() m.ctrl.T.Helper()

122
util/crc16/crc16.go Normal file
View File

@ -0,0 +1,122 @@
// Package crc16 is implementation according to CCITT standards.
//
// Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the
// following parameters:
//
// Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN"
// Width : 16 bit
// Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1)
// Initialization : 0000
// Reflect Input byte : False
// Reflect Output CRC : False
// Xor constant to output CRC : 0000
// Output for "123456789" : 31C3
//
// ported from the c++ code in the stellar-core codebase
// (https://github.com/stellar/stellar-core). The code is licensed
// as:
/*
* Copyright 2001-2010 Georges Menie (www.menie.org)
* Copyright 2010-2012 Salvatore Sanfilippo (adapted to Redis coding style)
* Copyright 2015 Stellar Development Foundation (ported to go)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the University of California, Berkeley nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package crc16
import (
"bytes"
"encoding/binary"
"fmt"
)
// ErrInvalidChecksum is returned when Validate determines either the checksum
// or the payload has been corrupted
var ErrInvalidChecksum = fmt.Errorf("invalid checksum")
var crc16tab = [256]uint16{
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
}
// Checksum returns the 2-byte checksum for the provided data
func Checksum(data []byte) []byte {
var crc uint16
var out bytes.Buffer
for _, b := range data {
crc = ((crc << 8) & 0xffff) ^ crc16tab[((crc>>8)^uint16(b))&0x00FF]
}
err := binary.Write(&out, binary.LittleEndian, crc)
if err != nil {
panic(err)
}
return out.Bytes()
}
// Validate returns an error if the provided checksum does not match
// the calculated checksum of the provided data
func Validate(data []byte, expected []byte) error {
actual := Checksum(data)
// validate the provided checksum against the calculated
if !bytes.Equal(actual, expected) {
return ErrInvalidChecksum
}
return nil
}

117
util/strkey/strkey.go Normal file
View File

@ -0,0 +1,117 @@
package strkey
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/anytypeio/any-sync/util/crc16"
"github.com/mr-tron/base58/base58"
)
// ErrInvalidVersionByte is returned when the version byte from a provided
// strkey-encoded string is not one of the valid values.
var ErrInvalidVersionByte = fmt.Errorf("invalid version byte")
// VersionByte represents one of the possible prefix values for a StrKey base
// string--the string the when encoded using base58 yields a final StrKey.
type VersionByte byte
// Decode decodes the provided StrKey into a raw value, checking the checksum
// and ensuring the expected VersionByte (the version parameter) is the value
// actually encoded into the provided src string.
func Decode(expected VersionByte, src string) ([]byte, error) {
raw, err := decodeString(src)
if err != nil {
return nil, err
}
// decode into components
version := VersionByte(raw[0])
vp := raw[0 : len(raw)-2]
payload := raw[1 : len(raw)-2]
checksum := raw[len(raw)-2:]
// ensure version byte is expected
if version != expected {
return nil, ErrInvalidVersionByte
}
// ensure checksum is valid
if err := crc16.Validate(vp, checksum); err != nil {
return nil, err
}
// if we made it through the gaunlet, return the decoded value
return payload, nil
}
// MustDecode is like Decode, but panics on error
func MustDecode(expected VersionByte, src string) []byte {
d, err := Decode(expected, src)
if err != nil {
panic(err)
}
return d
}
// Encode encodes the provided data to a StrKey, using the provided version
// byte.
func Encode(version VersionByte, src []byte) (string, error) {
var raw bytes.Buffer
// write version byte
if err := binary.Write(&raw, binary.LittleEndian, version); err != nil {
return "", err
}
// write payload
if _, err := raw.Write(src); err != nil {
return "", err
}
// calculate and write checksum
checksum := crc16.Checksum(raw.Bytes())
if _, err := raw.Write(checksum); err != nil {
return "", err
}
result := base58.FastBase58Encoding(raw.Bytes())
return result, nil
}
// MustEncode is like Encode, but panics on error
func MustEncode(version VersionByte, src []byte) string {
e, err := Encode(version, src)
if err != nil {
panic(err)
}
return e
}
// Version extracts and returns the version byte from the provided source
// string.
func Version(src string) (VersionByte, error) {
raw, err := decodeString(src)
if err != nil {
return VersionByte(0), err
}
return VersionByte(raw[0]), nil
}
// decodeString decodes a base58 string into the raw bytes, and ensures it could
// potentially be strkey encoded (i.e. it has both a version byte and a
// checksum, neither of which are explicitly checked by this func)
func decodeString(src string) ([]byte, error) {
raw, err := base58.FastBase58Decoding(src)
if err != nil {
return nil, fmt.Errorf("base58 decode failed: %s", err)
}
if len(raw) < 3 {
return nil, fmt.Errorf("encoded value is %d bytes; minimum valid length is 3", len(raw))
}
return raw, nil
}

7
util/strkey/versions.go Normal file
View File

@ -0,0 +1,7 @@
package strkey
const (
AccountAddressVersionByte VersionByte = 0x5b // Base58-encodes to 'A...'
AccountSeedVersionByte VersionByte = 0xff // Base58-encodes to 'S...'
DeviceSeedVersionByte VersionByte = 0x7d // Base58-encodes to 'D...'
)