benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change naming, add debug mode and remove sh script

Browse Source
This commit is contained in:
mcrakhman 2022-12-21 00:23:09 +01:00 committed by Mikhail Iudin
parent 7a08bd8fb3
commit 6861aa2c51
No known key found for this signature in database
GPG Key ID: FAAAA8BAABDFF1C0
3 changed files with 66 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
playground/Makefile
View File

@ -1,44 +0,0 @@
.PHONY: nodes-start nodes-stop clients-start clients-stop clean-all log-node-1 log-node-2 log-node3 log-client-1 log-client-2 gen-configs
export GOPRIVATE=github.com/anytypeio
# TODO: make different folders in etc for different programs
gen-configs:
./init.sh config_gen
nodes-start:
./init.sh nodes_start
nodes-stop:
./init.sh nodes_stop
clean-all:
rm -rf tmp
clients-start:
./init.sh clients_start
clients-stop:
./init.sh clients_stop
log-node-1:
./init.sh node_log 1
log-node-2:
./init.sh node_log 2
log-node-3:
./init.sh node_log 3
log-client-1:
./init.sh client_log 1
log-client-2:
./init.sh client_log 2
debug:
@(echo "To run debug api please run './init.sh debug [params]' with respective args to debug client")
run-all: nodes-start clients-start
sleep 3000000
run-clean: clean-all run-all

104
playground/init.sh
View File

@ -1,104 +0,0 @@
#!/bin/bash
export GOPRIVATE=github.com/anytypeio
NODE_GO="../../../node/cmd/node.go"
CLIENT_GO="../../../client/cmd/client.go"
DEBUG_GO="../util/cmd/debug/debug.go"
NODEMAP_YML="../util/cmd/nodesgen/nodemap.yml"
CONFIGS_DIR="../../../etc/configs"
NODESGEN_GO="../util/cmd/nodesgen/gen.go"
ETC_DIR="../etc"
do_usage() {
echo "usage: $0 {nodes_start|nodes_stop|clients_start|clients_stop|debug|config_gen}"
echo "usage: $0 node_log <N>"
echo "usage: $0 client_log <N>"
exit 1
}
do_nodes_start() {
for NUMBER in {1..3}; do
install -d tmp/node$NUMBER/ tmp/log/
export ANYPROF="127.0.0.1:607$NUMBER"
(cd tmp/node$NUMBER && go run $NODE_GO -c $CONFIGS_DIR/node$NUMBER.yml &>../log/node$NUMBER.log) &
NODE_PID=$!
echo $NODE_PID >tmp/node$NUMBER.pid
echo NODE_PID=$NODE_PID
done
}
do_nodes_stop() {
for NUMBER in {1..3}; do
NODE_PID=$(cat tmp/node$NUMBER.pid)
pkill -P $NODE_PID
pkill -f $CONFIGS_DIR/node$NUMBER.yml
done
}
do_node_log() {
local NODE_NUMBER=$1
tail -f -n 500 tmp/log/node$NODE_NUMBER.log
}
do_client_log() {
local CLIENT_NUMBER=$1
tail -f -n 500 tmp/log/client$CLIENT_NUMBER.log
}
do_config_gen() {
go run $NODESGEN_GO -n $NODEMAP_YML -e $ETC_DIR
cp -rf $ETC_DIR/configs/node1.yml $ETC_DIR/config.yml
cp -rf $ETC_DIR/configs/client1.yml $ETC_DIR/client.yml
}
do_clients_start() {
for NUMBER in {1..2}; do
export ANYPROF="127.0.0.1:606$NUMBER"
install -d tmp/client$NUMBER/ tmp/log/
(cd tmp/client$NUMBER && go run $CLIENT_GO -c $CONFIGS_DIR/client$NUMBER.yml &>../log/client$NUMBER.log) &
CLIENT_PID=$!
echo $CLIENT_PID >tmp/client$NUMBER.pid
echo CLIENT_PID=$CLIENT_PID
done
}
do_clients_stop() {
for NUMBER in {1..2}; do
CLIENT_PID=$(cat tmp/client$NUMBER.pid)
pkill -P $CLIENT_PID
pkill -f $CONFIGS_DIR/client$NUMBER.yml
done
}
do_debug() {
go run $DEBUG_GO --config $NODEMAP_YML $*
}
case $1 in
nodes_start | nodes_stop | clients_start | clients_stop | config_gen)
do_$1
;;
debug)
first_arg=$1
shift
do_$first_arg $*
;;
node_log)
if [[ -z $2 ]]; then
do_usage
else
do_$1 $2
fi
;;
client_log)
if [[ -z $2 ]]; then
do_usage
else
do_$1 $2
fi
;;
*)
do_usage
;;
esac

161
util/cmd/deploy/deploy.go
View File

@ -15,7 +15,7 @@ import (
var log = logger.NewNamed("cmd.deploy") var log = logger.NewNamed("cmd.deploy")
type absolutePaths struct { type rootArgs struct {
configPath string configPath string
nodePkgPath string nodePkgPath string
nodeBinaryPath string nodeBinaryPath string
@ -26,13 +26,17 @@ type absolutePaths struct {
nodePkgName string nodePkgName string
clientPkgName string clientPkgName string
isDebug bool
} }
type appPath struct { type appPath struct {
wdPath string wdPath string
binaryPath string binaryPath string
configPath string configPath string
logPath string logPath string
debugPortNum int
isDebug bool
} }
const ( const (
@ -43,107 +47,73 @@ const (
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "deploy", Use: "deploy",
PersistentPreRun: func(cmd *cobra.Command, args []string) { PersistentPreRun: func(cmd *cobra.Command, args []string) {
absolute := absolutePaths{} rootArguments := rootArgs{}
// checking package names rootArguments.nodePkgName, _ = cmd.Flags().GetString("node-pkg")
nodePkgName, err := cmd.Flags().GetString("node-pkg") rootArguments.clientPkgName, _ = cmd.Flags().GetString("client-pkg")
if err != nil {
log.With(zap.Error(err)).Fatal("node package is not specified")
}
absolute.nodePkgName = nodePkgName
clientPkgName, err := cmd.Flags().GetString("client-pkg")
if err != nil {
log.With(zap.Error(err)).Fatal("client package is not specified")
}
absolute.clientPkgName = clientPkgName
// checking configs // checking configs
cfgPath, err := cmd.Flags().GetString("config-dir") cfgPath, _ := cmd.Flags().GetString("config-dir")
if err != nil {
log.With(zap.Error(err)).Fatal("no config-dir flag is registered")
}
if _, err := os.Stat(cfgPath); os.IsNotExist(err) { if _, err := os.Stat(cfgPath); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the config directory doesn't exist") log.With(zap.Error(err)).Fatal("the config directory doesn't exist")
} }
absolute.configPath, _ = filepath.Abs(cfgPath) rootArguments.configPath, _ = filepath.Abs(cfgPath)
// checking node package // checking node package
nodePath, err := cmd.Flags().GetString("node-path") nodePath, _ := cmd.Flags().GetString("node-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no node-path flag is registered")
}
if _, err := os.Stat(path.Join(nodePath, "go.mod")); os.IsNotExist(err) { if _, err := os.Stat(path.Join(nodePath, "go.mod")); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the path to node does not contain a go module") log.With(zap.Error(err)).Fatal("the path to node does not contain a go module")
} }
absolute.nodePkgPath, _ = filepath.Abs(nodePath) rootArguments.nodePkgPath, _ = filepath.Abs(nodePath)
// checking client package // checking client package
clientPath, err := cmd.Flags().GetString("client-path") clientPath, _ := cmd.Flags().GetString("client-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no client-path flag is registered")
}
if _, err := os.Stat(path.Join(clientPath, "go.mod")); os.IsNotExist(err) { if _, err := os.Stat(path.Join(clientPath, "go.mod")); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the path to client does not contain a go module") log.With(zap.Error(err)).Fatal("the path to client does not contain a go module")
} }
absolute.clientPkgPath, _ = filepath.Abs(clientPath) rootArguments.clientPkgPath, _ = filepath.Abs(clientPath)
// checking binary path // checking binary path
binaryPath, err := cmd.Flags().GetString("bin") binaryPath, _ := cmd.Flags().GetString("bin")
if err != nil { err := createDirectoryIfNotExists(binaryPath)
log.With(zap.Error(err)).Fatal("no bin flag is registered")
}
err = createDirectoryIfNotExists(binaryPath)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatal("failed to create directory") log.With(zap.Error(err)).Fatal("failed to create directory")
} }
absoluteBinPath, _ := filepath.Abs(binaryPath) absoluteBinPath, _ := filepath.Abs(binaryPath)
absolute.clientBinaryPath = path.Join(absoluteBinPath, anytypeClientBinaryName) rootArguments.clientBinaryPath = path.Join(absoluteBinPath, anytypeClientBinaryName)
absolute.nodeBinaryPath = path.Join(absoluteBinPath, anytypeNodeBinaryName) rootArguments.nodeBinaryPath = path.Join(absoluteBinPath, anytypeNodeBinaryName)
// getting debug mode
rootArguments.isDebug, _ = cmd.Flags().GetBool("debug")
// checking db path // checking db path
dbPath, err := cmd.Flags().GetString("db-path") dbPath, _ := cmd.Flags().GetString("db-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no db-path flag is registered")
}
err = createDirectoryIfNotExists(dbPath) err = createDirectoryIfNotExists(dbPath)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatal("failed to create directory") log.With(zap.Error(err)).Fatal("failed to create directory")
} }
absolute.dbPath, _ = filepath.Abs(dbPath) rootArguments.dbPath, _ = filepath.Abs(dbPath)
ctx := context.WithValue(context.Background(), "paths", absolute) ctx := context.WithValue(context.Background(), "rootArguments", rootArguments)
cmd.SetContext(ctx) cmd.SetContext(ctx)
}, },
} }
var buildRunAllCmd = &cobra.Command{ var buildRunAllCmd = &cobra.Command{
Use: "build-and-run", Use: "build-run-all",
Long: "build and then run all clients and nodes", Long: "build and then run all clients and nodes",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths) rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok { if !ok {
log.Fatal("did not get context") log.Fatal("did not get context")
} }
// checking number of nodes to deploy numNodes, _ := cmd.Flags().GetUint("nodes")
numNodes, err := cmd.Flags().GetUint("nodes") numClients, _ := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of nodes is not specified")
}
// checking number of clients to deploy
numClients, err := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of clients is not specified")
}
// running the script // running the script
err = buildRunAll(paths, numClients, numNodes) err := buildRunAll(rootArguments, numClients, numNodes)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command") log.With(zap.Error(err)).Fatal("failed to run the command")
} }
@ -154,12 +124,12 @@ var buildAllCmd = &cobra.Command{
Use: "build-all", Use: "build-all",
Long: "builds both the clients and nodes", Long: "builds both the clients and nodes",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths) rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok { if !ok {
log.Fatal("did not get context") log.Fatal("did not get context")
} }
err := buildAll(paths) err := buildAll(rootArguments)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command") log.With(zap.Error(err)).Fatal("failed to run the command")
return return
@ -171,24 +141,15 @@ var runAllCmd = &cobra.Command{
Use: "run-all", Use: "run-all",
Long: "runs all clients and nodes", Long: "runs all clients and nodes",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths) rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok { if !ok {
log.Fatal("did not get context") log.Fatal("did not get context")
} }
// checking number of nodes to deploy numNodes, _ := cmd.Flags().GetUint("nodes")
numNodes, err := cmd.Flags().GetUint("nodes") numClients, _ := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of nodes is not specified")
}
// checking number of clients to deploy err := runAll(rootArguments, numClients, numNodes)
numClients, err := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of clients is not specified")
}
err = runAll(paths, numClients, numNodes)
if err != nil { if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command") log.With(zap.Error(err)).Fatal("failed to run the command")
return return
@ -204,6 +165,7 @@ func init() {
rootCmd.PersistentFlags().String("client-path", "client", "path to client go.mod") rootCmd.PersistentFlags().String("client-path", "client", "path to client go.mod")
rootCmd.PersistentFlags().String("bin", "bin", "path to folder where all the binaries are") rootCmd.PersistentFlags().String("bin", "bin", "path to folder where all the binaries are")
rootCmd.PersistentFlags().String("db-path", "db", "path to folder where the working directories should be placed") rootCmd.PersistentFlags().String("db-path", "db", "path to folder where the working directories should be placed")
rootCmd.PersistentFlags().Bool("debug", false, "this tells if we should run the profiler")
buildRunAllCmd.Flags().UintP("nodes", "n", 3, "number of nodes to be generated") buildRunAllCmd.Flags().UintP("nodes", "n", 3, "number of nodes to be generated")
buildRunAllCmd.Flags().UintP("clients", "c", 2, "number of clients to be generated") buildRunAllCmd.Flags().UintP("clients", "c", 2, "number of clients to be generated")
@ -229,8 +191,8 @@ func createDirectoryIfNotExists(dirPath string) (err error) {
return os.Mkdir(dirPath, os.ModePerm) return os.Mkdir(dirPath, os.ModePerm)
} }
func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (appPaths []appPath, err error) { func createAppPaths(args rootArgs, binaryPath, appName string, portNum, num int) (appPaths []appPath, err error) {
appTypePath := path.Join(paths.dbPath, appName) appTypePath := path.Join(args.dbPath, appName)
err = createDirectoryIfNotExists(appTypePath) err = createDirectoryIfNotExists(appTypePath)
if err != nil { if err != nil {
return return
@ -238,7 +200,7 @@ func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (a
for i := 0; i < num; i++ { for i := 0; i < num; i++ {
// checking if relevant config exists // checking if relevant config exists
cfgPath := path.Join(paths.configPath, fmt.Sprintf("%s%d.yml", appName, i+1)) cfgPath := path.Join(args.configPath, fmt.Sprintf("%s%d.yml", appName, i+1))
if _, err = os.Stat(cfgPath); os.IsNotExist(err) { if _, err = os.Stat(cfgPath); os.IsNotExist(err) {
err = fmt.Errorf("not enough %s configs are generated: %w", appName, err) err = fmt.Errorf("not enough %s configs are generated: %w", appName, err)
return return
@ -252,33 +214,35 @@ func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (a
} }
appPaths = append(appPaths, appPath{ appPaths = append(appPaths, appPath{
wdPath: resPath, wdPath: resPath,
binaryPath: binaryPath, binaryPath: binaryPath,
configPath: cfgPath, configPath: cfgPath,
logPath: path.Join(resPath, "app.log"), logPath: path.Join(resPath, "app.log"),
debugPortNum: portNum + i + 1,
isDebug: args.isDebug,
}) })
} }
return return
} }
func buildRunAll(paths absolutePaths, numClients, numNodes uint) (err error) { func buildRunAll(args rootArgs, numClients, numNodes uint) (err error) {
err = buildAll(paths) err = buildAll(args)
if err != nil { if err != nil {
err = fmt.Errorf("failed to build all: %w", err) err = fmt.Errorf("failed to build all: %w", err)
return return
} }
return runAll(paths, numClients, numNodes) return runAll(args, numClients, numNodes)
} }
func runAll(paths absolutePaths, numClients uint, numNodes uint) (err error) { func runAll(args rootArgs, numClients uint, numNodes uint) (err error) {
nodePaths, err := createAppPaths(paths, paths.nodeBinaryPath, "node", int(numNodes)) nodePaths, err := createAppPaths(args, args.nodeBinaryPath, "node", 6060, int(numNodes))
if err != nil { if err != nil {
err = fmt.Errorf("failed to create working directories for nodes: %w", err) err = fmt.Errorf("failed to create working directories for nodes: %w", err)
return return
} }
clientPaths, err := createAppPaths(paths, paths.clientBinaryPath, "client", int(numClients)) clientPaths, err := createAppPaths(args, args.clientBinaryPath, "client", 6070, int(numClients))
if err != nil { if err != nil {
err = fmt.Errorf("failed to create working directories for clients: %w", err) err = fmt.Errorf("failed to create working directories for clients: %w", err)
return return
@ -306,14 +270,14 @@ func runAll(paths absolutePaths, numClients uint, numNodes uint) (err error) {
return return
} }
func buildAll(paths absolutePaths) (err error) { func buildAll(args rootArgs) (err error) {
err = build(paths.nodePkgPath, paths.nodeBinaryPath, paths.nodePkgName) err = build(args.nodePkgPath, args.nodeBinaryPath, args.nodePkgName)
if err != nil { if err != nil {
err = fmt.Errorf("failed to build node: %w", err) err = fmt.Errorf("failed to build node: %w", err)
return return
} }
err = build(paths.clientPkgPath, paths.clientBinaryPath, paths.clientPkgName) err = build(args.clientPkgPath, args.clientBinaryPath, args.clientPkgName)
if err != nil { if err != nil {
err = fmt.Errorf("failed to build client: %w", err) err = fmt.Errorf("failed to build client: %w", err)
return return
@ -337,6 +301,13 @@ func build(dirPath, binaryPath, packageName string) (err error) {
func runApp(app appPath, wg *sync.WaitGroup) (err error) { func runApp(app appPath, wg *sync.WaitGroup) (err error) {
cmd := exec.Command(app.binaryPath, "-c", app.configPath) cmd := exec.Command(app.binaryPath, "-c", app.configPath)
cmd.Dir = app.wdPath cmd.Dir = app.wdPath
log := log
if app.isDebug {
log = log.With(zap.String("debug on", fmt.Sprintf("localhost:%d/debug/pprof", app.debugPortNum)))
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("ANYPROF=127.0.0.1:%d", app.debugPortNum))
}
file, err := os.OpenFile(app.logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm) file, err := os.OpenFile(app.logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
if err != nil { if err != nil {
return return