benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change naming, add debug mode and remove sh script

Browse Source
This commit is contained in:
mcrakhman 2022-12-21 00:23:09 +01:00 committed by Mikhail Iudin
parent 7a08bd8fb3
commit 6861aa2c51
No known key found for this signature in database
GPG Key ID: FAAAA8BAABDFF1C0
3 changed files with 66 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
playground/Makefile
View File

@ -1,44 +0,0 @@
.PHONY: nodes-start nodes-stop clients-start clients-stop clean-all log-node-1 log-node-2 log-node3 log-client-1 log-client-2 gen-configs
export GOPRIVATE=github.com/anytypeio
# TODO: make different folders in etc for different programs
gen-configs:
./init.sh config_gen
nodes-start:
./init.sh nodes_start
nodes-stop:
./init.sh nodes_stop
clean-all:
rm -rf tmp
clients-start:
./init.sh clients_start
clients-stop:
./init.sh clients_stop
log-node-1:
./init.sh node_log 1
log-node-2:
./init.sh node_log 2
log-node-3:
./init.sh node_log 3
log-client-1:
./init.sh client_log 1
log-client-2:
./init.sh client_log 2
debug:
@(echo "To run debug api please run './init.sh debug [params]' with respective args to debug client")
run-all: nodes-start clients-start
sleep 3000000
run-clean: clean-all run-all

104
playground/init.sh
View File

@ -1,104 +0,0 @@
#!/bin/bash
export GOPRIVATE=github.com/anytypeio
NODE_GO="../../../node/cmd/node.go"
CLIENT_GO="../../../client/cmd/client.go"
DEBUG_GO="../util/cmd/debug/debug.go"
NODEMAP_YML="../util/cmd/nodesgen/nodemap.yml"
CONFIGS_DIR="../../../etc/configs"
NODESGEN_GO="../util/cmd/nodesgen/gen.go"
ETC_DIR="../etc"
do_usage() {
echo "usage: $0 {nodes_start|nodes_stop|clients_start|clients_stop|debug|config_gen}"
echo "usage: $0 node_log <N>"
echo "usage: $0 client_log <N>"
exit 1
}
do_nodes_start() {
for NUMBER in {1..3}; do
install -d tmp/node$NUMBER/ tmp/log/
export ANYPROF="127.0.0.1:607$NUMBER"
(cd tmp/node$NUMBER && go run $NODE_GO -c $CONFIGS_DIR/node$NUMBER.yml &>../log/node$NUMBER.log) &
NODE_PID=$!
echo $NODE_PID >tmp/node$NUMBER.pid
echo NODE_PID=$NODE_PID
done
}
do_nodes_stop() {
for NUMBER in {1..3}; do
NODE_PID=$(cat tmp/node$NUMBER.pid)
pkill -P $NODE_PID
pkill -f $CONFIGS_DIR/node$NUMBER.yml
done
}
do_node_log() {
local NODE_NUMBER=$1
tail -f -n 500 tmp/log/node$NODE_NUMBER.log
}
do_client_log() {
local CLIENT_NUMBER=$1
tail -f -n 500 tmp/log/client$CLIENT_NUMBER.log
}
do_config_gen() {
go run $NODESGEN_GO -n $NODEMAP_YML -e $ETC_DIR
cp -rf $ETC_DIR/configs/node1.yml $ETC_DIR/config.yml
cp -rf $ETC_DIR/configs/client1.yml $ETC_DIR/client.yml
}
do_clients_start() {
for NUMBER in {1..2}; do
export ANYPROF="127.0.0.1:606$NUMBER"
install -d tmp/client$NUMBER/ tmp/log/
(cd tmp/client$NUMBER && go run $CLIENT_GO -c $CONFIGS_DIR/client$NUMBER.yml &>../log/client$NUMBER.log) &
CLIENT_PID=$!
echo $CLIENT_PID >tmp/client$NUMBER.pid
echo CLIENT_PID=$CLIENT_PID
done
}
do_clients_stop() {
for NUMBER in {1..2}; do
CLIENT_PID=$(cat tmp/client$NUMBER.pid)
pkill -P $CLIENT_PID
pkill -f $CONFIGS_DIR/client$NUMBER.yml
done
}
do_debug() {
go run $DEBUG_GO --config $NODEMAP_YML $*
}
case $1 in
nodes_start | nodes_stop | clients_start | clients_stop | config_gen)
do_$1
;;
debug)
first_arg=$1
shift
do_$first_arg $*
;;
node_log)
if [[ -z $2 ]]; then
do_usage
else
do_$1 $2
fi
;;
client_log)
if [[ -z $2 ]]; then
do_usage
else
do_$1 $2
fi
;;
*)
do_usage
;;
esac

161
util/cmd/deploy/deploy.go
View File

@ -15,7 +15,7 @@ import (
var log = logger.NewNamed("cmd.deploy")
type absolutePaths struct {
type rootArgs struct {
configPath string
nodePkgPath string
nodeBinaryPath string
@ -26,13 +26,17 @@ type absolutePaths struct {
nodePkgName string
clientPkgName string
isDebug bool
}
type appPath struct {
wdPath string
binaryPath string
configPath string
logPath string
wdPath string
binaryPath string
configPath string
logPath string
debugPortNum int
isDebug bool
}
const (
@ -43,107 +47,73 @@ const (
var rootCmd = &cobra.Command{
Use: "deploy",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
absolute := absolutePaths{}
rootArguments := rootArgs{}
// checking package names
nodePkgName, err := cmd.Flags().GetString("node-pkg")
if err != nil {
log.With(zap.Error(err)).Fatal("node package is not specified")
}
absolute.nodePkgName = nodePkgName
clientPkgName, err := cmd.Flags().GetString("client-pkg")
if err != nil {
log.With(zap.Error(err)).Fatal("client package is not specified")
}
absolute.clientPkgName = clientPkgName
rootArguments.nodePkgName, _ = cmd.Flags().GetString("node-pkg")
rootArguments.clientPkgName, _ = cmd.Flags().GetString("client-pkg")
// checking configs
cfgPath, err := cmd.Flags().GetString("config-dir")
if err != nil {
log.With(zap.Error(err)).Fatal("no config-dir flag is registered")
}
cfgPath, _ := cmd.Flags().GetString("config-dir")
if _, err := os.Stat(cfgPath); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the config directory doesn't exist")
}
absolute.configPath, _ = filepath.Abs(cfgPath)
rootArguments.configPath, _ = filepath.Abs(cfgPath)
// checking node package
nodePath, err := cmd.Flags().GetString("node-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no node-path flag is registered")
}
nodePath, _ := cmd.Flags().GetString("node-path")
if _, err := os.Stat(path.Join(nodePath, "go.mod")); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the path to node does not contain a go module")
}
absolute.nodePkgPath, _ = filepath.Abs(nodePath)
rootArguments.nodePkgPath, _ = filepath.Abs(nodePath)
// checking client package
clientPath, err := cmd.Flags().GetString("client-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no client-path flag is registered")
}
clientPath, _ := cmd.Flags().GetString("client-path")
if _, err := os.Stat(path.Join(clientPath, "go.mod")); os.IsNotExist(err) {
log.With(zap.Error(err)).Fatal("the path to client does not contain a go module")
}
absolute.clientPkgPath, _ = filepath.Abs(clientPath)
rootArguments.clientPkgPath, _ = filepath.Abs(clientPath)
// checking binary path
binaryPath, err := cmd.Flags().GetString("bin")
if err != nil {
log.With(zap.Error(err)).Fatal("no bin flag is registered")
}
err = createDirectoryIfNotExists(binaryPath)
binaryPath, _ := cmd.Flags().GetString("bin")
err := createDirectoryIfNotExists(binaryPath)
if err != nil {
log.With(zap.Error(err)).Fatal("failed to create directory")
}
absoluteBinPath, _ := filepath.Abs(binaryPath)
absolute.clientBinaryPath = path.Join(absoluteBinPath, anytypeClientBinaryName)
absolute.nodeBinaryPath = path.Join(absoluteBinPath, anytypeNodeBinaryName)
rootArguments.clientBinaryPath = path.Join(absoluteBinPath, anytypeClientBinaryName)
rootArguments.nodeBinaryPath = path.Join(absoluteBinPath, anytypeNodeBinaryName)
// getting debug mode
rootArguments.isDebug, _ = cmd.Flags().GetBool("debug")
// checking db path
dbPath, err := cmd.Flags().GetString("db-path")
if err != nil {
log.With(zap.Error(err)).Fatal("no db-path flag is registered")
}
dbPath, _ := cmd.Flags().GetString("db-path")
err = createDirectoryIfNotExists(dbPath)
if err != nil {
log.With(zap.Error(err)).Fatal("failed to create directory")
}
absolute.dbPath, _ = filepath.Abs(dbPath)
rootArguments.dbPath, _ = filepath.Abs(dbPath)
ctx := context.WithValue(context.Background(), "paths", absolute)
ctx := context.WithValue(context.Background(), "rootArguments", rootArguments)
cmd.SetContext(ctx)
},
}
var buildRunAllCmd = &cobra.Command{
Use: "build-and-run",
Use: "build-run-all",
Long: "build and then run all clients and nodes",
Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths)
rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok {
log.Fatal("did not get context")
}
// checking number of nodes to deploy
numNodes, err := cmd.Flags().GetUint("nodes")
if err != nil {
log.With(zap.Error(err)).Fatal("number of nodes is not specified")
}
// checking number of clients to deploy
numClients, err := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of clients is not specified")
}
numNodes, _ := cmd.Flags().GetUint("nodes")
numClients, _ := cmd.Flags().GetUint("clients")
// running the script
err = buildRunAll(paths, numClients, numNodes)
err := buildRunAll(rootArguments, numClients, numNodes)
if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command")
}
@ -154,12 +124,12 @@ var buildAllCmd = &cobra.Command{
Use: "build-all",
Long: "builds both the clients and nodes",
Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths)
rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok {
log.Fatal("did not get context")
}
err := buildAll(paths)
err := buildAll(rootArguments)
if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command")
return
@ -171,24 +141,15 @@ var runAllCmd = &cobra.Command{
Use: "run-all",
Long: "runs all clients and nodes",
Run: func(cmd *cobra.Command, args []string) {
paths, ok := cmd.Context().Value("paths").(absolutePaths)
rootArguments, ok := cmd.Context().Value("rootArguments").(rootArgs)
if !ok {
log.Fatal("did not get context")
}
// checking number of nodes to deploy
numNodes, err := cmd.Flags().GetUint("nodes")
if err != nil {
log.With(zap.Error(err)).Fatal("number of nodes is not specified")
}
numNodes, _ := cmd.Flags().GetUint("nodes")
numClients, _ := cmd.Flags().GetUint("clients")
// checking number of clients to deploy
numClients, err := cmd.Flags().GetUint("clients")
if err != nil {
log.With(zap.Error(err)).Fatal("number of clients is not specified")
}
err = runAll(paths, numClients, numNodes)
err := runAll(rootArguments, numClients, numNodes)
if err != nil {
log.With(zap.Error(err)).Fatal("failed to run the command")
return
@ -204,6 +165,7 @@ func init() {
rootCmd.PersistentFlags().String("client-path", "client", "path to client go.mod")
rootCmd.PersistentFlags().String("bin", "bin", "path to folder where all the binaries are")
rootCmd.PersistentFlags().String("db-path", "db", "path to folder where the working directories should be placed")
rootCmd.PersistentFlags().Bool("debug", false, "this tells if we should run the profiler")
buildRunAllCmd.Flags().UintP("nodes", "n", 3, "number of nodes to be generated")
buildRunAllCmd.Flags().UintP("clients", "c", 2, "number of clients to be generated")
@ -229,8 +191,8 @@ func createDirectoryIfNotExists(dirPath string) (err error) {
return os.Mkdir(dirPath, os.ModePerm)
}
func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (appPaths []appPath, err error) {
appTypePath := path.Join(paths.dbPath, appName)
func createAppPaths(args rootArgs, binaryPath, appName string, portNum, num int) (appPaths []appPath, err error) {
appTypePath := path.Join(args.dbPath, appName)
err = createDirectoryIfNotExists(appTypePath)
if err != nil {
return
@ -238,7 +200,7 @@ func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (a
for i := 0; i < num; i++ {
// checking if relevant config exists
cfgPath := path.Join(paths.configPath, fmt.Sprintf("%s%d.yml", appName, i+1))
cfgPath := path.Join(args.configPath, fmt.Sprintf("%s%d.yml", appName, i+1))
if _, err = os.Stat(cfgPath); os.IsNotExist(err) {
err = fmt.Errorf("not enough %s configs are generated: %w", appName, err)
return
@ -252,33 +214,35 @@ func createAppPaths(paths absolutePaths, binaryPath, appName string, num int) (a
}
appPaths = append(appPaths, appPath{
wdPath: resPath,
binaryPath: binaryPath,
configPath: cfgPath,
logPath: path.Join(resPath, "app.log"),
wdPath: resPath,
binaryPath: binaryPath,
configPath: cfgPath,
logPath: path.Join(resPath, "app.log"),
debugPortNum: portNum + i + 1,
isDebug: args.isDebug,
})
}
return
}
func buildRunAll(paths absolutePaths, numClients, numNodes uint) (err error) {
err = buildAll(paths)
func buildRunAll(args rootArgs, numClients, numNodes uint) (err error) {
err = buildAll(args)
if err != nil {
err = fmt.Errorf("failed to build all: %w", err)
return
}
return runAll(paths, numClients, numNodes)
return runAll(args, numClients, numNodes)
}
func runAll(paths absolutePaths, numClients uint, numNodes uint) (err error) {
nodePaths, err := createAppPaths(paths, paths.nodeBinaryPath, "node", int(numNodes))
func runAll(args rootArgs, numClients uint, numNodes uint) (err error) {
nodePaths, err := createAppPaths(args, args.nodeBinaryPath, "node", 6060, int(numNodes))
if err != nil {
err = fmt.Errorf("failed to create working directories for nodes: %w", err)
return
}
clientPaths, err := createAppPaths(paths, paths.clientBinaryPath, "client", int(numClients))
clientPaths, err := createAppPaths(args, args.clientBinaryPath, "client", 6070, int(numClients))
if err != nil {
err = fmt.Errorf("failed to create working directories for clients: %w", err)
return
@ -306,14 +270,14 @@ func runAll(paths absolutePaths, numClients uint, numNodes uint) (err error) {
return
}
func buildAll(paths absolutePaths) (err error) {
err = build(paths.nodePkgPath, paths.nodeBinaryPath, paths.nodePkgName)
func buildAll(args rootArgs) (err error) {
err = build(args.nodePkgPath, args.nodeBinaryPath, args.nodePkgName)
if err != nil {
err = fmt.Errorf("failed to build node: %w", err)
return
}
err = build(paths.clientPkgPath, paths.clientBinaryPath, paths.clientPkgName)
err = build(args.clientPkgPath, args.clientBinaryPath, args.clientPkgName)
if err != nil {
err = fmt.Errorf("failed to build client: %w", err)
return
@ -337,6 +301,13 @@ func build(dirPath, binaryPath, packageName string) (err error) {
func runApp(app appPath, wg *sync.WaitGroup) (err error) {
cmd := exec.Command(app.binaryPath, "-c", app.configPath)
cmd.Dir = app.wdPath
log := log
if app.isDebug {
log = log.With(zap.String("debug on", fmt.Sprintf("localhost:%d/debug/pprof", app.debugPortNum)))
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("ANYPROF=127.0.0.1:%d", app.debugPortNum))
}
file, err := os.OpenFile(app.logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
if err != nil {
return