package milvus import ( "context" "encoding/json" "flag" "fmt" "io" "os" "path" "runtime" "strconv" "strings" "time" "github.com/cockroachdb/errors" "github.com/gofrs/flock" "github.com/samber/lo" clientv3 "go.etcd.io/etcd/client/v3" "github.com/milvus-io/milvus/cmd/roles" "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/util/etcd" "github.com/milvus-io/milvus/pkg/v3/util/hardware" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) func makeRuntimeDir(dir string) error { perm := os.FileMode(0o755) // os.MkdirAll equal to `mkdir -p` err := os.MkdirAll(dir, perm) if err != nil { // err will be raised only when dir exists and dir is a file instead of a directory. return fmt.Errorf("create runtime dir %s failed, err: %s", dir, err.Error()) } tmpFile, err := os.CreateTemp(dir, "tmp") if err != nil { return err } fileName := tmpFile.Name() tmpFile.Close() os.Remove(fileName) return nil } // create runtime folder func createRuntimeDir(sType string) string { var writer io.Writer if sType == typeutil.EmbeddedRole { writer = io.Discard } else { writer = os.Stderr } runtimeDir := "/run/milvus" if runtime.GOOS == "windows" { runtimeDir = "run" if err := makeRuntimeDir(runtimeDir); err != nil { fmt.Fprintf(writer, "Create runtime directory at %s failed\n", runtimeDir) os.Exit(-1) } } else { if err := makeRuntimeDir(runtimeDir); err != nil { fmt.Fprintf(writer, "Set runtime dir at %s failed, set it to /tmp/milvus directory\n", runtimeDir) runtimeDir = "/tmp/milvus" if err = makeRuntimeDir(runtimeDir); err != nil { fmt.Fprintf(writer, "Create runtime directory at %s failed\n", runtimeDir) os.Exit(-1) } } } return runtimeDir } func createPidFile(w io.Writer, filename string, runtimeDir string) (*flock.Flock, error) { fileFullName := path.Join(runtimeDir, filename) fd, err := os.OpenFile(fileFullName, os.O_CREATE|os.O_RDWR, 0o664) if err != nil { return nil, fmt.Errorf("file %s is locked, error = %w", filename, err) } fmt.Fprintln(w, "open pid file:", fileFullName) defer fd.Close() fd.Truncate(0) _, err = fmt.Fprintf(fd, "%d", os.Getpid()) if err != nil { return nil, fmt.Errorf("file %s write fail, error = %w", filename, err) } lock := flock.New(fileFullName) _, err = lock.TryLock() if err != nil { return nil, fmt.Errorf("file %s is locked, error = %w", filename, err) } fmt.Fprintln(w, "lock pid file:", fileFullName) return lock, nil } func getPidFileName(serverType string, alias string) string { var filename string if len(alias) != 0 { filename = fmt.Sprintf("%s-%s.pid", serverType, alias) } else { filename = serverType + ".pid" } return filename } func closePidFile(fd *os.File) { fd.Close() } func removePidFile(lock *flock.Flock) { filename := lock.Path() lock.Close() os.Remove(filename) } func GetMilvusRoles(args []string, flags *flag.FlagSet) *roles.MilvusRoles { alias, enableRootCoord, enableQueryCoord, enableDataCoord, enableQueryNode, enableDataNode, enableProxy, enableStreamingNode := formatFlags(args, flags) serverType := args[2] role := roles.NewMilvusRoles() role.Alias = alias role.ServerType = serverType switch serverType { case typeutil.ProxyRole: role.EnableProxy = true case typeutil.QueryNodeRole: role.EnableQueryNode = true case typeutil.DataNodeRole: role.EnableDataNode = true case typeutil.StreamingNodeRole: sessionutil.EnableEmbededQueryNodeLabel() role.EnableStreamingNode = true role.EnableQueryNode = true case typeutil.StandaloneRole, typeutil.EmbeddedRole: role.EnableMixCoord = true role.EnableProxy = true role.EnableQueryNode = true role.EnableDataNode = true role.EnableStreamingNode = true role.Local = true role.Embedded = serverType == typeutil.EmbeddedRole sessionutil.EnableStandaloneLabel() case typeutil.MixCoordRole: role.EnableMixCoord = true case typeutil.MixtureRole: role.EnableRootCoord = enableRootCoord role.EnableQueryCoord = enableQueryCoord role.EnableDataCoord = enableDataCoord role.EnableQueryNode = enableQueryNode role.EnableDataNode = enableDataNode role.EnableProxy = enableProxy role.EnableStreamingNode = enableStreamingNode if enableStreamingNode && !enableQueryNode { role.EnableQueryNode = true sessionutil.EnableEmbededQueryNodeLabel() } case typeutil.CDCRole: role.EnableCDC = true default: fmt.Fprintf(os.Stderr, "Unknown server type = %s\n%s", serverType, getHelp()) os.Exit(-1) } return role } func formatFlags(args []string, flags *flag.FlagSet) (alias string, enableRootCoord, enableQueryCoord, enableDataCoord, enableQueryNode, enableDataNode, enableProxy bool, enableStreamingNode bool, ) { flags.StringVar(&alias, "alias", "", "set alias") var enableIndexCoord bool flags.BoolVar(&enableRootCoord, typeutil.RootCoordRole, false, "enable root coordinator") flags.BoolVar(&enableQueryCoord, typeutil.QueryCoordRole, false, "enable query coordinator") flags.BoolVar(&enableDataCoord, typeutil.DataCoordRole, false, "enable data coordinator") flags.BoolVar(&enableIndexCoord, typeutil.IndexCoordRole, false, "enable index coordinator") flags.BoolVar(&enableQueryNode, typeutil.QueryNodeRole, false, "enable query node") flags.BoolVar(&enableDataNode, typeutil.DataNodeRole, false, "enable data node") flags.BoolVar(&enableProxy, typeutil.ProxyRole, false, "enable proxy node") flags.BoolVar(&enableStreamingNode, typeutil.StreamingNodeRole, false, "enable streaming node") serverType := args[2] if serverType == typeutil.EmbeddedRole { flags.SetOutput(io.Discard) } hardware.InitMaxprocs(serverType, flags) if err := flags.Parse(args[3:]); err != nil { os.Exit(-1) } return alias, enableRootCoord, enableQueryCoord, enableDataCoord, enableQueryNode, enableDataNode, enableProxy, enableStreamingNode } func getHelp() string { return runLine + "\n" + serverTypeLine } func CleanSession(metaPath string, etcdEndpoints []string, sessionSuffix []string) error { if len(sessionSuffix) == 0 { mlog.Warn(context.TODO(), "not found session info , skip to clean sessions") return nil } etcdCli, err := etcd.GetRemoteEtcdClient(etcdEndpoints) if err != nil { return err } defer etcdCli.Close() ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) defer cancel() keys := getSessionPaths(ctx, etcdCli, metaPath, sessionSuffix) if len(keys) == 0 { return nil } for _, key := range keys { _, _ = etcdCli.Delete(ctx, key) } mlog.Info(ctx, "clean sessions from etcd", mlog.Any("keys", keys)) return nil } func getSessionPaths(ctx context.Context, client *clientv3.Client, metaPath string, sessionSuffix []string) []string { sessionKeys := make([]string, 0) sessionPathPrefix := path.Join(metaPath, sessionutil.DefaultServiceRoot) newSessionSuffixSet := addActiveKeySuffix(ctx, client, sessionPathPrefix, sessionSuffix) for _, suffix := range newSessionSuffixSet { key := path.Join(sessionPathPrefix, suffix) sessionKeys = append(sessionKeys, key) } return sessionKeys } // filterUnmatchedKey skip active keys that don't match completed key, the latest active key may from standby server func addActiveKeySuffix(ctx context.Context, client *clientv3.Client, sessionPathPrefix string, sessionSuffix []string) []string { suffixSet := lo.SliceToMap(sessionSuffix, func(t string) (string, struct{}) { return t, struct{}{} }) for _, suffix := range sessionSuffix { if strings.Contains(suffix, "-") && (strings.HasPrefix(suffix, typeutil.MixCoordRole) || strings.HasPrefix(suffix, typeutil.QueryCoordRole) || strings.HasPrefix(suffix, typeutil.DataCoordRole)) { res := strings.Split(suffix, "-") if len(res) != 2 { // skip illegal keys mlog.Warn(ctx, "skip illegal key", mlog.String("suffix", suffix)) continue } serverType := res[0] targetServerID, err := strconv.ParseInt(res[1], 10, 64) if err != nil { mlog.Warn(ctx, "get server id failed from key", mlog.String("suffix", suffix), mlog.Err(err)) continue } key := path.Join(sessionPathPrefix, serverType) serverID, err := getServerID(ctx, client, key) if err != nil { mlog.Warn(ctx, "get server id failed from key", mlog.String("suffix", suffix), mlog.Err(err)) continue } if serverID == targetServerID { mlog.Info(ctx, "add active serverID key", mlog.String("suffix", suffix), mlog.String("key", key)) suffixSet[serverType] = struct{}{} } } } return lo.MapToSlice(suffixSet, func(key string, v struct{}) string { return key }) } func getServerID(ctx context.Context, client *clientv3.Client, key string) (int64, error) { resp, err := client.Get(ctx, key) if err != nil { return 0, err } if len(resp.Kvs) == 0 { return 0, errors.New("not found value") } value := resp.Kvs[0].Value session := &sessionutil.SessionRaw{} err = json.Unmarshal(value, &session) if err != nil { return 0, err } return session.ServerID, nil }