// Copyright 2021 Dolthub, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package commands import ( "context" "fmt" "io" "math" "path/filepath" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/nbs" ) const tableFileIndexFlag = "index" type InspectCmd struct { } // Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command func (cmd InspectCmd) Name() string { return "inspect" } // Hidden should return true if this command should be hidden from the help text func (cmd InspectCmd) Hidden() bool { return true } // RequiresRepo should return false if this interface is implemented, and the command does not have the requirement // that it be run from within a data repository directory func (cmd InspectCmd) RequiresRepo() bool { return true } // Description returns a description of the command func (cmd InspectCmd) Description() string { return "Inspects a Dolt Database and collects stats." } func (cmd InspectCmd) Docs() *cli.CommandDocumentation { return nil } func (cmd InspectCmd) ArgParser() *argparser.ArgParser { ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0) ap.SupportsFlag(tableFileIndexFlag, "i", "Measure distribution error in table file chunk indexes.") return ap } // Exec executes the command func (cmd InspectCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int { ap := cmd.ArgParser() help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, cli.CommandDocumentationContent{}, ap)) apr := cli.ParseArgsOrDie(ap, args, help) var verr errhand.VerboseError if apr.Contains(tableFileIndexFlag) { verr = cmd.measureChunkIndexDistribution(ctx, dEnv) } return HandleVErrAndExitCode(verr, usage) } func (cmd InspectCmd) measureChunkIndexDistribution(ctx context.Context, dEnv *env.DoltEnv) errhand.VerboseError { doltDir := dEnv.GetDoltDir() if doltDir == "" { return errhand.VerboseErrorFromError(fmt.Errorf("can no longer find a database on disk")) } newGen := filepath.Join(doltDir, dbfactory.DataDir) oldGen := filepath.Join(newGen, "oldgen") itr, err := NewTableFileIter([]string{newGen, oldGen}, dEnv.FS) if err != nil { return errhand.VerboseErrorFromError(err) } sumErr, sumCnt := 0.0, 0 for { path, _ := itr.next() if path == "" { break } summary, err := cmd.processTableFile(ctx, path, dEnv.FS) if err != nil { return errhand.VerboseErrorFromError(err) } sumErr += summary.sumErr sumCnt += int(summary.count) cli.Println(summary.format()) } cli.Printf("average guess error: %f", sumErr/float64(sumCnt)) return nil } func (cmd InspectCmd) processTableFile(ctx context.Context, path string, fs filesys.Filesys) (sum *chunkIndexSummary, err error) { var rdr io.ReadCloser rdr, err = fs.OpenForRead(path) if err != nil { return sum, err } defer func() { cerr := rdr.Close() if err == nil { err = cerr } }() var prefixes []uint64 var cleanup func() prefixes, cleanup, err = nbs.GetTableIndexPrefixes(ctx, rdr.(io.ReadSeeker)) if err != nil { return sum, err } defer cleanup() sum = &chunkIndexSummary{ file: path, count: uint32(len(prefixes)), // errs: make([]float64, 0, len(prefixes)), } for i, prefix := range prefixes { sum.addPrefix(i, prefix) } return } type chunkIndexSummary struct { file string count uint32 // errs []float64 sumErr float64 maxErr float64 } func (s *chunkIndexSummary) format() string { return fmt.Sprintf("file: %s \t count: %d sum error: %f \t max error: %f ", s.file, s.count, s.sumErr, s.maxErr) } func (s *chunkIndexSummary) addPrefix(i int, prefix uint64) { g := nbs.GuessPrefixOrdinal(prefix, s.count) guessErr := math.Abs(float64(i - g)) // s.errs = append(s.errs, guessErr) s.sumErr += guessErr if guessErr > s.maxErr { s.maxErr = guessErr } }