From e5ef839a62ff4fa1d7df3e53b109baa4171ddd87 Mon Sep 17 00:00:00 2001 From: Sirish Bathina Date: Fri, 5 Feb 2021 15:26:10 -1000 Subject: [PATCH] docs --- .github/workflows/docker-publish.yml | 53 + .github/workflows/go.yml | 29 + .github/workflows/golangci-lint.yml | 21 + .github/workflows/release.yaml | 24 + .gitignore | 1 + Dockerfile | 26 + FIO.md | 7 + LICENSE | 201 +++ README.md | 34 + _config.yml | 3 + _posts/2021-02-07-Faster-Storage.md | 6 + cmd/rootCmd.go | 227 +++ docs/README.md | 7 + docs/_config.yml | 1 + docs/kubestr/404.html | 25 + docs/kubestr/Gemfile | 30 + docs/kubestr/Gemfile.lock | 80 ++ docs/kubestr/_config.yml | 55 + .../2021-02-05-welcome-to-jekyll.markdown | 29 + docs/kubestr/about.markdown | 18 + docs/kubestr/index.markdown | 6 + extra/csi-drivers | 121 ++ go.mod | 35 + go.sum | 1223 +++++++++++++++++ main.go | 30 + pkg/common/common.go | 24 + pkg/csi/csi.go | 11 + pkg/csi/mocks/mock_api_version_fetcher.go | 49 + pkg/csi/mocks/mock_application_creator.go | 80 ++ pkg/csi/mocks/mock_argument_validator.go | 81 ++ pkg/csi/mocks/mock_cleaner.go | 77 ++ pkg/csi/mocks/mock_data_validator.go | 48 + pkg/csi/mocks/mock_snapshot_creator.go | 82 ++ .../mocks/mock_snapshot_restore_stepper.go | 124 ++ pkg/csi/snapshot_restore.go | 649 +++++++++ pkg/csi/snapshot_restore_ops_test.go | 1158 ++++++++++++++++ pkg/csi/snapshot_restore_steps_test.go | 904 ++++++++++++ pkg/csi/snapshot_restore_test.go | 421 ++++++ pkg/csi/types/csi_types.go | 92 ++ pkg/fio/_config.yml | 1 + pkg/fio/dbench_license | 21 + pkg/fio/fio.go | 382 +++++ pkg/fio/fio_jobs.go | 68 + pkg/fio/fio_test.go | 889 ++++++++++++ pkg/fio/fio_types.go | 186 +++ pkg/fio/parsable_fio_output.go | 491 +++++++ pkg/kubestr/csi-drivers.go | 113 ++ pkg/kubestr/kubernetes_checks.go | 118 ++ pkg/kubestr/kubernetes_checks_test.go | 160 +++ pkg/kubestr/kubestr.go | 91 ++ pkg/kubestr/storage_provisioners.go | 429 ++++++ pkg/kubestr/storage_provisioners_test.go | 352 +++++ pkg/kubestr/utils.go | 126 ++ scripts/load_csi_provisioners.sh | 35 + 54 files changed, 9554 insertions(+) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 FIO.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 _config.yml create mode 100644 _posts/2021-02-07-Faster-Storage.md create mode 100644 cmd/rootCmd.go create mode 100644 docs/README.md create mode 100644 docs/_config.yml create mode 100644 docs/kubestr/404.html create mode 100644 docs/kubestr/Gemfile create mode 100644 docs/kubestr/Gemfile.lock create mode 100644 docs/kubestr/_config.yml create mode 100644 docs/kubestr/_posts/2021-02-05-welcome-to-jekyll.markdown create mode 100644 docs/kubestr/about.markdown create mode 100644 docs/kubestr/index.markdown create mode 100644 extra/csi-drivers create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/common/common.go create mode 100644 pkg/csi/csi.go create mode 100644 pkg/csi/mocks/mock_api_version_fetcher.go create mode 100644 pkg/csi/mocks/mock_application_creator.go create mode 100644 pkg/csi/mocks/mock_argument_validator.go create mode 100644 pkg/csi/mocks/mock_cleaner.go create mode 100644 pkg/csi/mocks/mock_data_validator.go create mode 100644 pkg/csi/mocks/mock_snapshot_creator.go create mode 100644 pkg/csi/mocks/mock_snapshot_restore_stepper.go create mode 100644 pkg/csi/snapshot_restore.go create mode 100644 pkg/csi/snapshot_restore_ops_test.go create mode 100644 pkg/csi/snapshot_restore_steps_test.go create mode 100644 pkg/csi/snapshot_restore_test.go create mode 100644 pkg/csi/types/csi_types.go create mode 100644 pkg/fio/_config.yml create mode 100644 pkg/fio/dbench_license create mode 100644 pkg/fio/fio.go create mode 100644 pkg/fio/fio_jobs.go create mode 100644 pkg/fio/fio_test.go create mode 100644 pkg/fio/fio_types.go create mode 100644 pkg/fio/parsable_fio_output.go create mode 100644 pkg/kubestr/csi-drivers.go create mode 100644 pkg/kubestr/kubernetes_checks.go create mode 100644 pkg/kubestr/kubernetes_checks_test.go create mode 100644 pkg/kubestr/kubestr.go create mode 100644 pkg/kubestr/storage_provisioners.go create mode 100644 pkg/kubestr/storage_provisioners_test.go create mode 100644 pkg/kubestr/utils.go create mode 100755 scripts/load_csi_provisioners.sh diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..420b5b7 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,53 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + +env: + IMAGE_NAME: kubestr + +jobs: + push: + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME + + - name: Log into GitHub Container Registry + # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` + run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..f0db7db --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,29 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.14 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build + run: go build -v . + + - name: Test + run: go test -v ./... diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..908d97a --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,21 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches: + - master + - main + pull_request: +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. + version: v1.29 + args: --timeout=5m diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..863be70 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,24 @@ +name: Release + +on: + release: + types: [created, edited] + +jobs: + releases-matrix: + name: Release Go Binary + runs-on: ubuntu-latest + strategy: + matrix: + # build and publish in parallel: linux/amd64, windows/amd64, darwin/amd64 + goos: [linux, windows, darwin] + goarch: [amd64] + steps: + - uses: actions/checkout@v2 + - uses: wangyoucao577/go-release-action@v1.11 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + binary_name: "kubestr" + extra_files: LICENSE README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3e4e48b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..34bb921 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM golang:alpine3.12 AS builder + +ENV GO111MODULE=on \ + CGO_ENABLED=0 \ + GOOS=linux \ + GOARCH=amd64 \ + GOBIN=/dist + +WORKDIR /app + +COPY go.mod . +COPY go.sum . + +RUN go mod download + +COPY . . + +RUN go get -ldflags="-w -s" . + +FROM alpine:3.12 + +RUN apk --no-cache add fio + +COPY --from=builder /dist/kubestr / + +ENTRYPOINT ["/kubestr"] diff --git a/FIO.md b/FIO.md new file mode 100644 index 0000000..f1e4cec --- /dev/null +++ b/FIO.md @@ -0,0 +1,7 @@ +layout: page +title: FIO +permalink: /fio/ + +# FIO + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b657e16 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# Kubestr + +## What is it? + +Kubestr is a collection of tools to discover, validate and evaluate your kubernetes storage options. + +As adoption of kubernetes grows so have the persistent storage offerings that are available to users. The introduction of [CSI](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/)(Container Storage Interface) has enabled storage providers to develop drivers with ease. In fact there are around a 100 different CSI drivers available today. Along with the existing in-tree providers, these options can make choosing the right storage difficult. + +Kubestr can assist in the following ways- +- Identify the various storage options present in a cluster. +- Validate if the storage options are configured correctly. +- Evaluate the storage using common benchmarking tools like FIO. + + + +## Using Kubestr +### To install the tool - +- Ensure that the kubernetes context is set and the cluster is accessible through your terminal. (Does [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) work?) +- Download the latest release [here](https://github.com/kastenhq/kubestr/releases/latest). +- Unpack the tool and make it an executable `chmod +x kubestr`. + +### To discover available storage options - +- Run `./kubestr` + +### To run an FIO test - +- Run `./kubestr fio -s ` +- Additional options like `--size` and `--fiofile` can be specified. +- For more information visit our [fio](https://github.com/kastenhq/kubestr/blob/master/FIO.md) page. + +### To check a CSI drivers snapshot and restore capabilities - +- Run `./kubestr csicheck -s -v ` + +## Roadmap +- In the future we plan to allow users to post their FIO results and compare to others. diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..e502ae0 --- /dev/null +++ b/_config.yml @@ -0,0 +1,3 @@ +theme: jekyll-theme-cayman +title: Kubestr +description: Explore your kubernetes storage options diff --git a/_posts/2021-02-07-Faster-Storage.md b/_posts/2021-02-07-Faster-Storage.md new file mode 100644 index 0000000..7c6ba46 --- /dev/null +++ b/_posts/2021-02-07-Faster-Storage.md @@ -0,0 +1,6 @@ +layout: post +title: "Faster Storage" +date: 2021-02-07 +categories: fio storage + +Some content diff --git a/cmd/rootCmd.go b/cmd/rootCmd.go new file mode 100644 index 0000000..279d904 --- /dev/null +++ b/cmd/rootCmd.go @@ -0,0 +1,227 @@ +// Copyright 2020 Kubestr Developers + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/kastenhq/kubestr/pkg/csi" + csitypes "github.com/kastenhq/kubestr/pkg/csi/types" + "github.com/kastenhq/kubestr/pkg/fio" + "github.com/kastenhq/kubestr/pkg/kubestr" + "github.com/spf13/cobra" +) + +var ( + output string + rootCmd = &cobra.Command{ + Use: "kubestr", + Short: "A tool to validate kubernetes storage", + Long: `kubestr is a tool that will scan your k8s cluster + and validate that the storage systems in place as well as run + performance tests.`, + Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + Baseline(ctx, output) + }, + } + + storageClass string + namespace string + containerImage string + + fioCheckerSize string + fioCheckerFilePath string + fioCheckerTestName string + fioCmd = &cobra.Command{ + Use: "fio", + Short: "Runs an fio test", + Long: `Run an fio test`, + Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + Fio(ctx, output, storageClass, fioCheckerSize, namespace, fioCheckerTestName, fioCheckerFilePath, containerImage) + }, + } + + csiCheckVolumeSnapshotClass string + csiCheckRunAsUser int64 + csiCheckCleanup bool + csiCheckSkipCFSCheck bool + csiCheckCmd = &cobra.Command{ + Use: "csicheck", + Short: "Runs the CSI snapshot restore check", + Long: "Validates a CSI provisioners ability to take a snapshot of an application and restore it", + Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + CSICheck(ctx, output, namespace, storageClass, csiCheckVolumeSnapshotClass, csiCheckRunAsUser, containerImage, csiCheckCleanup, csiCheckSkipCFSCheck) + }, + } +) + +func init() { + rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", "Options(json)") + + rootCmd.AddCommand(fioCmd) + fioCmd.Flags().StringVarP(&storageClass, "storageclass", "s", "", "The name of a Storageclass. (Required)") + _ = fioCmd.MarkFlagRequired("storageclass") + fioCmd.Flags().StringVarP(&fioCheckerSize, "size", "z", fio.DefaultPVCSize, "The size of the volume used to run FIO.") + fioCmd.Flags().StringVarP(&namespace, "namespace", "n", fio.DefaultNS, "The namespace used to run FIO.") + fioCmd.Flags().StringVarP(&fioCheckerFilePath, "fiofile", "f", "", "The path to a an fio config file.") + fioCmd.Flags().StringVarP(&fioCheckerTestName, "testname", "t", "", "The Name of a predefined kubestr fio test. Options(default-fio)") + fioCmd.Flags().StringVarP(&containerImage, "image", "i", "", "The container image used to create a pod.") + + rootCmd.AddCommand(csiCheckCmd) + csiCheckCmd.Flags().StringVarP(&storageClass, "storageclass", "s", "", "The name of a Storageclass. (Required)") + _ = csiCheckCmd.MarkFlagRequired("storageclass") + csiCheckCmd.Flags().StringVarP(&csiCheckVolumeSnapshotClass, "volumesnapshotclass", "v", "", "The name of a VolumeSnapshotClass. (Required)") + _ = csiCheckCmd.MarkFlagRequired("volumesnapshotclass") + csiCheckCmd.Flags().StringVarP(&namespace, "namespace", "n", fio.DefaultNS, "The namespace used to run the check.") + csiCheckCmd.Flags().StringVarP(&containerImage, "image", "i", "", "The container image used to create a pod.") + csiCheckCmd.Flags().BoolVarP(&csiCheckCleanup, "cleanup", "c", true, "Clean up the objects created by tool") + csiCheckCmd.Flags().Int64VarP(&csiCheckRunAsUser, "runAsUser", "u", 0, "Runs the CSI check using pods as a user (int)") + csiCheckCmd.Flags().BoolVarP(&csiCheckSkipCFSCheck, "skipCFScheck", "k", false, "Use this flag to skip validating the ability to clone a snapshot.") +} + +// Execute executes the main command +func Execute() error { + return rootCmd.Execute() +} + +// Baseline executes the baseline check +func Baseline(ctx context.Context, output string) { + p, err := kubestr.NewKubestr() + if err != nil { + fmt.Println(err.Error()) + return + } + fmt.Print(kubestr.Logo) + result := p.KubernetesChecks() + if output == "json" { + jsonRes, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(jsonRes)) + return + } + for _, retval := range result { + retval.Print() + fmt.Println() + time.Sleep(500 * time.Millisecond) + } + + provisionerList, err := p.ValidateProvisioners(ctx) + if err != nil { + fmt.Println(err.Error()) + return + } + if output == "json" { + jsonRes, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(jsonRes)) + return + } + fmt.Println("Available Storage Provisioners:") + fmt.Println() + time.Sleep(500 * time.Millisecond) // Added to introduce lag. + for _, provisioner := range provisionerList { + provisioner.Print() + fmt.Println() + time.Sleep(500 * time.Millisecond) + } +} + +// Fio executes the FIO test. +func Fio(ctx context.Context, output, storageclass, size, namespace, jobName, fioFilePath string, containerImage string) { + cli, err := kubestr.LoadKubeCli() + if err != nil { + fmt.Println(err.Error()) + return + } + fioRunner := &fio.FIOrunner{ + Cli: cli, + } + testName := "FIO test results" + var result *kubestr.TestOutput + if fioResult, err := fioRunner.RunFio(ctx, &fio.RunFIOArgs{ + StorageClass: storageclass, + Size: size, + Namespace: namespace, + FIOJobName: jobName, + FIOJobFilepath: fioFilePath, + Image: containerImage, + }); err != nil { + result = kubestr.MakeTestOutput(testName, kubestr.StatusError, err.Error(), fioResult) + } else { + result = kubestr.MakeTestOutput(testName, kubestr.StatusOK, fmt.Sprintf("\n%s", fioResult.Result.Print()), fioResult) + } + + if output == "json" { + jsonRes, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(jsonRes)) + return + } + result.Print() +} + +func CSICheck(ctx context.Context, output, + namespace string, + storageclass string, + volumesnapshotclass string, + runAsUser int64, + containerImage string, + cleanup bool, + skipCFScheck bool, +) { + testName := "CSI checker test" + kubecli, err := kubestr.LoadKubeCli() + if err != nil { + fmt.Printf("Failed to load kubeCLi (%s)", err.Error()) + return + } + dyncli, err := kubestr.LoadDynCli() + if err != nil { + fmt.Printf("Failed to load kubeCLi (%s)", err.Error()) + return + } + csiCheckRunner := &csi.SnapshotRestoreRunner{ + KubeCli: kubecli, + DynCli: dyncli, + } + var result *kubestr.TestOutput + csiCheckResult, err := csiCheckRunner.RunSnapshotRestore(ctx, &csitypes.CSISnapshotRestoreArgs{ + StorageClass: storageclass, + VolumeSnapshotClass: volumesnapshotclass, + Namespace: namespace, + RunAsUser: runAsUser, + ContainerImage: containerImage, + Cleanup: cleanup, + SkipCFSCheck: skipCFScheck, + }) + if err != nil { + result = kubestr.MakeTestOutput(testName, kubestr.StatusError, err.Error(), csiCheckResult) + } else { + result = kubestr.MakeTestOutput(testName, kubestr.StatusOK, "CSI application successfully snapshotted and restored.", csiCheckResult) + } + + if output == "json" { + jsonRes, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(jsonRes)) + return + } + result.Print() +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..597a72e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,7 @@ +# Kubestr + +Kubestr is a tool that qualifies the storage options present in a cluster. +In upcoming releases we plan to suport running an FIO test on the storage as well as testing the snapshotting capabilities of a storage provisioner. + +To run the tool - +`curl https://kastenhq.github.io/kubestr/run_kubestr.sh | bash` diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..fc24e7a --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-hacker \ No newline at end of file diff --git a/docs/kubestr/404.html b/docs/kubestr/404.html new file mode 100644 index 0000000..086a5c9 --- /dev/null +++ b/docs/kubestr/404.html @@ -0,0 +1,25 @@ +--- +permalink: /404.html +layout: default +--- + + + +
+

404

+ +

Page not found :(

+

The requested page could not be found.

+
diff --git a/docs/kubestr/Gemfile b/docs/kubestr/Gemfile new file mode 100644 index 0000000..6d5ab76 --- /dev/null +++ b/docs/kubestr/Gemfile @@ -0,0 +1,30 @@ +source "https://rubygems.org" +# Hello! This is where you manage which Jekyll version is used to run. +# When you want to use a different version, change it below, save the +# file and run `bundle install`. Run Jekyll with `bundle exec`, like so: +# +# bundle exec jekyll serve +# +# This will help ensure the proper Jekyll version is running. +# Happy Jekylling! +gem "jekyll", "~> 4.2.0" +# This is the default theme for new Jekyll sites. You may change this to anything you like. +gem "minima", "~> 2.5" +# If you want to use GitHub Pages, remove the "gem "jekyll"" above and +# uncomment the line below. To upgrade, run `bundle update github-pages`. +# gem "github-pages", group: :jekyll_plugins +# If you have any plugins, put them here! +group :jekyll_plugins do + gem "jekyll-feed", "~> 0.12" +end + +# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem +# and associated library. +platforms :mingw, :x64_mingw, :mswin, :jruby do + gem "tzinfo", "~> 1.2" + gem "tzinfo-data" +end + +# Performance-booster for watching directories on Windows +gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] + diff --git a/docs/kubestr/Gemfile.lock b/docs/kubestr/Gemfile.lock new file mode 100644 index 0000000..b3af526 --- /dev/null +++ b/docs/kubestr/Gemfile.lock @@ -0,0 +1,80 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + colorator (1.1.0) + concurrent-ruby (1.1.8) + em-websocket (0.5.2) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0.6.0) + eventmachine (1.2.7) + ffi (1.14.2) + forwardable-extended (2.6.0) + http_parser.rb (0.6.0) + i18n (1.8.8) + concurrent-ruby (~> 1.0) + jekyll (4.2.0) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (~> 2.0) + jekyll-watch (~> 2.0) + kramdown (~> 2.3) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (~> 0.4.0) + pathutil (~> 0.9) + rouge (~> 3.0) + safe_yaml (~> 1.0) + terminal-table (~> 2.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (2.1.0) + sassc (> 2.0.1, < 3.0) + jekyll-seo-tag (2.7.1) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + kramdown (2.3.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.3) + listen (3.4.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.4.0) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.6) + rb-fsevent (0.10.4) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.4) + rouge (3.26.0) + safe_yaml (1.0.5) + sassc (2.4.0) + ffi (~> 1.9) + terminal-table (2.0.0) + unicode-display_width (~> 1.1, >= 1.1.1) + unicode-display_width (1.7.0) + +PLATFORMS + x86_64-darwin-20 + +DEPENDENCIES + jekyll (~> 4.2.0) + jekyll-feed (~> 0.12) + minima (~> 2.5) + tzinfo (~> 1.2) + tzinfo-data + wdm (~> 0.1.1) + +BUNDLED WITH + 2.2.8 diff --git a/docs/kubestr/_config.yml b/docs/kubestr/_config.yml new file mode 100644 index 0000000..60e14aa --- /dev/null +++ b/docs/kubestr/_config.yml @@ -0,0 +1,55 @@ +# Welcome to Jekyll! +# +# This config file is meant for settings that affect your whole blog, values +# which you are expected to set up once and rarely edit after that. If you find +# yourself editing this file very often, consider using Jekyll's data files +# feature for the data you need to update frequently. +# +# For technical reasons, this file is *NOT* reloaded automatically when you use +# 'bundle exec jekyll serve'. If you change this file, please restart the server process. +# +# If you need help with YAML syntax, here are some quick references for you: +# https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml +# https://learnxinyminutes.com/docs/yaml/ +# +# Site settings +# These are used to personalize your new site. If you look in the HTML files, +# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. +# You can create any custom variable you would like, and they will be accessible +# in the templates via {{ site.myvariable }}. + +title: Your awesome title +email: your-email@example.com +description: >- # this means to ignore newlines until "baseurl:" + Write an awesome description for your new site here. You can edit this + line in _config.yml. It will appear in your document head meta (for + Google search results) and in your feed.xml site description. +baseurl: "" # the subpath of your site, e.g. /blog +url: "" # the base hostname & protocol for your site, e.g. http://example.com +twitter_username: jekyllrb +github_username: jekyll + +# Build settings +theme: minima +plugins: + - jekyll-feed + +# Exclude from processing. +# The following items will not be processed, by default. +# Any item listed under the `exclude:` key here will be automatically added to +# the internal "default list". +# +# Excluded items can be processed by explicitly listing the directories or +# their entries' file path in the `include:` list. +# +# exclude: +# - .sass-cache/ +# - .jekyll-cache/ +# - gemfiles/ +# - Gemfile +# - Gemfile.lock +# - node_modules/ +# - vendor/bundle/ +# - vendor/cache/ +# - vendor/gems/ +# - vendor/ruby/ diff --git a/docs/kubestr/_posts/2021-02-05-welcome-to-jekyll.markdown b/docs/kubestr/_posts/2021-02-05-welcome-to-jekyll.markdown new file mode 100644 index 0000000..8bb0ec5 --- /dev/null +++ b/docs/kubestr/_posts/2021-02-05-welcome-to-jekyll.markdown @@ -0,0 +1,29 @@ +--- +layout: post +title: "Welcome to Jekyll!" +date: 2021-02-05 15:20:32 -1000 +categories: jekyll update +--- +You’ll find this post in your `_posts` directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run `jekyll serve`, which launches a web server and auto-regenerates your site when a file is updated. + +Jekyll requires blog post files to be named according to the following format: + +`YEAR-MONTH-DAY-title.MARKUP` + +Where `YEAR` is a four-digit number, `MONTH` and `DAY` are both two-digit numbers, and `MARKUP` is the file extension representing the format used in the file. After that, include the necessary front matter. Take a look at the source for this post to get an idea about how it works. + +Jekyll also offers powerful support for code snippets: + +{% highlight ruby %} +def print_hi(name) + puts "Hi, #{name}" +end +print_hi('Tom') +#=> prints 'Hi, Tom' to STDOUT. +{% endhighlight %} + +Check out the [Jekyll docs][jekyll-docs] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll’s GitHub repo][jekyll-gh]. If you have questions, you can ask them on [Jekyll Talk][jekyll-talk]. + +[jekyll-docs]: https://jekyllrb.com/docs/home +[jekyll-gh]: https://github.com/jekyll/jekyll +[jekyll-talk]: https://talk.jekyllrb.com/ diff --git a/docs/kubestr/about.markdown b/docs/kubestr/about.markdown new file mode 100644 index 0000000..8b4e0b2 --- /dev/null +++ b/docs/kubestr/about.markdown @@ -0,0 +1,18 @@ +--- +layout: page +title: About +permalink: /about/ +--- + +This is the base Jekyll theme. You can find out more info about customizing your Jekyll theme, as well as basic Jekyll usage documentation at [jekyllrb.com](https://jekyllrb.com/) + +You can find the source code for Minima at GitHub: +[jekyll][jekyll-organization] / +[minima](https://github.com/jekyll/minima) + +You can find the source code for Jekyll at GitHub: +[jekyll][jekyll-organization] / +[jekyll](https://github.com/jekyll/jekyll) + + +[jekyll-organization]: https://github.com/jekyll diff --git a/docs/kubestr/index.markdown b/docs/kubestr/index.markdown new file mode 100644 index 0000000..0671507 --- /dev/null +++ b/docs/kubestr/index.markdown @@ -0,0 +1,6 @@ +--- +# Feel free to add content and custom Front Matter to this file. +# To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults + +layout: home +--- diff --git a/extra/csi-drivers b/extra/csi-drivers new file mode 100644 index 0000000..35f209b --- /dev/null +++ b/extra/csi-drivers @@ -0,0 +1,121 @@ +# Drivers +The following are a set of CSI driver which can be used with Kubernetes: + +> NOTE: If you would like your driver to be added to this table, please open a pull request in [this repo](https://github.com/kubernetes-csi/docs/pulls) updating this file. Other Features is allowed to be filled in Raw Block, Snapshot, Expansion and Cloning. If driver did not implement any Other Features, please leave it blank. + +> DISCLAIMER: Information in this table has not been validated by Kubernetes SIG-Storage. Users who want to use these CSI drivers need to contact driver maintainers for driver capabilities. + +## Production Drivers + +Name | CSI Driver Name | Compatible with CSI Version(s) | Description | Persistence (Beyond Pod Lifetime) | Supported Access Modes | Dynamic Provisioning | Other Features +-----|-----------------|--------------------------------|-------------|-----------------------------------|------------------------|----------------------|-------- +[Alicloud Disk](https://github.com/AliyunContainerService/csi-plugin) | `diskplugin.csi.alibabacloud.com` | v1.0 | A Container Storage Interface (CSI) Driver for Alicloud Disk | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot +[Alicloud NAS](https://github.com/AliyunContainerService/csi-plugin) | `nasplugin.csi.alibabacloud.com` | v1.0 | A Container Storage Interface (CSI) Driver for Alicloud Network Attached Storage (NAS) | Persistent | Read/Write Multiple Pods | No | +[Alicloud OSS](https://github.com/AliyunContainerService/csi-plugin)| `ossplugin.csi.alibabacloud.com` | v1.0 | A Container Storage Interface (CSI) Driver for Alicloud Object Storage Service (OSS) | Persistent | Read/Write Multiple Pods | No | +[ArStor CSI](https://github.com/huayun-docs/csi-driver-arstor) | `arstor.csi.huayun.io` | v1.0 | A Container Storage Interface (CSI) Driver for Huayun Storage Service (ArStor) | Persistent and Ephemeral | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[AWS Elastic Block Storage](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) | `ebs.csi.aws.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for AWS Elastic Block Storage (EBS) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion +[AWS Elastic File System](https://github.com/aws/aws-efs-csi-driver) | `efs.csi.aws.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for AWS Elastic File System (EFS) | Persistent | Read/Write Multiple Pods | No | +[AWS FSx for Lustre](https://github.com/aws/aws-fsx-csi-driver) | `fsx.csi.aws.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for AWS FSx for Lustre (EBS) | Persistent | Read/Write Multiple Pods | Yes | +[Azure disk](https://github.com/kubernetes-sigs/azuredisk-csi-driver) | `disk.csi.azure.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Azure disk | Persistent | Read/Write Single Pod | Yes | +[Azure file](https://github.com/kubernetes-sigs/azurefile-csi-driver) | `file.csi.azure.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Azure file | Persistent | Read/Write Multiple Pods | Yes | +[Bigtera VirtualStor (block)](https://github.com/bigtera-ce/ceph-csi) | `csi.block.bigtera.com` | v0.3, v1.0.0, v1.1.0 | A Container Storage Interface (CSI) Driver for Bigtera VirtualStor block storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion +[Bigtera VirtualStor (filesystem)](https://github.com/bigtera-ce/ceph-csi) | `csi.fs.bigtera.com` | v0.3, v1.0.0, v1.1.0 | A Container Storage Interface (CSI) Driver for Bigtera VirtualStor filesystem | Persistent | Read/Write Multiple Pods | Yes | Expansion +[CephFS](https://github.com/ceph/ceph-csi) | `cephfs.csi.ceph.com` | v0.3, v1.0.0, v1.1.0, v1.2.0 | A Container Storage Interface (CSI) Driver for CephFS | Persistent | Read/Write Multiple Pods | Yes | Expansion, Snapshot, Clone +[Ceph RBD](https://github.com/ceph/ceph-csi) | `rbd.csi.ceph.com` | v0.3, v1.0.0, v1.1.0, v1.2.0 | A Container Storage Interface (CSI) Driver for Ceph RBD | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Topology, Cloning +[ChubaoFS](https://github.com/chubaofs/chubaofs-csi) | `csi.chubaofs.com` | v1.0.0 | A Container Storage Interface (CSI) Driver for ChubaoFS Storage | Persistent | Read/Write Multiple Pods | Yes | +[Cinder](https://github.com/kubernetes/cloud-provider-openstack/tree/master/pkg/csi/cinder) | `cinder.csi.openstack.org` | v0.3, v1.0, v1.1 | A Container Storage Interface (CSI) Driver for OpenStack Cinder | Persistent and Ephemeral | Depends on the storage backend used | Yes, if storage backend supports it | Raw Block, Snapshot, Expansion +[cloudscale.ch](https://github.com/cloudscale-ch/csi-cloudscale) | `csi.cloudscale.ch` | v1.0 | A Container Storage Interface (CSI) Driver for the [cloudscale.ch](https://www.cloudscale.ch/) IaaS platform | Persistent | Read/Write Single Pod | Yes |Snapshot +[Datatom-InfinityCSI](https://github.com/datatom-infinity/infinity-csi) | `csi-infiblock-plugin` | v0.3, v1.0.0, v1.1.0 | A Container Storage Interface (CSI) Driver for DATATOM Infinity storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Topology +[Datatom-InfinityCSI (filesystem)](https://github.com/datatom-infinity/infinity-csi) | `csi-infifs-plugin` | v0.3, v1.0.0, v1.1.0 | A Container Storage Interface (CSI) Driver for DATATOM Infinity filesystem storage | Persistent | Read/Write Multiple Pods | Yes | Expansion +[Datera](https://github.com/Datera/datera-csi) | `dsp.csi.daterainc.io` | v1.0 | A Container Storage Interface (CSI) Driver for Datera Data Services Platform (DSP) | Persistent | Read/Write Single Pod | Yes |Snapshot +[DDN EXAScaler](https://github.com/DDNStorage/exa-csi-driver) | `exa.csi.ddn.com` | v1.0, v1.1 | A Container Storage Interface (CSI) Driver for DDN EXAScaler filesystems | Persistent | Read/Write Multiple Pods | Yes | Expansion +[Dell EMC PowerScale](https://github.com/dell/csi-powerscale) | `csi-isilon.dellemc.com` | v1.1 | A Container Storage Interface (CSI) Driver for [Dell EMC PowerScale](https://www.delltechnologies.com/en-us/storage/powerscale.htm) | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion, Cloning +[Dell EMC PowerMax](https://github.com/dell/csi-powermax) | `csi-powermax.dellemc.com` | v1.1 | A Container Storage Interface (CSI) Driver for [Dell EMC PowerMax](https://www.delltechnologies.com/en-us/storage/powermax.htm) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[Dell EMC PowerStore](https://github.com/dell/csi-powerstore) | `csi-powerstore.dellemc.com` | v1.1 | A Container Storage Interface (CSI) Driver for [Dell EMC PowerStore](https://www.delltechnologies.com/en-us/storage/powerstore-storage-appliance.htm) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[Dell EMC Unity](https://github.com/dell/csi-unity) | `csi-unity.dellemc.com` | v1.1 | A Container Storage Interface (CSI) Driver for [Dell EMC Unity](https://www.delltechnologies.com/en-us/storage/unity.htm) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[Dell EMC VxFlexOS](https://github.com/dell/csi-vxflexos) | `csi-vxflexos.dellemc.com` | v1.1 | A Container Storage Interface (CSI) Driver for [Dell EMC VxFlexOS](https://www.delltechnologies.com/en-us/hyperconverged-infrastructure/vxflex.htm) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Topology +[democratic-csi](https://github.com/democratic-csi/democratic-csi) | `org.democratic-csi.[X]` | v1.0,v1.1,v1.2 | Generic CSI plugin supporting zfs based solutions ([FreeNAS](https://www.freenas.org/) / [TrueNAS](https://www.truenas.com/) and [ZoL](https://zfsonlinux.org/) solutions such as [Ubuntu](https://ubuntu.com/)) | Persistent and Ephemeral | Read/Write Single Pod (Block Volume)

Read/Write Multiple Pods (File Volume) | Yes | Raw Block, Snapshot, Expansion, Cloning +[Diamanti-CSI](https://diamanti.com/use-cases/io-acceleration/#csi) | `dcx.csi.diamanti.com` | v1.0 | A Container Storage Interface (CSI) Driver for Diamanti DCX Platform | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion +[DigitalOcean Block Storage](https://github.com/digitalocean/csi-digitalocean) | `dobs.csi.digitalocean.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for DigitalOcean Block Storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion +[DriveScale](https://github.com/DriveScale/k8s-plugins) | `csi.drivescale.com` | v1.0 |A Container Storage Interface (CSI) Driver for DriveScale software composable infrastructure solution | Persistent | Read/Write Single Pod | Yes | +[Ember CSI](https://ember-csi.io) | `[x].ember-csi.io` | v0.2, v0.3, v1.0 | Multi-vendor CSI plugin supporting over 80 Drivers to provide block and mount storage to Container Orchestration systems. | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot +[Excelero NVMesh](https://github.com/Excelero/nvmesh-csi-driver) | `nvmesh-csi.excelero.com` | v1.0, v1.1 | A Container Storage Interface (CSI) Driver for Excelero NVMesh | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Expansion +[GCE Persistent Disk](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) | `pd.csi.storage.gke.io` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Google Compute Engine Persistent Disk (GCE PD) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Topology +[Google Cloud Filestore](https://github.com/kubernetes-sigs/gcp-filestore-csi-driver) | `com.google.csi.filestore` | v0.3 | A Container Storage Interface (CSI) Driver for Google Cloud Filestore | Persistent | Read/Write Multiple Pods | Yes | +[Google Cloud Storage](https://github.com/ofek/csi-gcs) | `gcs.csi.ofek.dev` | v1.0 | A Container Storage Interface (CSI) Driver for Google Cloud Storage | Persistent and Ephemeral | Read/Write Multiple Pods | Yes | Expansion +[GlusterFS](https://github.com/gluster/gluster-csi-driver) | `org.gluster.glusterfs` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for GlusterFS | Persistent | Read/Write Multiple Pods | Yes | Snapshot +[Gluster VirtBlock](https://github.com/gluster/gluster-csi-driver) | `org.gluster.glustervirtblock` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Gluster Virtual Block volumes | Persistent | Read/Write Single Pod | Yes | +[Hammerspace CSI](https://github.com/hammer-space/csi-plugin) | `com.hammerspace.csi` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Hammerspace Storage | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot +[Hedvig](https://documentation.commvault.com/commvault/hedvig/others/pdf/Hedvig_CSI_User_Guide.pdf) | `io.hedvig.csi` | v1.0 | A Container Storage Interface (CSI) Driver for Hedvig | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion +[Hetzner Cloud Volumes CSI](https://github.com/hetznercloud/csi-driver) | `csi.hetzner.cloud` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for Hetzner Cloud Volumes | Persistent | Read/Write Single Pod | Yes | Raw Block, Expansion +[Hitachi Vantara](https://knowledge.hitachivantara.com/Documents/Adapters_and_Drivers/Storage_Adapters_and_Drivers/Containers) | `hspc.csi.hitachi.com` | v1.2 | A Container Storage Interface (CSI) Driver for VSP series Storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[HPE](https://github.com/hpe-storage/csi-driver) | `csi.hpe.com` | v1.0, v1.1, v1.2 | A [multi-platform](https://scod.hpedev.io/csi_driver) Container Storage Interface (CSI) driver. Supports [HPE Nimble Storage](https://hpe.com/storage/nimble), [HPE Primera](https://hpe.com/storage/primera) and [HPE 3PAR](https://hpe.com/storage/3par) | Persistent and Ephemeral | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning +[Huawei Storage CSI](https://github.com/Huawei/eSDK_K8S_Plugin) | `csi.huawei.com` | v1.0 | A Container Storage Interface (CSI) Driver for FusionStorage, OceanStor 100D, OceanStor Dorado V3, OceanStor Dorado V6, OceanStor V3, OceanStor V5 | Persistent | Read/Write Multiple Pod | Yes | Snapshot, Expansion, Cloning +[HyperV CSI](https://github.com/Zetanova/hyperv-csi-driver) | `eu.zetanova.csi.hyperv` | v1.0, v1.1 | A Container Storage Interface (CSI) driver to manage hyperv hosts | Persistent | Read/Write Multiple Pods | Yes | +[IBM Block Storage](https://github.com/ibm/ibm-block-csi-driver) | `block.csi.ibm.com` | v1.0, v1.1, v1.2 | A Container Storage Interface (CSI) [Driver](https://www.ibm.com/support/knowledgecenter/SSRQ8T) for IBM Spectrum Virtualize Family, IBM FlashSystem A9000 and A9000R, IBM DS8880 and DS8900. | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[IBM Spectrum Scale](https://github.com/IBM/ibm-spectrum-scale-csi) | `spectrumscale.csi.ibm.com` | v1.0, v1.1 | A Container Storage Interface (CSI) [Driver](https://www.ibm.com/support/knowledgecenter/STXKQY_CSI_SHR) for the IBM Spectrum Scale File System | Persistent | Read/Write Multiple Pod | Yes | +[IBM Cloud Block Storage VPC CSI Driver](https://cloud.ibm.com/docs/containers?topic=containers-vpc-block) | `vpc.block.csi.ibm.io` | v1.0 | A Container Storage Interface (CSI) [Driver](https://cloud.ibm.com/docs/containers?topic=containers-vpc-block) for IBM Cloud Kubernetes Service and Red Hat OpenShift on IBM Cloud | Persistent | Read/Write Single Pod | Yes | Raw Block | +[Infinidat](https://github.com/Infinidat/infinibox-csi-driver) | `infinibox-csi-driver` | v1.0, v1.1 | A Container Storage Interface (CSI) Driver for Infinidat [InfiniBox](https://infinidat.com/en/products-technology/infinibox) | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning +[Inspur InStorage CSI](https://github.com/OpenInspur/instorage-k8s) | `csi-instorage` | v1.0 | A Container Storage Interface (CSI) Driver for inspur AS/HF/CS/CF Series Primary Storage, inspur AS13000 Series SDS Storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[Intel PMEM-CSI](https://github.com/intel/pmem-csi) | `pmem-csi.intel.com` | v1.0 | A Container Storage Interface (CSI) driver for [PMEM](https://pmem.io/) from Intel | Persistent and Ephemeral | Read/Write Single Pod | Yes | Raw Block +[JuiceFS](https://github.com/juicedata/juicefs-csi-driver) | `csi.juicefs.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for JuiceFS File System | Persistent | Read/Write Multiple Pod | Yes | +[kaDalu](https://github.com/kadalu/kadalu) | `org.kadalu.gluster` | v0.3 | A CSI Driver (and operator) for GlusterFS | Persistent | Read/Write Multiple Pods | Yes | +[KumoScale Block Storage](https://github.com/KioxiaAmerica/kumoscale-csi) | `kumoscale.kioxia.com` | v1.0 | A Container Storage Interface (CSI) Driver for KumoScale Block Storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Topology +[Linode Block Storage](https://github.com/linode/linode-blockstorage-csi-driver) | `linodebs.csi.linode.com` | v1.0 | A Container Storage Interface (CSI) Driver for Linode Block Storage | Persistent | Read/Write Single Pod | Yes | +[LINSTOR](https://github.com/LINBIT/linstor-csi) | `io.drbd.linstor-csi` | v1.1 | A Container Storage Interface (CSI) Driver for [LINSTOR](https://www.linbit.com/en/linstor/) volumes | Persistent | Read/Write Single Pod | Yes | Snapshot +[Longhorn](https://github.com/longhorn/longhorn) | `driver.longhorn.io` | v1.1 | A Container Storage Interface (CSI) Driver for [Longhorn](https://longhorn.io/) volumes | Persistent | Read/Write Single Node | Yes | Raw Block +[MacroSAN](https://github.com/macrosan-csi/macrosan-csi-driver) | `csi-macrosan` | v1.0 | A Container Storage Interface (CSI) Driver for MacroSAN Block Storage | Persistent | Read/Write Single Pod | Yes | +[Manila](https://github.com/kubernetes/cloud-provider-openstack/tree/master/pkg/csi/manila) | `manila.csi.openstack.org` | v1.1, v1.2 | A Container Storage Interface (CSI) Driver for OpenStack Shared File System Service (Manila) | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Topology +[MapR](https://github.com/mapr/mapr-csi) | `com.mapr.csi-kdf` | v1.0 | A Container Storage Interface (CSI) Driver for MapR Data Platform | Persistent | Read/Write Multiple Pods | Yes | Snapshot +[MooseFS](https://github.com/moosefs/moosefs-csi) | `com.tuxera.csi.moosefs` | v1.0 | A Container Storage Interface (CSI) Driver for [MooseFS](https://moosefs.com/) clusters. | Persistent | Read/Write Multiple Pods | Yes | +[NetApp](https://github.com/NetApp/trident) | `csi.trident.netapp.io` | v1.0, v1.1, v1.2 | A Container Storage Interface (CSI) Driver for NetApp's [Trident](https://netapp-trident.readthedocs.io/) container storage orchestrator | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning, Topology +[NexentaStor File Storage](https://github.com/Nexenta/nexentastor-csi-driver) | `nexentastor-csi-driver.nexenta.com` | v1.0, v1.1, v1.2 | A Container Storage Interface (CSI) Driver for NexentaStor File Storage | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion, Cloning, Topology +[NexentaStor Block Storage](https://github.com/Nexenta/nexentastor-csi-driver-block) | `nexentastor-block-csi-driver.nexenta.com` | v1.0, v1.1, v1.2 | A Container Storage Interface (CSI) Driver for NexentaStor over iSCSI protocol | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion, Cloning, Topology, Raw block +[Nutanix](https://github.com/nutanix/csi-plugin) | `com.nutanix.csi` | v0.3, v1.0, v1.2 | A Container Storage Interface (CSI) Driver for Nutanix | Persistent | "Read/Write Single Pod" with Nutanix Volumes and "Read/Write Multiple Pods" with Nutanix Files | Yes | Raw Block, Snapshot, Expansion, Cloning +[OpenEBS](https://github.com/openebs/csi)| `cstor.csi.openebs.io` | v1.0 | A Container Storage Interface (CSI) Driver for [OpenEBS](https://www.openebs.io/)| Persistent | Read/Write Single Pod | Yes | Expansion, Snapshot, Cloning +[OpenSDS](https://github.com/opensds/nbp/tree/master/csi) | `csi-opensdsplugin` | v1.0 | A Container Storage Interface (CSI) Driver for [OpenSDS]((https://www.opensds.io/)) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot +[Open-E](https://github.com/open-e/JovianDSS-KubernetesCSI) | `com.open-e.joviandss.csi` | v1.0 | A Container Storage Interface (CSI) Driver for Open-E JovianDSS Storage | Persistent | Read/Write Single Pod | Yes | Snapshot, Cloning +[oVirt](https://github.com/openshift/ovirt-csi-driver) | `csi.ovirt.org` | v1.0 | A Container Storage Interface (CSI) Driver for [oVirt](https://ovirt.org) | Persistent | Read/Write Single Pod | Yes | Block, File Storage +[Portworx](https://github.com/libopenstorage/openstorage/tree/master/csi) | `pxd.openstorage.org` | v0.3, v1.1 | A Container Storage Interface (CSI) Driver for [Portworx](https://docs.portworx.com/portworx-install-with-kubernetes/storage-operations/csi/) | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion +[Pure Storage CSI](https://github.com/purestorage/pso-csi)| `pure-csi` | v1.0, v1.1, v1.2, v1.3 | A Container Storage Interface (CSI) Driver for Pure Storage's [Pure Service Orchestrator](https://purestorage.com/containers) | Persistent and Ephemeral | Read/Write Multiple Pods | Yes | Snapshot, Cloning, Raw Block, Topology, Expansion +[QingCloud CSI](https://github.com/yunify/qingcloud-csi)| `disk.csi.qingcloud.com` | v1.1 | A Container Storage Interface (CSI) Driver for QingCloud Block Storage | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[QingStor CSI](https://github.com/yunify/qingstor-csi) | `neonsan.csi.qingstor.com` | v0.3, v1.1 | A Container Storage Interface (CSI) Driver for NeonSAN storage system | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning +[Quobyte](https://github.com/quobyte/quobyte-csi) | `quobyte-csi` | v0.2 | A Container Storage Interface (CSI) Driver for Quobyte | Persistent | Read/Write Multiple Pods | Yes | +[ROBIN](https://get.robin.io/) | `robin` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for [ROBIN](https://docs.robin.io) | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning +[SandStone](https://github.com/sandstone-storage/sandstone-csi-driver) | `csi-sandstone-plugin` | v1.0 | A Container Storage Interface (CSI) Driver for SandStone USP | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning +[Sangfor-EDS-File-Storage](https://github.com/evan37717/sangfor-eds-csi) | `eds.csi.file.sangfor.com` | v1.0 | A Container Storage Interface (CSI) Driver for Sangfor Distributed File Storage(EDS) | Persistent | Read/Write Multiple Pods | Yes | +[Sangfor-EDS-Block-Storage](https://github.com/eds-wzc/sangfor-eds-csi) | `eds.csi.block.sangfor.com` | v1.0 | A Container Storage Interface (CSI) Driver for Sangfor Block Storage(EDS) | Persistent | Read/Write Single Pod | Yes | +[SeaweedFS](https://github.com/seaweedfs/seaweedfs-csi-driver) | `seaweedfs-csi-driver` | v1.0 | A Container Storage Interface (CSI Driver for [SeaweedFS](https://github.com/chrislusf/seaweedfs)) | Persistent | Read/Write Multiple Pods | Yes | +[Secrets Store CSI Driver](https://github.com/kubernetes-sigs/secrets-store-csi-driver) | `secrets-store.csi.k8s.io` | v0.0.10 | A Container Storage Interface (CSI) Driver for mounting secrets, keys, and certs stored in enterprise-grade external secrets stores as volumes. | Ephemeral | N/A | N/A | +[SmartX](http://www.smartx.com/?locale=en) | `csi-smtx-plugin` | v1.0 | A Container Storage Interface (CSI) Driver for SmartX ZBS Storage | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion +[SPDK-CSI](https://github.com/spdk/spdk-csi) | `csi.spdk.io` | v1.1 | A Container Storage Interface (CSI) Driver for [SPDK](https://spdk.io/) | Persistent and Ephemeral | Read/Write Single Pod | Yes | +[StorageOS](https://docs.storageos.com/docs/platforms/kubernetes/install/) | `storageos` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for [StorageOS](https://storageos.com/) | Persistent | Read/Write Multiple Pods | Yes | +[Storidge](https://docs.storidge.com/kubernetes_storage/overview.html) | `csi.cio.storidge.com` | v0.3, v1.0 | A Container Storage Interface (CSI) Driver for [Storidge CIO](https://storidge.com/) | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion +[StorPool](https://kb.storpool.com/storpool_integrations/github/kubernetes.html) | `csi-driver.storpool.com` | v1.0 | A Container Storage Interface (CSI) Driver for [StorPool](https://storpool.com/) | Persistent and Ephemeral | Read/Write Multiple Pods | Yes | Expansion +[Tencent Cloud Block Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)| `com.tencent.cloud.csi.cbs` | v1.0 | A Container Storage Interface (CSI) Driver for Tencent Cloud Block Storage | Persistent | Read/Write Single Pod | Yes | Snapshot +[Tencent Cloud File Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)| `com.tencent.cloud.csi.cfs` | v1.0 | A Container Storage Interface (CSI) Driver for Tencent Cloud File Storage | Persistent | Read/Write Multiple Pods | Yes | +[Tencent Cloud Object Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)| `com.tencent.cloud.csi.cosfs` | v1.0 | A Container Storage Interface (CSI) Driver for Tencent Cloud Object Storage | Persistent | Read/Write Multiple Pods | No | +[TopoLVM](https://github.com/cybozu-go/topolvm)| `topolvm.cybozu.com` | v1.1 | A Container Storage Interface (CSI) Driver for LVM | Persistent and Ephemeral | Read/Write Single Pod | Yes | Raw Block, Expansion, Topology Aware +[VAST Data](https://github.com/vast-data/vast-csi) | `csi.vastdata.com` | v1.0 | A Container Storage Interface (CSI) Driver for VAST Data | Persistent | Read/Write Multiple Pods | Yes | +[XSKY-EBS](https://xsky-storage.github.io/xsky-csi-driver/csi-block.html) | `csi.block.xsky.com` | v1.0 | A Container Storage Interface (CSI) Driver for XSKY Distributed Block Storage (X-EBS) | Persistent | Read/Write Single Pod | Yes | Raw Block, Snapshot, Expansion, Cloning +[XSKY-EUS](https://xsky-storage.github.io/xsky-csi-driver/csi-fs.html) | `csi.fs.xsky.com` | v1.0 | A Container Storage Interface (CSI) Driver for XSKY Distributed File Storage (X-EUS) | Persistent | Read/Write Multiple Pods | Yes | +[Vault](https://github.com/kubevault/csi-driver) | `secrets.csi.kubevault.com` | v1.0 | A Container Storage Interface (CSI) Driver for mounting HashiCorp Vault secrets as volumes. | Ephemeral | N/A | N/A | +[Veritas InfoScale Volumes](https://www.veritas.com/solution/virtualization/containers.html) | `org.veritas.infoscale` | v1.2 | A Container Storage Interface (CSI) Driver for Veritas InfoScale volumes | Persistent | Read/Write Multiple Pods | Yes | Snapshot, Expansion, Cloning +[vSphere](https://github.com/kubernetes-sigs/vsphere-csi-driver) | `csi.vsphere.vmware.com` | v1.0 | A Container Storage Interface (CSI) Driver for VMware vSphere | Persistent | Read/Write Single Pod (Block Volume)

Read/Write Multiple Pods (File Volume) | Yes | Raw Block,

Expansion (Block Volume),

Topology Aware (Block Volume) +[Vultr Block Storage](https://github.com/vultr/vultr-csi) | `block.csi.vultr.com` | v1.2 | A Container Storage Interface (CSI) Driver for Vultr Block Storage | Persistent | Read/Write Single Pod | Yes | +[WekaIO](https://github.com/weka/csi-wekafs) | `csi.weka.io` | v1.0 | A Container Storage Interface (CSI) Driver for mounting WekaIO WekaFS filesystem as volumes | Persistent | Read/Write Multiple Pods | Yes | +[Yandex.Cloud](https://github.com/flant/yandex-csi-driver) | `yandex.csi.flant.com` | v1.2 | A Container Storage Interface (CSI) plugin for Yandex.Cloud Compute Disks | Persistent | Read/Write Single Pod | Yes | +[YanRongYun](http://www.yanrongyun.com/) | ? | v1.0 | A Container Storage Interface (CSI) Driver for YanRong YRCloudFile Storage | Persistent | Read/Write Multiple Pods | Yes | +[Zadara-CSI](https://github.com/zadarastorage/zadara-csi) | `csi.zadara.com` | v1.0, v1.1 | A Container Storage Interface (CSI) plugin for Zadara VPSA Storage Array & VPSA All-Flash | Persistent | Read/Write Multiple Pods | Yes | Raw Block, Snapshot, Expansion, Cloning + +## Sample Drivers + +Name | Status | More Information +-----|--------|------- +[Flexvolume](https://github.com/kubernetes-csi/csi-driver-flex) | Sample | +[HostPath](https://github.com/kubernetes-csi/csi-driver-host-path) | v1.2.0 | Only use for a single node tests. See the [Example](example.html) page for Kubernetes-specific instructions. +[ImagePopulator](https://github.com/kubernetes-csi/csi-driver-image-populator) | Prototype | Driver that lets you use a container image as an ephemeral volume. +[In-memory Sample Mock Driver](https://github.com/kubernetes-csi/csi-test/tree/master/mock/service) | v0.3.0 | The sample mock driver used for [csi-sanity](https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity) +[NFS](https://github.com/kubernetes-csi/csi-driver-nfs) | Sample | +[Synology NAS](https://github.com/jparklab/synology-csi) | v1.0.0 | An unofficial (and unsupported) Container Storage Interface Driver for Synology NAS. +[VFS Driver](https://github.com/thecodeteam/csi-vfs) | Released | A CSI plugin that provides a virtual file system. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b7f0c1f --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module github.com/kastenhq/kubestr + +go 1.14 + +replace github.com/graymeta/stow => github.com/kastenhq/stow v0.1.2-kasten + +require ( + github.com/Masterminds/semver v1.5.0 // indirect + github.com/aws/aws-sdk-go v1.31.6 // indirect + github.com/briandowns/spinner v1.12.0 + github.com/golang/mock v1.4.4 + github.com/google/go-cmp v0.4.1 // indirect + github.com/hashicorp/golang-lru v0.5.3 // indirect + github.com/imdario/mergo v0.3.9 // indirect + github.com/jarcoal/httpmock v1.0.5 // indirect + github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 // indirect + github.com/kanisterio/kanister v0.0.0-20210112095936-bf04d6102c72 + github.com/kr/text v0.2.0 // indirect + github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 + github.com/mitchellh/mapstructure v1.3.1 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/onsi/ginkgo v1.12.0 // indirect + github.com/onsi/gomega v1.9.0 // indirect + github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.7.0 // indirect + github.com/spf13/cobra v1.0.0 + golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d // indirect + google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece // indirect + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f + gopkg.in/yaml.v2 v2.3.0 // indirect + honnef.co/go/tools v0.0.1-2020.1.5 // indirect + k8s.io/api v0.19.5 + k8s.io/apimachinery v0.19.5 + k8s.io/client-go v0.19.5 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f1bd438 --- /dev/null +++ b/go.sum @@ -0,0 +1,1223 @@ +bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= +contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= +contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= +contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v49.1.0+incompatible h1:Sz6TnzkzwsiEgD84Vujpjo0vEox9/UMeyLVWBCkZwQ4= +github.com/Azure/azure-sdk-for-go v49.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v11.7.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.0 h1:tnO41Uo+/0sxTMFY/U7aKg2abek3JOnnXcuSuba74jI= +github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.15 h1:S5SDFpmgoVyvMEOcULyEDlYFrdPmu6Wl0Ic+shkEwzg= +github.com/Azure/go-autorest/autorest v0.11.15/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0 h1:SigMbuFNuKgc1xcGhaeapbh+8fgsu+GxgDRFyg7f5lM= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.10 h1:r6fZHMaHD8B6LDCn0o5vyBFHIHrM6Ywwx7mb49lPItI= +github.com/Azure/go-autorest/autorest/adal v0.9.10/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.0 h1:nSMjYIe24eBYasAIxt859TxyXef/IqoH+8/g4+LmcVs= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.0/go.mod h1:QRTvSZQpxqm8mSErhnbI+tANIBAKP7B+UIE2z4ypUO0= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 h1:Ml+UCrnlKD+cJmSzrZ/RDcDw86NjkRUpnFh7V5JUhzU= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.0/go.mod h1:JljT387FplPzBA31vUcvsetLKF3pec5bdAxjVU4kI2s= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0 h1:z20OWOSG5aCye0HEkDp6TPmP17ZcfeMxPi6HnSALa8c= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= +github.com/IBM-Cloud/ibm-cloud-cli-sdk v0.3.0 h1:gVDpgO5vjgBdr+dSCvHf13WCC2vLsfQM+iyWsn7MjkQ= +github.com/IBM-Cloud/ibm-cloud-cli-sdk v0.3.0/go.mod h1:RiUvKuHKTBmBApDMUQzBL14pQUGKcx/IioKQPIcRQjs= +github.com/IBM/ibmcloud-storage-volume-lib v1.0.2-beta02.0.20190828145158-1da4543a60af h1:1L8wyNvXB1hzvgb8qfJXX/Y9DhFcUGHGhvdw9c30/lY= +github.com/IBM/ibmcloud-storage-volume-lib v1.0.2-beta02.0.20190828145158-1da4543a60af/go.mod h1:xFXCpJFg3Nj3HDgP79Uq/t+DTVFH/9lKxia5e6t49pg= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.24.1/go.mod h1:fGP8eQ6PugKEI0iUETYYtnP6d1pH/bdDMTel1X5ajsU= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alecthomas/participle v0.2.1/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/aokoli/goutils v1.1.0 h1:jy4ghdcYvs5EIoGssZNslIASX5m+KNMfyyKvRQ0TEVE= +github.com/aokoli/goutils v1.1.0/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.31.3 h1:vJDjoM+VlM/ZEmGyaIhUXaYAtB9lra7Qhr58SSHHjPE= +github.com/aws/aws-sdk-go v1.31.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.31.6 h1:nKjQbpXhdImctBh1e0iLg9iQW/X297LPPuY/9f92R2k= +github.com/aws/aws-sdk-go v1.31.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= +github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/briandowns/spinner v1.12.0 h1:72O0PzqGJb6G3KgrcIOtL/JAGGZ5ptOMCn9cUHmqsmw= +github.com/briandowns/spinner v1.12.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coredns/coredns v1.4.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/efarrer/iothrottler v0.0.1/go.mod h1:zGWF5N0NKSCskcPFytDAFwI121DdU/NfW4XOjpTR+ys= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.3 h1:eRfyY5SkaNJCAwmmMcADjY31ow9+N7MCLW7oRkbsINA= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/fswalker v0.2.1-0.20200214223026-f0e929ba4126/go.mod h1:ZSEBqY0IHKqWPeAbTyvccv9bb9vCnaQfHe31cm911Ng= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= +github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= +github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/raft v1.1.1-0.20190703171940-f639636d18e0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= +github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jarcoal/httpmock v1.0.5 h1:cHtVEcTxRSX4J0je7mWPfc9BpDpqzXSJ5HbymZmyHck= +github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d h1:ix3WmphUvN0GDd0DO9MH0v6/5xTv+Xm1bPN+1UJn58k= +github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kanisterio/kanister v0.0.0-20210112095936-bf04d6102c72 h1:GOupe9mgSwWp/3hUR2oRhFZWDooh4clM6OdThsJ4Wz8= +github.com/kanisterio/kanister v0.0.0-20210112095936-bf04d6102c72/go.mod h1:PAprswl9Kmod22n6LoDxWKN/BNLnr7nONinOWg4zlAk= +github.com/kastenhq/stow v0.1.2-kasten/go.mod h1:ABI2whmZOX25JbmbVuHRLFuPiGnv5lxXhduCtof7UHk= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/readahead v1.3.1/go.mod h1:AH9juHzNH7xqdqFHrMRSHeH2Ps+vFf+kblDqzPFiLJg= +github.com/klauspost/reedsolomon v1.9.7/go.mod h1:+8WD025Xpby8/kG5h/HDPIFhiiuGEtZOKw+5Y4drAD8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kopia/kopia v0.4.1-0.20200608151401-25934a544df4/go.mod h1:vYkRDf/i7NZQLTkaFsP5OqgriGP7Ki3vaznZ3ZGa65A= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 h1:ipLtV9ubLEYx42YvwDa12eVPQvjuGZoPdbCozGzVNRc= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f h1:WVPqVsbUsrzAebTEgWRAZMdDOfkFx06iyhbIoyMgtkE= +github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f/go.mod h1:aS446i8akEg0DAtNKTVYpNpLPMc0SzsZ0RtGhjl0uFM= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/gokrb5/v7 v7.2.5/go.mod h1:z6fE6twrvMN004M+KRTHnmtfpxsBIztP0PVsak0/4f8= +github.com/minio/hdfs/v3 v3.0.1/go.mod h1:6ALh9HsAwG9xAXdpdrZJcSY0vR6z3K+9XIz6Y9pQG/c= +github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= +github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA= +github.com/minio/minio v0.0.0-20200521180750-fade0562443f/go.mod h1:wymaytM/HELuwdz7BGZHmQ3XKq2SxPsLeGxyOCaCLiA= +github.com/minio/minio-go/v6 v6.0.53/go.mod h1:DIvC/IApeHX8q1BAMVCXSXwpmrmM+I+iBvhvztQorfI= +github.com/minio/minio-go/v6 v6.0.55-0.20200425081427-89eebdef2af0/go.mod h1:KQMM+/44DSlSGSQWSfRrAZ12FVMmpWNuX37i2AX0jfI= +github.com/minio/minio-go/v6 v6.0.55/go.mod h1:KQMM+/44DSlSGSQWSfRrAZ12FVMmpWNuX37i2AX0jfI= +github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61/go.mod h1:4trzEJ7N1nBTd5Tt7OCZT5SEin+WiAXpdJ/WgPkESA8= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= +github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.1 h1:cCBH2gTD2K0OtLlv/Y5H01VQCqmlDxz30kS5Y5bqfLA= +github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcloughlin/avo v0.0.0-20200303042253-6df701fe672f/go.mod h1:L0u9qfRMLNBO97u6pPukRp6ncoQz0Q25W69fvtht3vA= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/natefinch/atomic v0.0.0-20150920032501-a62ce929ffcc/go.mod h1:1rLVY/DWf3U6vSZgH16S7pymfrhK2lcUlXjgGglw/lY= +github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.7.2/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= +github.com/nats-io/go-nats-streaming v0.4.4/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server v1.4.1/go.mod h1:c8f/fHd2B6Hgms3LtCaI7y6pC4WD1f4SUxcCud5vhBc= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats-streaming-server v0.14.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/stan.go v0.4.5/go.mod h1:Ji7mK6gRZJSH1nc3ZJH6vi7zn/QnZhpR9Arm4iuzsUQ= +github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/openshift/api v0.0.0-20200521101457-60c476765272/go.mod h1:TkhafijfTiRi1Q3120/ZSE4oIWKQ4DGRh3byPywv4Mw= +github.com/openshift/api v0.0.0-20200526144822-34f54f12813a h1:riE/kCXnb051RWT/z+DytxKEZ3+JromVDl79rXAKyFY= +github.com/openshift/api v0.0.0-20200526144822-34f54f12813a/go.mod h1:l6TGeqJ92DrZBuWMNKcot1iZUHfbYSJyBWHGgg6Dn6s= +github.com/openshift/build-machinery-go v0.0.0-20200424080330-082bf86082cc/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= +github.com/openshift/client-go v0.0.0-20200521150516-05eb9880269c h1:l7CmbzzkyWl4Y6qHmy6m4FvbH4iLnIXGrXqOfE5IFNA= +github.com/openshift/client-go v0.0.0-20200521150516-05eb9880269c/go.mod h1:kCMeo6IE4o4qvnepM9lgHQ4j/ZFfvY/N/2G/jpJdwm4= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/pkg/sftp v1.11.0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= +github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= +github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/secure-io/sio-go v0.3.0/go.mod h1:D3KmXgKETffyYxBdFRN+Hpd2WzhzqS0EQwT3XWsAcBU= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/softlayer/softlayer-go v0.0.0-20190615201252-ba6e7f295217 h1:MFHQI+AYM6otrSP+l3dLhE2DjrSr5HXfV4mt4M6pjPs= +github.com/softlayer/softlayer-go v0.0.0-20190615201252-ba6e7f295217/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s= +github.com/tg123/go-htpasswd v1.0.0/go.mod h1:eQTgl67UrNKQvEPKrDLGBssjVwYQClFZjALVLhIv8C0= +github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= +github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vmware/govmomi v0.21.1-0.20191008161538-40aebf13ba45/go.mod h1:zbnFoBQ9GIjs2RVETy8CNEpb+L+Lwkjs3XZUL0B3/m0= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zalando/go-keyring v0.0.0-20200121091418-667557018717/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= +go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= +golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190914235951-31e00f45c22e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200115044656-831fdb1e1868/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200428211428-0c9eba77bc32/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200521155704-91d71f6c2f04 h1:LbW6ziLoA0vw8ZR4bmjKAzgEpunUBaEX1ia1Q1jEGC4= +golang.org/x/tools v0.0.0-20200521155704-91d71f6c2f04/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d h1:szSOL78iTCl0LF1AMjhSWJj8tIM0KixlUUnBtYXsmd8= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190513181449-d00d292a067c/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece h1:1YM0uhfumvoDu9sx8+RyWwTI63zoCQvI23IYFRlvte0= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20191215213626-7594ed38700f/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= +gopkg.in/ldap.v3 v3.0.3/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/olivere/elastic.v5 v5.0.80/go.mod h1:uhHoB4o3bvX5sorxBU29rPcmBQdV2Qfg0FBrx5D6pV0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.5 h1:nI5egYTGJakVyOryqLs1cQO5dO0ksin5XXs2pspk75k= +honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= +k8s.io/api v0.19.0 h1:XyrFIJqTYZJ2DU7FBE/bSPz7b1HvbVBuBf07oeo6eTc= +k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= +k8s.io/api v0.19.5 h1:p0MRzyhokJ9Kn5jcJAHNup0s+COMBPfn1mTasls6mMg= +k8s.io/api v0.19.5/go.mod h1:yGZReuNa0vj56op6eT+NLrXJne0R0u9ktexZ8jdJzpc= +k8s.io/apiextensions-apiserver v0.19.5 h1:LMlydaZH8UXo8Qi86IFXSQHdjb13uij7D6D+OubX5MU= +k8s.io/apiextensions-apiserver v0.19.5/go.mod h1:7jrSIYOYk22PnQv03LJrcSBS0WA0v1WsGJzd0Gv3BiQ= +k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ= +k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.19.5 h1:Yvz6dOE0WbVE+FXBEFqc9lSvo87VPtq6mCSsrtC95HI= +k8s.io/apimachinery v0.19.5/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= +k8s.io/apiserver v0.19.5/go.mod h1:CDH8gHbC/y1nhkkN9NTZ2Nr29BNA0OWCJDP0QcKwsuE= +k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= +k8s.io/client-go v0.19.0 h1:1+0E0zfWFIWeyRhQYWzimJOyAk2UT7TiARaLNwJCf7k= +k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= +k8s.io/client-go v0.19.5 h1:Y7LsFwgbm9+5oVXER04KNCSPhY6TblYRgG1DQdVq+ig= +k8s.io/client-go v0.19.5/go.mod h1:BSG3iuxI40Bs0nNDLS1JRa/7ReBQDHzf0x8nZZrK0fo= +k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.19.5/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.19.5/go.mod h1:5N/uv5A7fyr0d+t/b1HynXKkUVPEhc8ljkMaBJv4Tp8= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/main.go b/main.go new file mode 100644 index 0000000..cf2a242 --- /dev/null +++ b/main.go @@ -0,0 +1,30 @@ +// Copyright 2020 Kubestr Developers + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +//go:generate ./scripts/load_csi_provisioners.sh + +import ( + "github.com/kastenhq/kubestr/cmd" +) + +func main() { + _ = Execute() +} + +// Execute executes the main command +func Execute() error { + return cmd.Execute() +} diff --git a/pkg/common/common.go b/pkg/common/common.go new file mode 100644 index 0000000..cbd966c --- /dev/null +++ b/pkg/common/common.go @@ -0,0 +1,24 @@ +package common + +const ( + // VolSnapClassAlphaDriverKey describes alpha driver key + VolSnapClassAlphaDriverKey = "snapshotter" + // VolSnapClassBetaDriverKey describes beta driver key + VolSnapClassBetaDriverKey = "driver" + // VolSnapClassStableDriverKey describes the stable driver key + VolSnapClassStableDriverKey = "driver" + // DefaultPodImage the default pod image + DefaultPodImage = "ghcr.io/kastenhq/kubestr:latest" + // SnapGroupName describes the snapshot group name + SnapGroupName = "snapshot.storage.k8s.io" + // VolumeSnapshotClassResourcePlural describes volume snapshot classses + VolumeSnapshotClassResourcePlural = "volumesnapshotclasses" + // VolumeSnapshotResourcePlural is "volumesnapshots" + VolumeSnapshotResourcePlural = "volumesnapshots" + // SnapshotAlphaVersion is the apiversion of the alpha relase + SnapshotAlphaVersion = "snapshot.storage.k8s.io/v1alpha1" + // SnapshotBetaVersion is the apiversion of the beta relase + SnapshotBetaVersion = "snapshot.storage.k8s.io/v1beta1" + // SnapshotStableVersion is the apiversion of the stable release + SnapshotStableVersion = "snapshot.storage.k8s.io/v1" +) diff --git a/pkg/csi/csi.go b/pkg/csi/csi.go new file mode 100644 index 0000000..4fbad04 --- /dev/null +++ b/pkg/csi/csi.go @@ -0,0 +1,11 @@ +package csi + +import ( + "context" + + "github.com/kastenhq/kubestr/pkg/csi/types" +) + +type CSI interface { + RunSnapshotRestore(ctx context.Context, args *types.CSISnapshotRestoreArgs) (*types.CSISnapshotRestoreResults, error) +} diff --git a/pkg/csi/mocks/mock_api_version_fetcher.go b/pkg/csi/mocks/mock_api_version_fetcher.go new file mode 100644 index 0000000..83e5482 --- /dev/null +++ b/pkg/csi/mocks/mock_api_version_fetcher.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: ApiVersionFetcher) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + reflect "reflect" +) + +// MockApiVersionFetcher is a mock of ApiVersionFetcher interface +type MockApiVersionFetcher struct { + ctrl *gomock.Controller + recorder *MockApiVersionFetcherMockRecorder +} + +// MockApiVersionFetcherMockRecorder is the mock recorder for MockApiVersionFetcher +type MockApiVersionFetcherMockRecorder struct { + mock *MockApiVersionFetcher +} + +// NewMockApiVersionFetcher creates a new mock instance +func NewMockApiVersionFetcher(ctrl *gomock.Controller) *MockApiVersionFetcher { + mock := &MockApiVersionFetcher{ctrl: ctrl} + mock.recorder = &MockApiVersionFetcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockApiVersionFetcher) EXPECT() *MockApiVersionFetcherMockRecorder { + return m.recorder +} + +// GetCSISnapshotGroupVersion mocks base method +func (m *MockApiVersionFetcher) GetCSISnapshotGroupVersion() (*v1.GroupVersionForDiscovery, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCSISnapshotGroupVersion") + ret0, _ := ret[0].(*v1.GroupVersionForDiscovery) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCSISnapshotGroupVersion indicates an expected call of GetCSISnapshotGroupVersion +func (mr *MockApiVersionFetcherMockRecorder) GetCSISnapshotGroupVersion() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCSISnapshotGroupVersion", reflect.TypeOf((*MockApiVersionFetcher)(nil).GetCSISnapshotGroupVersion)) +} diff --git a/pkg/csi/mocks/mock_application_creator.go b/pkg/csi/mocks/mock_application_creator.go new file mode 100644 index 0000000..aa8aa8a --- /dev/null +++ b/pkg/csi/mocks/mock_application_creator.go @@ -0,0 +1,80 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: ApplicationCreator) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + types "github.com/kastenhq/kubestr/pkg/csi/types" + v1 "k8s.io/api/core/v1" + reflect "reflect" +) + +// MockApplicationCreator is a mock of ApplicationCreator interface +type MockApplicationCreator struct { + ctrl *gomock.Controller + recorder *MockApplicationCreatorMockRecorder +} + +// MockApplicationCreatorMockRecorder is the mock recorder for MockApplicationCreator +type MockApplicationCreatorMockRecorder struct { + mock *MockApplicationCreator +} + +// NewMockApplicationCreator creates a new mock instance +func NewMockApplicationCreator(ctrl *gomock.Controller) *MockApplicationCreator { + mock := &MockApplicationCreator{ctrl: ctrl} + mock.recorder = &MockApplicationCreatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockApplicationCreator) EXPECT() *MockApplicationCreatorMockRecorder { + return m.recorder +} + +// CreatePVC mocks base method +func (m *MockApplicationCreator) CreatePVC(arg0 context.Context, arg1 *types.CreatePVCArgs) (*v1.PersistentVolumeClaim, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreatePVC", arg0, arg1) + ret0, _ := ret[0].(*v1.PersistentVolumeClaim) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreatePVC indicates an expected call of CreatePVC +func (mr *MockApplicationCreatorMockRecorder) CreatePVC(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePVC", reflect.TypeOf((*MockApplicationCreator)(nil).CreatePVC), arg0, arg1) +} + +// CreatePod mocks base method +func (m *MockApplicationCreator) CreatePod(arg0 context.Context, arg1 *types.CreatePodArgs) (*v1.Pod, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreatePod", arg0, arg1) + ret0, _ := ret[0].(*v1.Pod) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreatePod indicates an expected call of CreatePod +func (mr *MockApplicationCreatorMockRecorder) CreatePod(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePod", reflect.TypeOf((*MockApplicationCreator)(nil).CreatePod), arg0, arg1) +} + +// WaitForPodReady mocks base method +func (m *MockApplicationCreator) WaitForPodReady(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitForPodReady", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitForPodReady indicates an expected call of WaitForPodReady +func (mr *MockApplicationCreatorMockRecorder) WaitForPodReady(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForPodReady", reflect.TypeOf((*MockApplicationCreator)(nil).WaitForPodReady), arg0, arg1, arg2) +} diff --git a/pkg/csi/mocks/mock_argument_validator.go b/pkg/csi/mocks/mock_argument_validator.go new file mode 100644 index 0000000..41d40d4 --- /dev/null +++ b/pkg/csi/mocks/mock_argument_validator.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: ArgumentValidator) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/api/storage/v1" + v10 "k8s.io/apimachinery/pkg/apis/meta/v1" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + reflect "reflect" +) + +// MockArgumentValidator is a mock of ArgumentValidator interface +type MockArgumentValidator struct { + ctrl *gomock.Controller + recorder *MockArgumentValidatorMockRecorder +} + +// MockArgumentValidatorMockRecorder is the mock recorder for MockArgumentValidator +type MockArgumentValidatorMockRecorder struct { + mock *MockArgumentValidator +} + +// NewMockArgumentValidator creates a new mock instance +func NewMockArgumentValidator(ctrl *gomock.Controller) *MockArgumentValidator { + mock := &MockArgumentValidator{ctrl: ctrl} + mock.recorder = &MockArgumentValidatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockArgumentValidator) EXPECT() *MockArgumentValidatorMockRecorder { + return m.recorder +} + +// ValidateNamespace mocks base method +func (m *MockArgumentValidator) ValidateNamespace(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateNamespace", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateNamespace indicates an expected call of ValidateNamespace +func (mr *MockArgumentValidatorMockRecorder) ValidateNamespace(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateNamespace", reflect.TypeOf((*MockArgumentValidator)(nil).ValidateNamespace), arg0, arg1) +} + +// ValidateStorageClass mocks base method +func (m *MockArgumentValidator) ValidateStorageClass(arg0 context.Context, arg1 string) (*v1.StorageClass, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateStorageClass", arg0, arg1) + ret0, _ := ret[0].(*v1.StorageClass) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ValidateStorageClass indicates an expected call of ValidateStorageClass +func (mr *MockArgumentValidatorMockRecorder) ValidateStorageClass(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateStorageClass", reflect.TypeOf((*MockArgumentValidator)(nil).ValidateStorageClass), arg0, arg1) +} + +// ValidateVolumeSnapshotClass mocks base method +func (m *MockArgumentValidator) ValidateVolumeSnapshotClass(arg0 context.Context, arg1 string, arg2 *v10.GroupVersionForDiscovery) (*unstructured.Unstructured, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateVolumeSnapshotClass", arg0, arg1, arg2) + ret0, _ := ret[0].(*unstructured.Unstructured) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ValidateVolumeSnapshotClass indicates an expected call of ValidateVolumeSnapshotClass +func (mr *MockArgumentValidatorMockRecorder) ValidateVolumeSnapshotClass(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateVolumeSnapshotClass", reflect.TypeOf((*MockArgumentValidator)(nil).ValidateVolumeSnapshotClass), arg0, arg1, arg2) +} diff --git a/pkg/csi/mocks/mock_cleaner.go b/pkg/csi/mocks/mock_cleaner.go new file mode 100644 index 0000000..3f73b50 --- /dev/null +++ b/pkg/csi/mocks/mock_cleaner.go @@ -0,0 +1,77 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: Cleaner) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + reflect "reflect" +) + +// MockCleaner is a mock of Cleaner interface +type MockCleaner struct { + ctrl *gomock.Controller + recorder *MockCleanerMockRecorder +} + +// MockCleanerMockRecorder is the mock recorder for MockCleaner +type MockCleanerMockRecorder struct { + mock *MockCleaner +} + +// NewMockCleaner creates a new mock instance +func NewMockCleaner(ctrl *gomock.Controller) *MockCleaner { + mock := &MockCleaner{ctrl: ctrl} + mock.recorder = &MockCleanerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockCleaner) EXPECT() *MockCleanerMockRecorder { + return m.recorder +} + +// DeletePVC mocks base method +func (m *MockCleaner) DeletePVC(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePVC", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeletePVC indicates an expected call of DeletePVC +func (mr *MockCleanerMockRecorder) DeletePVC(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePVC", reflect.TypeOf((*MockCleaner)(nil).DeletePVC), arg0, arg1, arg2) +} + +// DeletePod mocks base method +func (m *MockCleaner) DeletePod(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePod", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeletePod indicates an expected call of DeletePod +func (mr *MockCleanerMockRecorder) DeletePod(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePod", reflect.TypeOf((*MockCleaner)(nil).DeletePod), arg0, arg1, arg2) +} + +// DeleteSnapshot mocks base method +func (m *MockCleaner) DeleteSnapshot(arg0 context.Context, arg1, arg2 string, arg3 *v1.GroupVersionForDiscovery) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSnapshot", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSnapshot indicates an expected call of DeleteSnapshot +func (mr *MockCleanerMockRecorder) DeleteSnapshot(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSnapshot", reflect.TypeOf((*MockCleaner)(nil).DeleteSnapshot), arg0, arg1, arg2, arg3) +} diff --git a/pkg/csi/mocks/mock_data_validator.go b/pkg/csi/mocks/mock_data_validator.go new file mode 100644 index 0000000..50a750d --- /dev/null +++ b/pkg/csi/mocks/mock_data_validator.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: DataValidator) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockDataValidator is a mock of DataValidator interface +type MockDataValidator struct { + ctrl *gomock.Controller + recorder *MockDataValidatorMockRecorder +} + +// MockDataValidatorMockRecorder is the mock recorder for MockDataValidator +type MockDataValidatorMockRecorder struct { + mock *MockDataValidator +} + +// NewMockDataValidator creates a new mock instance +func NewMockDataValidator(ctrl *gomock.Controller) *MockDataValidator { + mock := &MockDataValidator{ctrl: ctrl} + mock.recorder = &MockDataValidatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockDataValidator) EXPECT() *MockDataValidatorMockRecorder { + return m.recorder +} + +// FetchPodData mocks base method +func (m *MockDataValidator) FetchPodData(arg0, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FetchPodData", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FetchPodData indicates an expected call of FetchPodData +func (mr *MockDataValidatorMockRecorder) FetchPodData(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchPodData", reflect.TypeOf((*MockDataValidator)(nil).FetchPodData), arg0, arg1) +} diff --git a/pkg/csi/mocks/mock_snapshot_creator.go b/pkg/csi/mocks/mock_snapshot_creator.go new file mode 100644 index 0000000..8c6c7ad --- /dev/null +++ b/pkg/csi/mocks/mock_snapshot_creator.go @@ -0,0 +1,82 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: SnapshotCreator) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + snapshot "github.com/kanisterio/kanister/pkg/kube/snapshot" + types "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + reflect "reflect" +) + +// MockSnapshotCreator is a mock of SnapshotCreator interface +type MockSnapshotCreator struct { + ctrl *gomock.Controller + recorder *MockSnapshotCreatorMockRecorder +} + +// MockSnapshotCreatorMockRecorder is the mock recorder for MockSnapshotCreator +type MockSnapshotCreatorMockRecorder struct { + mock *MockSnapshotCreator +} + +// NewMockSnapshotCreator creates a new mock instance +func NewMockSnapshotCreator(ctrl *gomock.Controller) *MockSnapshotCreator { + mock := &MockSnapshotCreator{ctrl: ctrl} + mock.recorder = &MockSnapshotCreatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSnapshotCreator) EXPECT() *MockSnapshotCreatorMockRecorder { + return m.recorder +} + +// CreateFromSourceCheck mocks base method +func (m *MockSnapshotCreator) CreateFromSourceCheck(arg0 context.Context, arg1 snapshot.Snapshotter, arg2 *types.CreateFromSourceCheckArgs, arg3 *v1.GroupVersionForDiscovery) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFromSourceCheck", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateFromSourceCheck indicates an expected call of CreateFromSourceCheck +func (mr *MockSnapshotCreatorMockRecorder) CreateFromSourceCheck(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFromSourceCheck", reflect.TypeOf((*MockSnapshotCreator)(nil).CreateFromSourceCheck), arg0, arg1, arg2, arg3) +} + +// CreateSnapshot mocks base method +func (m *MockSnapshotCreator) CreateSnapshot(arg0 context.Context, arg1 snapshot.Snapshotter, arg2 *types.CreateSnapshotArgs) (*snapv1.VolumeSnapshot, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSnapshot", arg0, arg1, arg2) + ret0, _ := ret[0].(*snapv1.VolumeSnapshot) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSnapshot indicates an expected call of CreateSnapshot +func (mr *MockSnapshotCreatorMockRecorder) CreateSnapshot(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSnapshot", reflect.TypeOf((*MockSnapshotCreator)(nil).CreateSnapshot), arg0, arg1, arg2) +} + +// NewSnapshotter mocks base method +func (m *MockSnapshotCreator) NewSnapshotter() (snapshot.Snapshotter, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewSnapshotter") + ret0, _ := ret[0].(snapshot.Snapshotter) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewSnapshotter indicates an expected call of NewSnapshotter +func (mr *MockSnapshotCreatorMockRecorder) NewSnapshotter() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewSnapshotter", reflect.TypeOf((*MockSnapshotCreator)(nil).NewSnapshotter)) +} diff --git a/pkg/csi/mocks/mock_snapshot_restore_stepper.go b/pkg/csi/mocks/mock_snapshot_restore_stepper.go new file mode 100644 index 0000000..9e04851 --- /dev/null +++ b/pkg/csi/mocks/mock_snapshot_restore_stepper.go @@ -0,0 +1,124 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kastenhq/kubestr/pkg/csi (interfaces: SnapshotRestoreStepper) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + types "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + v1 "k8s.io/api/core/v1" + reflect "reflect" +) + +// MockSnapshotRestoreStepper is a mock of SnapshotRestoreStepper interface +type MockSnapshotRestoreStepper struct { + ctrl *gomock.Controller + recorder *MockSnapshotRestoreStepperMockRecorder +} + +// MockSnapshotRestoreStepperMockRecorder is the mock recorder for MockSnapshotRestoreStepper +type MockSnapshotRestoreStepperMockRecorder struct { + mock *MockSnapshotRestoreStepper +} + +// NewMockSnapshotRestoreStepper creates a new mock instance +func NewMockSnapshotRestoreStepper(ctrl *gomock.Controller) *MockSnapshotRestoreStepper { + mock := &MockSnapshotRestoreStepper{ctrl: ctrl} + mock.recorder = &MockSnapshotRestoreStepperMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSnapshotRestoreStepper) EXPECT() *MockSnapshotRestoreStepperMockRecorder { + return m.recorder +} + +// Cleanup mocks base method +func (m *MockSnapshotRestoreStepper) Cleanup(arg0 context.Context, arg1 *types.CSISnapshotRestoreResults) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Cleanup", arg0, arg1) +} + +// Cleanup indicates an expected call of Cleanup +func (mr *MockSnapshotRestoreStepperMockRecorder) Cleanup(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).Cleanup), arg0, arg1) +} + +// CreateApplication mocks base method +func (m *MockSnapshotRestoreStepper) CreateApplication(arg0 context.Context, arg1 *types.CSISnapshotRestoreArgs, arg2 string) (*v1.Pod, *v1.PersistentVolumeClaim, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateApplication", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Pod) + ret1, _ := ret[1].(*v1.PersistentVolumeClaim) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateApplication indicates an expected call of CreateApplication +func (mr *MockSnapshotRestoreStepperMockRecorder) CreateApplication(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateApplication", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).CreateApplication), arg0, arg1, arg2) +} + +// RestoreApplication mocks base method +func (m *MockSnapshotRestoreStepper) RestoreApplication(arg0 context.Context, arg1 *types.CSISnapshotRestoreArgs, arg2 *snapv1.VolumeSnapshot) (*v1.Pod, *v1.PersistentVolumeClaim, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RestoreApplication", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Pod) + ret1, _ := ret[1].(*v1.PersistentVolumeClaim) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// RestoreApplication indicates an expected call of RestoreApplication +func (mr *MockSnapshotRestoreStepperMockRecorder) RestoreApplication(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RestoreApplication", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).RestoreApplication), arg0, arg1, arg2) +} + +// SnapshotApplication mocks base method +func (m *MockSnapshotRestoreStepper) SnapshotApplication(arg0 context.Context, arg1 *types.CSISnapshotRestoreArgs, arg2 *v1.PersistentVolumeClaim, arg3 string) (*snapv1.VolumeSnapshot, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SnapshotApplication", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*snapv1.VolumeSnapshot) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SnapshotApplication indicates an expected call of SnapshotApplication +func (mr *MockSnapshotRestoreStepperMockRecorder) SnapshotApplication(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SnapshotApplication", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).SnapshotApplication), arg0, arg1, arg2, arg3) +} + +// ValidateArgs mocks base method +func (m *MockSnapshotRestoreStepper) ValidateArgs(arg0 context.Context, arg1 *types.CSISnapshotRestoreArgs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateArgs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateArgs indicates an expected call of ValidateArgs +func (mr *MockSnapshotRestoreStepperMockRecorder) ValidateArgs(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateArgs", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).ValidateArgs), arg0, arg1) +} + +// ValidateData mocks base method +func (m *MockSnapshotRestoreStepper) ValidateData(arg0 context.Context, arg1 *v1.Pod, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateData", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateData indicates an expected call of ValidateData +func (mr *MockSnapshotRestoreStepperMockRecorder) ValidateData(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateData", reflect.TypeOf((*MockSnapshotRestoreStepper)(nil).ValidateData), arg0, arg1, arg2) +} diff --git a/pkg/csi/snapshot_restore.go b/pkg/csi/snapshot_restore.go new file mode 100644 index 0000000..0cea917 --- /dev/null +++ b/pkg/csi/snapshot_restore.go @@ -0,0 +1,649 @@ +package csi + +import ( + "context" + "fmt" + "time" + + kankube "github.com/kanisterio/kanister/pkg/kube" + kansnapshot "github.com/kanisterio/kanister/pkg/kube/snapshot" + "github.com/kastenhq/kubestr/pkg/common" + "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +const ( + originalPVCGenerateName = "kubestr-csi-original-pvc" + originalPodGenerateName = "kubestr-csi-original-pod" + clonedPVCGenerateName = "kubestr-csi-cloned-pvc" + clonedPodGenerateName = "kubestr-csi-cloned-pod" + createdByLabel = "created-by-kubestr-csi" + clonePrefix = "kubestr-clone-" + snapshotPrefix = "kubestr-snapshot-" +) + +type SnapshotRestoreRunner struct { + KubeCli kubernetes.Interface + DynCli dynamic.Interface + srSteps SnapshotRestoreStepper +} + +func (r *SnapshotRestoreRunner) RunSnapshotRestore(ctx context.Context, args *types.CSISnapshotRestoreArgs) (*types.CSISnapshotRestoreResults, error) { + r.srSteps = &snapshotRestoreSteps{ + validateOps: &validateOperations{ + kubeCli: r.KubeCli, + dynCli: r.DynCli, + }, + versionFetchOps: &apiVersionFetch{ + kubeCli: r.KubeCli, + }, + createAppOps: &applicationCreate{ + kubeCli: r.KubeCli, + }, + dataValidatorOps: &validateData{ + kubeCli: r.KubeCli, + }, + snapshotCreateOps: &snapshotCreate{ + kubeCli: r.KubeCli, + dynCli: r.DynCli, + }, + cleanerOps: &cleanse{ + kubeCli: r.KubeCli, + dynCli: r.DynCli, + }, + } + return r.RunSnapshotRestoreHelper(ctx, args) +} + +func (r *SnapshotRestoreRunner) RunSnapshotRestoreHelper(ctx context.Context, args *types.CSISnapshotRestoreArgs) (*types.CSISnapshotRestoreResults, error) { + results := &types.CSISnapshotRestoreResults{} + var err error + if r.KubeCli == nil || r.DynCli == nil { + return results, fmt.Errorf("cli uninitialized") + } + if err := r.srSteps.ValidateArgs(ctx, args); err != nil { + return results, errors.Wrap(err, "Failed to validate arguments.") + } + data := time.Now().Format("20060102150405") + + fmt.Println("Creating application") + results.OriginalPod, results.OriginalPVC, err = r.srSteps.CreateApplication(ctx, args, data) + + if err == nil { + if results.OriginalPod != nil && results.OriginalPVC != nil { + fmt.Printf(" -> Created pod (%s) and pvc (%s)\n", results.OriginalPod.Name, results.OriginalPVC.Name) + } + err = r.srSteps.ValidateData(ctx, results.OriginalPod, data) + } + + snapName := snapshotPrefix + data + if err == nil { + fmt.Println("Taking a snapshot") + results.Snapshot, err = r.srSteps.SnapshotApplication(ctx, args, results.OriginalPVC, snapName) + } + + if err == nil { + if results.Snapshot != nil { + fmt.Printf(" -> Created snapshot (%s)\n", results.Snapshot.Name) + } + fmt.Println("Restoring application") + results.ClonedPod, results.ClonedPVC, err = r.srSteps.RestoreApplication(ctx, args, results.Snapshot) + } + + if err == nil { + if results.ClonedPod != nil && results.ClonedPVC != nil { + fmt.Printf(" -> Restored pod (%s) and pvc (%s)\n", results.ClonedPod.Name, results.ClonedPVC.Name) + } + err = r.srSteps.ValidateData(ctx, results.ClonedPod, data) + } + + if args.Cleanup { + fmt.Println("Cleaning up resources") + r.srSteps.Cleanup(ctx, results) + } + + return results, err +} + +//go:generate mockgen -destination=mocks/mock_snapshot_restore_stepper.go -package=mocks . SnapshotRestoreStepper +type SnapshotRestoreStepper interface { + ValidateArgs(ctx context.Context, args *types.CSISnapshotRestoreArgs) error + CreateApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, data string) (*v1.Pod, *v1.PersistentVolumeClaim, error) + ValidateData(ctx context.Context, pod *v1.Pod, data string) error + SnapshotApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, pvc *v1.PersistentVolumeClaim, snapshotName string) (*snapv1.VolumeSnapshot, error) + RestoreApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, snapshot *snapv1.VolumeSnapshot) (*v1.Pod, *v1.PersistentVolumeClaim, error) + Cleanup(ctx context.Context, results *types.CSISnapshotRestoreResults) +} + +type snapshotRestoreSteps struct { + validateOps ArgumentValidator + versionFetchOps ApiVersionFetcher + createAppOps ApplicationCreator + dataValidatorOps DataValidator + snapshotCreateOps SnapshotCreator + cleanerOps Cleaner + SnapshotGroupVersion *metav1.GroupVersionForDiscovery +} + +func (s *snapshotRestoreSteps) ValidateArgs(ctx context.Context, args *types.CSISnapshotRestoreArgs) error { + if err := args.Validate(); err != nil { + return errors.Wrap(err, "Failed to validate input arguments") + } + if err := s.validateOps.ValidateNamespace(ctx, args.Namespace); err != nil { + return errors.Wrap(err, "Failed to validate Namespace") + } + sc, err := s.validateOps.ValidateStorageClass(ctx, args.StorageClass) + if err != nil { + return errors.Wrap(err, "Failed to validate Storageclass") + } + + groupVersion, err := s.versionFetchOps.GetCSISnapshotGroupVersion() + if err != nil { + return errors.Wrap(err, "Failed to fetch groupVersion") + } + s.SnapshotGroupVersion = groupVersion + + uVSC, err := s.validateOps.ValidateVolumeSnapshotClass(ctx, args.VolumeSnapshotClass, groupVersion) + if err != nil { + return errors.Wrap(err, "Failed to validate VolumeSnapshotClass") + } + + vscDriver := getDriverNameFromUVSC(*uVSC, groupVersion.GroupVersion) + if sc.Provisioner != vscDriver { + return fmt.Errorf("StorageClass provisioner (%s) and VolumeSnapshotClass driver (%s) are different.", sc.Provisioner, vscDriver) + } + return nil +} + +func (s *snapshotRestoreSteps) CreateApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, genString string) (*v1.Pod, *v1.PersistentVolumeClaim, error) { + pvcArgs := &types.CreatePVCArgs{ + GenerateName: originalPVCGenerateName, + StorageClass: args.StorageClass, + Namespace: args.Namespace, + } + pvc, err := s.createAppOps.CreatePVC(ctx, pvcArgs) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to create PVC") + } + podArgs := &types.CreatePodArgs{ + GenerateName: originalPodGenerateName, + PVCName: pvc.Name, + Namespace: args.Namespace, + Cmd: fmt.Sprintf("echo '%s' >> /data/out.txt; sync; tail -f /dev/null", genString), + RunAsUser: args.RunAsUser, + ContainerImage: args.ContainerImage, + } + pod, err := s.createAppOps.CreatePod(ctx, podArgs) + if err != nil { + return nil, pvc, errors.Wrap(err, "Failed to create POD") + } + if err = s.createAppOps.WaitForPodReady(ctx, args.Namespace, pod.Name); err != nil { + return pod, pvc, errors.Wrap(err, "Pod failed to become ready") + } + return pod, pvc, nil +} + +func (s *snapshotRestoreSteps) ValidateData(ctx context.Context, pod *v1.Pod, data string) error { + podData, err := s.dataValidatorOps.FetchPodData(pod.Name, pod.Namespace) + if err != nil { + return errors.Wrap(err, "Failed to fetch data from pod. Failure may be due to permissions issues. Try again with runAsUser=1000 option.") + } + if podData != data { + return fmt.Errorf("string didn't match (%s , %s)", podData, data) + } + return nil +} + +func (s *snapshotRestoreSteps) SnapshotApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, pvc *v1.PersistentVolumeClaim, snapshotName string) (*snapv1.VolumeSnapshot, error) { + snapshotter, err := s.snapshotCreateOps.NewSnapshotter() + if err != nil { + return nil, errors.Wrap(err, "Failed to load snapshotter") + } + createSnapshotArgs := &types.CreateSnapshotArgs{ + Namespace: args.Namespace, + PVCName: pvc.Name, + VolumeSnapshotClass: args.VolumeSnapshotClass, + SnapshotName: snapshotName, + } + snapshot, err := s.snapshotCreateOps.CreateSnapshot(ctx, snapshotter, createSnapshotArgs) + if err != nil { + return nil, errors.Wrap(err, "Failed to create Snapshot") + } + if !args.SkipCFSCheck { + cfsArgs := &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: args.VolumeSnapshotClass, + SnapshotName: snapshot.Name, + Namespace: args.Namespace, + } + if err = s.snapshotCreateOps.CreateFromSourceCheck(ctx, snapshotter, cfsArgs, s.SnapshotGroupVersion); err != nil { + return snapshot, errors.Wrap(err, "Failed to create duplicate snapshot from source. To skip check use '--skipcfs=true' option.") + } + } + return snapshot, nil +} + +func (s *snapshotRestoreSteps) RestoreApplication(ctx context.Context, args *types.CSISnapshotRestoreArgs, snapshot *snapv1.VolumeSnapshot) (*v1.Pod, *v1.PersistentVolumeClaim, error) { + snapshotAPIGroup := "snapshot.storage.k8s.io" + snapshotKind := "VolumeSnapshot" + dataSource := &v1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: snapshotKind, + Name: snapshot.Name, + } + pvcArgs := &types.CreatePVCArgs{ + GenerateName: clonedPVCGenerateName, + StorageClass: args.StorageClass, + Namespace: args.Namespace, + DataSource: dataSource, + RestoreSize: snapshot.Status.RestoreSize, + } + pvc, err := s.createAppOps.CreatePVC(ctx, pvcArgs) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to restore PVC") + } + podArgs := &types.CreatePodArgs{ + GenerateName: clonedPodGenerateName, + PVCName: pvc.Name, + Namespace: args.Namespace, + Cmd: "tail -f /dev/null", + RunAsUser: args.RunAsUser, + ContainerImage: args.ContainerImage, + } + pod, err := s.createAppOps.CreatePod(ctx, podArgs) + if err != nil { + return nil, pvc, errors.Wrap(err, "Failed to create restored Pod") + } + if err = s.createAppOps.WaitForPodReady(ctx, args.Namespace, pod.Name); err != nil { + return pod, pvc, errors.Wrap(err, "Pod failed to become ready") + } + return pod, pvc, nil +} + +func (s *snapshotRestoreSteps) Cleanup(ctx context.Context, results *types.CSISnapshotRestoreResults) { + if results == nil { + return + } + if results.OriginalPVC != nil { + err := s.cleanerOps.DeletePVC(ctx, results.OriginalPVC.Name, results.OriginalPVC.Namespace) + if err != nil { + fmt.Printf("Error deleteing PVC (%s) - (%v)\n", results.OriginalPVC.Name, err) + } + } + if results.OriginalPod != nil { + err := s.cleanerOps.DeletePod(ctx, results.OriginalPod.Name, results.OriginalPod.Namespace) + if err != nil { + fmt.Printf("Error deleteing Pod (%s) - (%v)\n", results.OriginalPod.Name, err) + } + } + if results.ClonedPVC != nil { + err := s.cleanerOps.DeletePVC(ctx, results.ClonedPVC.Name, results.ClonedPVC.Namespace) + if err != nil { + fmt.Printf("Error deleteing PVC (%s) - (%v)\n", results.ClonedPVC.Name, err) + } + } + if results.ClonedPod != nil { + err := s.cleanerOps.DeletePod(ctx, results.ClonedPod.Name, results.ClonedPod.Namespace) + if err != nil { + fmt.Printf("Error deleteing Pod (%s) - (%v)\n", results.ClonedPod.Name, err) + } + } + if results.Snapshot != nil { + err := s.cleanerOps.DeleteSnapshot(ctx, results.Snapshot.Name, results.Snapshot.Namespace, s.SnapshotGroupVersion) + if err != nil { + fmt.Printf("Error deleteing Snapshot (%s) - (%v)\n", results.Snapshot.Name, err) + } + } +} + +//go:generate mockgen -destination=mocks/mock_argument_validator.go -package=mocks . ArgumentValidator +type ArgumentValidator interface { + ValidateNamespace(ctx context.Context, namespace string) error + ValidateStorageClass(ctx context.Context, storageClass string) (*sv1.StorageClass, error) + ValidateVolumeSnapshotClass(ctx context.Context, volumeSnapshotClass string, groupVersion *metav1.GroupVersionForDiscovery) (*unstructured.Unstructured, error) +} + +type validateOperations struct { + kubeCli kubernetes.Interface + dynCli dynamic.Interface +} + +func (o *validateOperations) ValidateNamespace(ctx context.Context, namespace string) error { + if o.kubeCli == nil { + return fmt.Errorf("kubeCli not initialized") + } + _, err := o.kubeCli.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + return err +} + +func (o *validateOperations) ValidateStorageClass(ctx context.Context, storageClass string) (*sv1.StorageClass, error) { + if o.kubeCli == nil { + return nil, fmt.Errorf("kubeCli not initialized") + } + return o.kubeCli.StorageV1().StorageClasses().Get(ctx, storageClass, metav1.GetOptions{}) +} + +func (o *validateOperations) ValidateVolumeSnapshotClass(ctx context.Context, volumeSnapshotClass string, groupVersion *metav1.GroupVersionForDiscovery) (*unstructured.Unstructured, error) { + if o.dynCli == nil { + return nil, fmt.Errorf("dynCli not initialized") + } + VolSnapClassGVR := schema.GroupVersionResource{Group: common.SnapGroupName, Version: groupVersion.Version, Resource: common.VolumeSnapshotClassResourcePlural} + return o.dynCli.Resource(VolSnapClassGVR).Get(ctx, volumeSnapshotClass, metav1.GetOptions{}) +} + +//go:generate mockgen -destination=mocks/mock_application_creator.go -package=mocks . ApplicationCreator +type ApplicationCreator interface { + CreatePVC(ctx context.Context, args *types.CreatePVCArgs) (*v1.PersistentVolumeClaim, error) + CreatePod(ctx context.Context, args *types.CreatePodArgs) (*v1.Pod, error) + WaitForPodReady(ctx context.Context, namespace string, podName string) error +} + +type applicationCreate struct { + kubeCli kubernetes.Interface +} + +func (c *applicationCreate) CreatePVC(ctx context.Context, args *types.CreatePVCArgs) (*v1.PersistentVolumeClaim, error) { + if c.kubeCli == nil { + return nil, fmt.Errorf("kubeCli not initialized") + } + if err := args.Validate(); err != nil { + return nil, err + } + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: args.GenerateName, + Namespace: args.Namespace, + Labels: map[string]string{ + createdByLabel: "yes", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + StorageClassName: &args.StorageClass, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + + if args.DataSource != nil { + pvc.Spec.DataSource = args.DataSource + } + + if args.RestoreSize != nil && !args.RestoreSize.IsZero() { + pvc.Spec.Resources.Requests[v1.ResourceStorage] = *args.RestoreSize + } + + pvcRes, err := c.kubeCli.CoreV1().PersistentVolumeClaims(args.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return pvc, err + } + + return pvcRes, nil +} + +func (c *applicationCreate) CreatePod(ctx context.Context, args *types.CreatePodArgs) (*v1.Pod, error) { + if c.kubeCli == nil { + return nil, fmt.Errorf("kubeCli not initialized") + } + if err := args.Validate(); err != nil { + return nil, err + } + if args.ContainerImage == "" { + args.ContainerImage = common.DefaultPodImage + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: args.GenerateName, + Namespace: args.Namespace, + Labels: map[string]string{ + createdByLabel: "yes", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: args.GenerateName, + Image: args.ContainerImage, + Command: []string{"/bin/sh"}, + Args: []string{"-c", args.Cmd}, + VolumeMounts: []v1.VolumeMount{{ + Name: "persistent-storage", + MountPath: "/data", + }}, + }}, + Volumes: []v1.Volume{{ + Name: "persistent-storage", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: args.PVCName, + }, + }}, + }, + }, + } + + if args.RunAsUser > 0 { + pod.Spec.SecurityContext = &v1.PodSecurityContext{ + RunAsUser: &args.RunAsUser, + FSGroup: &args.RunAsUser, + } + } + + podRes, err := c.kubeCli.CoreV1().Pods(args.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return pod, err + } + return podRes, nil +} + +func (c *applicationCreate) WaitForPodReady(ctx context.Context, namespace string, podName string) error { + if c.kubeCli == nil { + return fmt.Errorf("kubeCli not initialized") + } + err := kankube.WaitForPodReady(ctx, c.kubeCli, namespace, podName) + return err +} + +//go:generate mockgen -destination=mocks/mock_snapshot_creator.go -package=mocks . SnapshotCreator +type SnapshotCreator interface { + NewSnapshotter() (kansnapshot.Snapshotter, error) + CreateSnapshot(ctx context.Context, snapshotter kansnapshot.Snapshotter, args *types.CreateSnapshotArgs) (*snapv1.VolumeSnapshot, error) + CreateFromSourceCheck(ctx context.Context, snapshotter kansnapshot.Snapshotter, args *types.CreateFromSourceCheckArgs, SnapshotGroupVersion *metav1.GroupVersionForDiscovery) error +} + +type snapshotCreate struct { + kubeCli kubernetes.Interface + dynCli dynamic.Interface +} + +func (c *snapshotCreate) NewSnapshotter() (kansnapshot.Snapshotter, error) { + if c.kubeCli == nil { + return nil, fmt.Errorf("kubeCli not initialized") + } + if c.dynCli == nil { + return nil, fmt.Errorf("dynCli not initialized") + } + return kansnapshot.NewSnapshotter(c.kubeCli, c.dynCli) +} + +func (c *snapshotCreate) CreateSnapshot(ctx context.Context, snapshotter kansnapshot.Snapshotter, args *types.CreateSnapshotArgs) (*snapv1.VolumeSnapshot, error) { + if snapshotter == nil || args == nil { + return nil, fmt.Errorf("snapshotter or args are empty") + } + if err := args.Validate(); err != nil { + return nil, err + } + err := snapshotter.Create(ctx, args.SnapshotName, args.Namespace, args.PVCName, &args.VolumeSnapshotClass, true) + if err != nil { + return nil, errors.Wrapf(err, "CSI Driver failed to create snapshot for PVC (%s) in Namspace (%s)", args.PVCName, args.Namespace) + } + snap, err := snapshotter.Get(ctx, args.SnapshotName, args.Namespace) + if err != nil { + return nil, errors.Wrapf(err, "Failed to get CSI snapshot (%s) in Namespace (%s)", args.SnapshotName, args.Namespace) + } + return snap, nil +} + +func (c *snapshotCreate) CreateFromSourceCheck(ctx context.Context, snapshotter kansnapshot.Snapshotter, args *types.CreateFromSourceCheckArgs, SnapshotGroupVersion *metav1.GroupVersionForDiscovery) error { + if c.dynCli == nil { + return fmt.Errorf("dynCli not initialized") + } + if SnapshotGroupVersion == nil || SnapshotGroupVersion.Version == "" { + return fmt.Errorf("snapshot group version not provided") + } + if snapshotter == nil || args == nil { + return fmt.Errorf("snapshotter or args are nil") + } + if err := args.Validate(); err != nil { + return err + } + targetSnapClassName := clonePrefix + args.VolumeSnapshotClass + err := snapshotter.CloneVolumeSnapshotClass(args.VolumeSnapshotClass, targetSnapClassName, kansnapshot.DeletionPolicyRetain, nil) + if err != nil { + return errors.Wrapf(err, "Failed to create a VolumeSnapshotClass to use to restore the snapshot") + } + defer func() { + VolSnapClassGVR := schema.GroupVersionResource{Group: common.SnapGroupName, Version: SnapshotGroupVersion.Version, Resource: common.VolumeSnapshotClassResourcePlural} + err := c.dynCli.Resource(VolSnapClassGVR).Delete(ctx, targetSnapClassName, metav1.DeleteOptions{}) + if err != nil { + fmt.Printf("Delete VSC Error (%s) - (%v)\n", targetSnapClassName, err) + } + }() + + snapSrc, err := snapshotter.GetSource(ctx, args.SnapshotName, args.Namespace) + if err != nil { + return errors.Wrapf(err, "Failed to get source snapshot source (%s)", args.SnapshotName) + } + snapshotCFSCloneName := clonePrefix + args.SnapshotName + // test the CreateFromSource API + defer func() { + _, _ = snapshotter.Delete(context.Background(), snapshotCFSCloneName, args.Namespace) + }() + src := &kansnapshot.Source{ + Handle: snapSrc.Handle, + Driver: snapSrc.Driver, + VolumeSnapshotClassName: targetSnapClassName, + } + err = snapshotter.CreateFromSource(ctx, src, snapshotCFSCloneName, args.Namespace, true) + if err != nil { + return errors.Wrapf(err, "Failed to clone snapshot from source (%s)", snapshotCFSCloneName) + } + return nil +} + +//go:generate mockgen -destination=mocks/mock_cleaner.go -package=mocks . Cleaner +type Cleaner interface { + DeletePVC(ctx context.Context, pvcName string, namespace string) error + DeletePod(ctx context.Context, podName string, namespace string) error + DeleteSnapshot(ctx context.Context, snapshotName string, namespace string, SnapshotGroupVersion *metav1.GroupVersionForDiscovery) error +} + +type cleanse struct { + kubeCli kubernetes.Interface + dynCli dynamic.Interface +} + +func (c *cleanse) DeletePVC(ctx context.Context, pvcName string, namespace string) error { + if c.kubeCli == nil { + return fmt.Errorf("kubeCli not initialized") + } + return c.kubeCli.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) +} + +func (c *cleanse) DeletePod(ctx context.Context, podName string, namespace string) error { + if c.kubeCli == nil { + return fmt.Errorf("kubeCli not initialized") + } + return c.kubeCli.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) +} + +func (c *cleanse) DeleteSnapshot(ctx context.Context, snapshotName string, namespace string, SnapshotGroupVersion *metav1.GroupVersionForDiscovery) error { + if c.dynCli == nil { + return fmt.Errorf("dynCli not initialized") + } + if SnapshotGroupVersion == nil || SnapshotGroupVersion.Version == "" { + return fmt.Errorf("snapshot group version not provided") + } + VolSnapGVR := schema.GroupVersionResource{Group: common.SnapGroupName, Version: SnapshotGroupVersion.Version, Resource: common.VolumeSnapshotResourcePlural} + return c.dynCli.Resource(VolSnapGVR).Namespace(namespace).Delete(ctx, snapshotName, metav1.DeleteOptions{}) +} + +//go:generate mockgen -destination=mocks/mock_api_version_fetcher.go -package=mocks . ApiVersionFetcher +type ApiVersionFetcher interface { + GetCSISnapshotGroupVersion() (*metav1.GroupVersionForDiscovery, error) +} + +type apiVersionFetch struct { + kubeCli kubernetes.Interface +} + +func (p *apiVersionFetch) GetCSISnapshotGroupVersion() (*metav1.GroupVersionForDiscovery, error) { + if p.kubeCli == nil { + return nil, fmt.Errorf("kubeCli not initialized") + } + groups, _, err := p.kubeCli.Discovery().ServerGroupsAndResources() + if err != nil { + return nil, err + } + for _, group := range groups { + if group.Name == common.SnapGroupName { + return &group.PreferredVersion, nil + } + } + return nil, fmt.Errorf("Snapshot API group not found") +} + +//go:generate mockgen -destination=mocks/mock_data_validator.go -package=mocks . DataValidator +type DataValidator interface { + FetchPodData(podName string, podNamespace string) (string, error) +} + +type validateData struct { + kubeCli kubernetes.Interface +} + +func (p *validateData) FetchPodData(podName string, podNamespace string) (string, error) { + if p.kubeCli == nil { + return "", fmt.Errorf("kubeCli not initialized") + } + stdout, _, err := kankube.Exec(p.kubeCli, podNamespace, podName, "", []string{"sh", "-c", "cat /data/out.txt"}, nil) + return stdout, err +} + +func getDriverNameFromUVSC(vsc unstructured.Unstructured, version string) string { + var driverName interface{} + var ok bool + switch version { + case common.SnapshotAlphaVersion: + driverName, ok = vsc.Object[common.VolSnapClassAlphaDriverKey] + if !ok { + return "" + } + case common.SnapshotBetaVersion: + driverName, ok = vsc.Object[common.VolSnapClassBetaDriverKey] + if !ok { + return "" + } + case common.SnapshotStableVersion: + driverName, ok = vsc.Object[common.VolSnapClassStableDriverKey] + if !ok { + return "" + } + } + driver, ok := driverName.(string) + if !ok { + return "" + } + return driver +} diff --git a/pkg/csi/snapshot_restore_ops_test.go b/pkg/csi/snapshot_restore_ops_test.go new file mode 100644 index 0000000..275b3d3 --- /dev/null +++ b/pkg/csi/snapshot_restore_ops_test.go @@ -0,0 +1,1158 @@ +package csi + +import ( + "context" + "errors" + "fmt" + + kansnapshot "github.com/kanisterio/kanister/pkg/kube/snapshot" + "github.com/kanisterio/kanister/pkg/kube/snapshot/apis/v1alpha1" + "github.com/kanisterio/kanister/pkg/kube/snapshot/apis/v1beta1" + "github.com/kastenhq/kubestr/pkg/common" + "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + . "gopkg.in/check.v1" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + discoveryfake "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/dynamic" + fakedynamic "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func (s *CSITestSuite) TestGetDriverNameFromUVSC(c *C) { + + for _, tc := range []struct { + vsc unstructured.Unstructured + version string + expOut string + }{ + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassAlphaDriverKey: "p2", + }, + }, + version: common.SnapshotAlphaVersion, + expOut: "p2", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + version: common.SnapshotAlphaVersion, + expOut: "", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassBetaDriverKey: "p2", + }, + }, + version: common.SnapshotBetaVersion, + expOut: "p2", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + version: common.SnapshotBetaVersion, + expOut: "", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassBetaDriverKey: map[string]string{}, + }, + }, + version: common.SnapshotBetaVersion, + expOut: "", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassStableDriverKey: "p2", + }, + }, + version: common.SnapshotStableVersion, + expOut: "p2", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + version: common.SnapshotStableVersion, + expOut: "", + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassStableDriverKey: map[string]string{}, + }, + }, + version: common.SnapshotStableVersion, + expOut: "", + }, + } { + driverName := getDriverNameFromUVSC(tc.vsc, tc.version) + c.Assert(driverName, Equals, tc.expOut) + } + +} + +func (s *CSITestSuite) TestGetCSISnapshotGroupVersion(c *C) { + for _, tc := range []struct { + cli kubernetes.Interface + resources []*metav1.APIResourceList + errChecker Checker + gvChecker Checker + }{ + { + cli: fake.NewSimpleClientset(), + resources: []*metav1.APIResourceList{ + { + GroupVersion: "/////", + }, + }, + errChecker: NotNil, + gvChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + resources: []*metav1.APIResourceList{ + { + GroupVersion: "snapshot.storage.k8s.io/v1alpha1", + }, + }, + errChecker: IsNil, + gvChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + resources: []*metav1.APIResourceList{ + { + GroupVersion: "snapshot.storage.k8s.io/v1beta1", + }, + }, + errChecker: IsNil, + gvChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + resources: []*metav1.APIResourceList{ + { + GroupVersion: "snapshot.storage.k8s.io/v1", + }, + }, + errChecker: IsNil, + gvChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + resources: []*metav1.APIResourceList{ + { + GroupVersion: "notrbac.authorization.k8s.io/v1", + }, + }, + errChecker: NotNil, + gvChecker: IsNil, + }, + { + cli: nil, + resources: nil, + errChecker: NotNil, + gvChecker: IsNil, + }, + } { + cli := tc.cli + if cli != nil { + cli.Discovery().(*discoveryfake.FakeDiscovery).Resources = tc.resources + } + p := &apiVersionFetch{kubeCli: cli} + gv, err := p.GetCSISnapshotGroupVersion() + c.Check(err, tc.errChecker) + c.Check(gv, tc.gvChecker) + } +} + +func (s *CSITestSuite) TestValidateNamespace(c *C) { + ctx := context.Background() + ops := &validateOperations{ + kubeCli: fake.NewSimpleClientset(), + } + err := ops.ValidateNamespace(ctx, "ns") + c.Check(err, NotNil) + + ops = &validateOperations{ + kubeCli: fake.NewSimpleClientset(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ns", + }, + }), + } + err = ops.ValidateNamespace(ctx, "ns") + c.Check(err, IsNil) + + ops = &validateOperations{ + kubeCli: nil, + } + err = ops.ValidateNamespace(ctx, "ns") + c.Check(err, NotNil) +} + +func (s *CSITestSuite) TestValidateStorageClass(c *C) { + ctx := context.Background() + ops := &validateOperations{ + kubeCli: fake.NewSimpleClientset(), + } + sc, err := ops.ValidateStorageClass(ctx, "sc") + c.Check(err, NotNil) + c.Check(sc, IsNil) + + ops = &validateOperations{ + kubeCli: fake.NewSimpleClientset(&sv1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sc", + }, + }), + } + sc, err = ops.ValidateStorageClass(ctx, "sc") + c.Check(err, IsNil) + c.Check(sc, NotNil) + + ops = &validateOperations{ + kubeCli: nil, + } + sc, err = ops.ValidateStorageClass(ctx, "sc") + c.Check(err, NotNil) + c.Check(sc, IsNil) +} + +func (s *CSITestSuite) TestValidateVolumeSnapshotClass(c *C) { + ctx := context.Background() + for _, tc := range []struct { + ops *validateOperations + groupVersion string + version string + errChecker Checker + uVCSChecker Checker + }{ + { + ops: &validateOperations{ + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + }, + groupVersion: common.SnapshotAlphaVersion, + errChecker: NotNil, + uVCSChecker: IsNil, + }, + { + ops: &validateOperations{ + dynCli: fakedynamic.NewSimpleDynamicClient( + runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1alpha1.GroupName, v1alpha1.Version), + "kind": "VolumeSnapshotClass", + "metadata": map[string]interface{}{ + "name": "vsc", + }, + "snapshotter": "somesnapshotter", + "deletionPolicy": "Delete", + }, + }, + ), + }, + version: v1alpha1.Version, + errChecker: IsNil, + uVCSChecker: NotNil, + }, + { + ops: &validateOperations{ + dynCli: nil, + }, + version: v1alpha1.Version, + errChecker: NotNil, + uVCSChecker: IsNil, + }, + { + ops: &validateOperations{ + dynCli: fakedynamic.NewSimpleDynamicClient( + runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1beta1.GroupName, v1beta1.Version), + "kind": "VolumeSnapshotClass", + "metadata": map[string]interface{}{ + "name": "vsc", + }, + "driver": "somesnapshotter", + "deletionPolicy": "Delete", + }, + }, + ), + }, + version: v1beta1.Version, + errChecker: IsNil, + uVCSChecker: NotNil, + }, + { + ops: &validateOperations{ + dynCli: fakedynamic.NewSimpleDynamicClient( + runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", kansnapshot.GroupName, kansnapshot.Version), + "kind": "VolumeSnapshotClass", + "metadata": map[string]interface{}{ + "name": "vsc", + }, + "driver": "somesnapshotter", + "deletionPolicy": "Delete", + }, + }, + ), + }, + groupVersion: common.SnapshotStableVersion, + version: kansnapshot.Version, + errChecker: IsNil, + uVCSChecker: NotNil, + }, + } { + uVSC, err := tc.ops.ValidateVolumeSnapshotClass(ctx, "vsc", &metav1.GroupVersionForDiscovery{GroupVersion: tc.groupVersion, Version: tc.version}) + c.Check(err, tc.errChecker) + c.Check(uVSC, tc.uVCSChecker) + } +} + +func (s *CSITestSuite) TestCreatePVC(c *C) { + ctx := context.Background() + resourceQuantity := resource.MustParse("1Gi") + for _, tc := range []struct { + cli kubernetes.Interface + args *types.CreatePVCArgs + failCreates bool + errChecker Checker + pvcChecker Checker + }{ + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "genName", + StorageClass: "sc", + Namespace: "ns", + DataSource: &v1.TypedLocalObjectReference{ + Name: "ds", + }, + RestoreSize: &resourceQuantity, + }, + errChecker: IsNil, + pvcChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "genName", + StorageClass: "sc", + Namespace: "ns", + DataSource: &v1.TypedLocalObjectReference{ + Name: "ds", + }, + }, + errChecker: IsNil, + pvcChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "genName", + StorageClass: "sc", + Namespace: "ns", + }, + errChecker: IsNil, + pvcChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "genName", + StorageClass: "sc", + Namespace: "ns", + }, + failCreates: true, + errChecker: NotNil, + pvcChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "", + StorageClass: "sc", + Namespace: "ns", + }, + errChecker: NotNil, + pvcChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "something", + StorageClass: "", + Namespace: "ns", + }, + errChecker: NotNil, + pvcChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePVCArgs{ + GenerateName: "Something", + StorageClass: "sc", + Namespace: "", + }, + errChecker: NotNil, + pvcChecker: IsNil, + }, + { + cli: nil, + args: &types.CreatePVCArgs{}, + errChecker: NotNil, + pvcChecker: IsNil, + }, + } { + creator := &applicationCreate{kubeCli: tc.cli} + if tc.failCreates { + creator.kubeCli.(*fake.Clientset).Fake.PrependReactor("create", "persistentvolumeclaims", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("Error creating object") + }) + } + pvc, err := creator.CreatePVC(ctx, tc.args) + c.Check(pvc, tc.pvcChecker) + c.Check(err, tc.errChecker) + if pvc != nil && err == nil { + _, ok := pvc.Labels[createdByLabel] + c.Assert(ok, Equals, true) + c.Assert(pvc.GenerateName, Equals, tc.args.GenerateName) + c.Assert(pvc.Namespace, Equals, tc.args.Namespace) + c.Assert(pvc.Spec.AccessModes, DeepEquals, []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}) + c.Assert(*pvc.Spec.StorageClassName, Equals, tc.args.StorageClass) + c.Assert(pvc.Spec.DataSource, DeepEquals, tc.args.DataSource) + if tc.args.RestoreSize != nil { + c.Assert(pvc.Spec.Resources, DeepEquals, v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: *tc.args.RestoreSize, + }, + }) + } else { + c.Assert(pvc.Spec.Resources, DeepEquals, v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }) + } + } + } +} + +func (s *CSITestSuite) TestCreatePod(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + args *types.CreatePodArgs + failCreates bool + errChecker Checker + podChecker Checker + }{ + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "pvcname", + Namespace: "ns", + Cmd: "somecommand", + RunAsUser: 1000, + ContainerImage: "containerimage", + }, + errChecker: IsNil, + podChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "pvcname", + Namespace: "ns", + Cmd: "somecommand", + }, + errChecker: IsNil, + podChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "pvcname", + Namespace: "ns", + Cmd: "somecommand", + }, + failCreates: true, + errChecker: NotNil, + podChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "", + PVCName: "pvcname", + Namespace: "ns", + Cmd: "somecommand", + }, + errChecker: NotNil, + podChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "", + Namespace: "ns", + Cmd: "somecommand", + }, + errChecker: NotNil, + podChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "pvcname", + Namespace: "", + Cmd: "somecommand", + }, + errChecker: NotNil, + podChecker: IsNil, + }, + { + cli: fake.NewSimpleClientset(), + args: &types.CreatePodArgs{ + GenerateName: "name", + PVCName: "pvcname", + Namespace: "ns", + Cmd: "", + }, + errChecker: NotNil, + podChecker: IsNil, + }, + { + cli: nil, + args: &types.CreatePodArgs{}, + errChecker: NotNil, + podChecker: IsNil, + }, + } { + creator := &applicationCreate{kubeCli: tc.cli} + if tc.failCreates { + creator.kubeCli.(*fake.Clientset).Fake.PrependReactor("create", "pods", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("Error creating object") + }) + } + pod, err := creator.CreatePod(ctx, tc.args) + c.Check(pod, tc.podChecker) + c.Check(err, tc.errChecker) + if pod != nil && err == nil { + _, ok := pod.Labels[createdByLabel] + c.Assert(ok, Equals, true) + c.Assert(pod.GenerateName, Equals, tc.args.GenerateName) + c.Assert(pod.Namespace, Equals, tc.args.Namespace) + c.Assert(len(pod.Spec.Containers), Equals, 1) + c.Assert(pod.Spec.Containers[0].Name, Equals, tc.args.GenerateName) + c.Assert(pod.Spec.Containers[0].Command, DeepEquals, []string{"/bin/sh"}) + c.Assert(pod.Spec.Containers[0].Args, DeepEquals, []string{"-c", tc.args.Cmd}) + c.Assert(pod.Spec.Containers[0].VolumeMounts, DeepEquals, []v1.VolumeMount{{ + Name: "persistent-storage", + MountPath: "/data", + }}) + c.Assert(pod.Spec.Volumes, DeepEquals, []v1.Volume{{ + Name: "persistent-storage", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: tc.args.PVCName, + }, + }}, + }) + if tc.args.ContainerImage == "" { + c.Assert(pod.Spec.Containers[0].Image, Equals, common.DefaultPodImage) + } else { + c.Assert(pod.Spec.Containers[0].Image, Equals, tc.args.ContainerImage) + } + if tc.args.RunAsUser > 0 { + c.Assert(pod.Spec.SecurityContext, DeepEquals, &v1.PodSecurityContext{ + RunAsUser: &tc.args.RunAsUser, + FSGroup: &tc.args.RunAsUser, + }) + } else { + c.Check(pod.Spec.SecurityContext, IsNil) + } + + } + } +} + +func (s *CSITestSuite) TestCreateSnapshot(c *C) { + ctx := context.Background() + for _, tc := range []struct { + snapshotter kansnapshot.Snapshotter + args *types.CreateSnapshotArgs + snapChecker Checker + errChecker Checker + }{ + { + snapshotter: &fakeSnapshotter{ + getSnap: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "createdName", + }, + }, + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }, + snapChecker: NotNil, + errChecker: IsNil, + }, + { + snapshotter: &fakeSnapshotter{ + getErr: fmt.Errorf("get Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{ + createErr: fmt.Errorf("create Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{ + createErr: fmt.Errorf("create Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "", + PVCName: "pvc", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{ + createErr: fmt.Errorf("create Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{ + createErr: fmt.Errorf("create Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc", + VolumeSnapshotClass: "", + SnapshotName: "snap1", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{ + createErr: fmt.Errorf("create Error"), + }, + args: &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc", + VolumeSnapshotClass: "vsc", + SnapshotName: "", + }, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapshotter: &fakeSnapshotter{}, + snapChecker: IsNil, + errChecker: NotNil, + }, + { + snapChecker: IsNil, + errChecker: NotNil, + }, + } { + snapCreator := &snapshotCreate{} + snapshot, err := snapCreator.CreateSnapshot(ctx, tc.snapshotter, tc.args) + c.Check(snapshot, tc.snapChecker) + c.Check(err, tc.errChecker) + } +} + +func (s *CSITestSuite) TestCreateFromSourceCheck(c *C) { + ctx := context.Background() + gv := &metav1.GroupVersionForDiscovery{Version: v1alpha1.Version} + for _, tc := range []struct { + dyncli dynamic.Interface + snapshotter kansnapshot.Snapshotter + args *types.CreateFromSourceCheckArgs + groupVersion *metav1.GroupVersionForDiscovery + errChecker Checker + }{ + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{ + gsSrc: &kansnapshot.Source{ + Handle: "handle", + Driver: "driver", + }, + }, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "snapshot", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: IsNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{ + gsSrc: &kansnapshot.Source{ + Handle: "handle", + Driver: "driver", + }, + cfsErr: fmt.Errorf("cfs error"), + }, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "snapshot", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{ + gsErr: fmt.Errorf("gs error"), + }, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "snapshot", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{ + cvsErr: fmt.Errorf("cvs error"), + }, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "snapshot", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{}, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "", + SnapshotName: "snapshot", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{}, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "", + Namespace: "ns", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{}, + args: &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "snapshot", + Namespace: "", + }, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotter: &fakeSnapshotter{}, + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + groupVersion: gv, + errChecker: NotNil, + }, + { + dyncli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + groupVersion: nil, + errChecker: NotNil, + }, + { + dyncli: nil, + errChecker: NotNil, + }, + } { + snapCreator := &snapshotCreate{ + dynCli: tc.dyncli, + } + err := snapCreator.CreateFromSourceCheck(ctx, tc.snapshotter, tc.args, tc.groupVersion) + c.Check(err, tc.errChecker) + } +} + +type fakeSnapshotter struct { + name string + + createErr error + + getSnap *snapv1.VolumeSnapshot + getErr error + + cvsErr error + + gsSrc *kansnapshot.Source + gsErr error + + cfsErr error +} + +func (f *fakeSnapshotter) GetVolumeSnapshotClass(annotationKey, annotationValue, storageClassName string) (string, error) { + return "", nil +} +func (f *fakeSnapshotter) CloneVolumeSnapshotClass(sourceClassName, targetClassName, newDeletionPolicy string, excludeAnnotations []string) error { + return f.cvsErr +} +func (f *fakeSnapshotter) Create(ctx context.Context, name, namespace, pvcName string, snapshotClass *string, waitForReady bool) error { + return f.createErr +} +func (f *fakeSnapshotter) Get(ctx context.Context, name, namespace string) (*snapv1.VolumeSnapshot, error) { + return f.getSnap, f.getErr +} +func (f *fakeSnapshotter) Delete(ctx context.Context, name, namespace string) (*snapv1.VolumeSnapshot, error) { + return nil, nil +} +func (f *fakeSnapshotter) DeleteContent(ctx context.Context, name string) error { return nil } +func (f *fakeSnapshotter) Clone(ctx context.Context, name, namespace, cloneName, cloneNamespace string, waitForReady bool) error { + return nil +} +func (f *fakeSnapshotter) GetSource(ctx context.Context, snapshotName, namespace string) (*kansnapshot.Source, error) { + return f.gsSrc, f.gsErr +} +func (f *fakeSnapshotter) CreateFromSource(ctx context.Context, source *kansnapshot.Source, snapshotName, namespace string, waitForReady bool) error { + return f.cfsErr +} +func (f *fakeSnapshotter) CreateContentFromSource(ctx context.Context, source *kansnapshot.Source, contentName, snapshotName, namespace, deletionPolicy string) error { + return nil +} +func (f *fakeSnapshotter) WaitOnReadyToUse(ctx context.Context, snapshotName, namespace string) error { + return nil +} + +func (s *CSITestSuite) TestDeletePVC(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + pvcName string + namespace string + errChecker Checker + }{ + { + cli: fake.NewSimpleClientset(), + pvcName: "pvc", + namespace: "ns", + errChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc", + Namespace: "notns", + }, + }), + pvcName: "pvc", + namespace: "ns", + errChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc", + Namespace: "ns", + }, + }), + pvcName: "pvc", + namespace: "ns", + errChecker: IsNil, + }, + { + cli: nil, + pvcName: "pvc", + namespace: "ns", + errChecker: NotNil, + }, + } { + cleaner := &cleanse{ + kubeCli: tc.cli, + } + err := cleaner.DeletePVC(ctx, tc.pvcName, tc.namespace) + c.Check(err, tc.errChecker) + } +} + +func (s *CSITestSuite) TestDeletePod(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + podName string + namespace string + errChecker Checker + }{ + { + cli: fake.NewSimpleClientset(), + podName: "pod", + namespace: "ns", + errChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "notns", + }, + }), + podName: "pod", + namespace: "ns", + errChecker: NotNil, + }, + { + cli: fake.NewSimpleClientset(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "ns", + }, + }), + podName: "pod", + namespace: "ns", + errChecker: IsNil, + }, + { + cli: nil, + podName: "pod", + namespace: "ns", + errChecker: NotNil, + }, + } { + cleaner := &cleanse{ + kubeCli: tc.cli, + } + err := cleaner.DeletePod(ctx, tc.podName, tc.namespace) + c.Check(err, tc.errChecker) + } +} + +func (s *CSITestSuite) TestDeleteSnapshot(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli dynamic.Interface + snapshotName string + namespace string + groupVersion *metav1.GroupVersionForDiscovery + errChecker Checker + }{ + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotName: "snap1", + namespace: "ns", + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1alpha1.Version, + }, + errChecker: NotNil, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1alpha1.GroupName, v1alpha1.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "notns", + }, + }, + }), + snapshotName: "pod", + namespace: "ns", + errChecker: NotNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1alpha1.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1alpha1.GroupName, v1alpha1.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "ns", + }, + }, + }), + snapshotName: "snap1", + namespace: "ns", + errChecker: IsNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1alpha1.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1beta1.GroupName, v1beta1.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "ns", + }, + }, + }), + snapshotName: "snap1", + namespace: "ns", + errChecker: NotNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1alpha1.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1beta1.GroupName, v1beta1.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "ns", + }, + }, + }), + snapshotName: "snap1", + namespace: "ns", + errChecker: IsNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1beta1.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", kansnapshot.GroupName, kansnapshot.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "ns", + }, + }, + }), + snapshotName: "snap1", + namespace: "ns", + errChecker: NotNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: v1alpha1.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", kansnapshot.GroupName, kansnapshot.Version), + "kind": "VolumeSnapshot", + "metadata": map[string]interface{}{ + "name": "snap1", + "namespace": "ns", + }, + }, + }), + snapshotName: "snap1", + namespace: "ns", + errChecker: IsNil, + groupVersion: &metav1.GroupVersionForDiscovery{ + Version: kansnapshot.Version, + }, + }, + { + cli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + snapshotName: "pod", + namespace: "ns", + errChecker: NotNil, + }, + { + cli: nil, + snapshotName: "pod", + namespace: "ns", + errChecker: NotNil, + }, + } { + cleaner := &cleanse{ + dynCli: tc.cli, + } + err := cleaner.DeleteSnapshot(ctx, tc.snapshotName, tc.namespace, tc.groupVersion) + c.Check(err, tc.errChecker) + } +} diff --git a/pkg/csi/snapshot_restore_steps_test.go b/pkg/csi/snapshot_restore_steps_test.go new file mode 100644 index 0000000..fe26dc7 --- /dev/null +++ b/pkg/csi/snapshot_restore_steps_test.go @@ -0,0 +1,904 @@ +package csi + +import ( + "context" + "fmt" + + "github.com/golang/mock/gomock" + "github.com/kastenhq/kubestr/pkg/common" + "github.com/kastenhq/kubestr/pkg/csi/mocks" + "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + . "gopkg.in/check.v1" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func (s *CSITestSuite) TestValidateArgs(c *C) { + ctx := context.Background() + type fields struct { + validateOps *mocks.MockArgumentValidator + versionOps *mocks.MockApiVersionFetcher + } + for _, tc := range []struct { + args *types.CSISnapshotRestoreArgs + prepare func(f *fields) + errChecker Checker + }{ + { // valid args + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(nil), + f.validateOps.EXPECT().ValidateStorageClass(gomock.Any(), "sc").Return( + &sv1.StorageClass{ + Provisioner: "p1", + }, nil), + f.versionOps.EXPECT().GetCSISnapshotGroupVersion().Return( + &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }, nil), + f.validateOps.EXPECT().ValidateVolumeSnapshotClass(gomock.Any(), "vsc", &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }).Return(&unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassAlphaDriverKey: "p1", + }, + }, nil), + ) + }, + errChecker: IsNil, + }, + { // driver mismatch + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(nil), + f.validateOps.EXPECT().ValidateStorageClass(gomock.Any(), "sc").Return( + &sv1.StorageClass{ + Provisioner: "p1", + }, nil), + f.versionOps.EXPECT().GetCSISnapshotGroupVersion().Return( + &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }, nil), + f.validateOps.EXPECT().ValidateVolumeSnapshotClass(gomock.Any(), "vsc", &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }).Return(&unstructured.Unstructured{ + Object: map[string]interface{}{ + common.VolSnapClassAlphaDriverKey: "p2", + }, + }, nil), + ) + }, + errChecker: NotNil, + }, + { // vsc error + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(nil), + f.validateOps.EXPECT().ValidateStorageClass(gomock.Any(), "sc").Return( + &sv1.StorageClass{ + Provisioner: "p1", + }, nil), + f.versionOps.EXPECT().GetCSISnapshotGroupVersion().Return( + &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }, nil), + f.validateOps.EXPECT().ValidateVolumeSnapshotClass(gomock.Any(), "vsc", &metav1.GroupVersionForDiscovery{ + GroupVersion: common.SnapshotAlphaVersion, + }).Return(nil, fmt.Errorf("vsc error")), + ) + }, + errChecker: NotNil, + }, + { // groupversion error + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(nil), + f.validateOps.EXPECT().ValidateStorageClass(gomock.Any(), "sc").Return( + &sv1.StorageClass{ + Provisioner: "p1", + }, nil), + f.versionOps.EXPECT().GetCSISnapshotGroupVersion().Return( + nil, fmt.Errorf("groupversion error")), + ) + }, + errChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(nil), + f.validateOps.EXPECT().ValidateStorageClass(gomock.Any(), "sc").Return( + nil, fmt.Errorf("sc error")), + ) + }, + errChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + prepare: func(f *fields) { + gomock.InOrder( + f.validateOps.EXPECT().ValidateNamespace(gomock.Any(), "ns").Return(fmt.Errorf("ns error")), + ) + }, + errChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "", + VolumeSnapshotClass: "vsc", + Namespace: "ns", + }, + errChecker: NotNil, + }, { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "", + Namespace: "ns", + }, + errChecker: NotNil, + }, { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + VolumeSnapshotClass: "vsc", + Namespace: "", + }, + errChecker: NotNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + validateOps: mocks.NewMockArgumentValidator(ctrl), + versionOps: mocks.NewMockApiVersionFetcher(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + validateOps: f.validateOps, + versionFetchOps: f.versionOps, + } + err := stepper.ValidateArgs(ctx, tc.args) + c.Check(err, tc.errChecker) + } +} + +func (s *CSITestSuite) TestCreateApplication(c *C) { + ctx := context.Background() + type fields struct { + createAppOps *mocks.MockApplicationCreator + } + for _, tc := range []struct { + args *types.CSISnapshotRestoreArgs + genString string + prepare func(f *fields) + errChecker Checker + podChecker Checker + pvcChecker Checker + }{ + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + genString: "some string", + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), &types.CreatePVCArgs{ + GenerateName: originalPVCGenerateName, + StorageClass: "sc", + Namespace: "ns", + }).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), &types.CreatePodArgs{ + GenerateName: originalPodGenerateName, + PVCName: "pvc1", + Namespace: "ns", + Cmd: "echo 'some string' >> /data/out.txt; sync; tail -f /dev/null", + RunAsUser: 100, + ContainerImage: "image", + }).Return(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + }, nil), + f.createAppOps.EXPECT().WaitForPodReady(gomock.Any(), "ns", "pod1").Return(nil), + ) + }, + errChecker: IsNil, + podChecker: NotNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + genString: "some string", + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), &types.CreatePVCArgs{ + GenerateName: originalPVCGenerateName, + StorageClass: "sc", + Namespace: "ns", + }).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), &types.CreatePodArgs{ + GenerateName: originalPodGenerateName, + PVCName: "pvc1", + Namespace: "ns", + Cmd: "echo 'some string' >> /data/out.txt; sync; tail -f /dev/null", + RunAsUser: 100, + ContainerImage: "image", + }).Return(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + }, nil), + f.createAppOps.EXPECT().WaitForPodReady(gomock.Any(), "ns", "pod1").Return(fmt.Errorf("pod ready error")), + ) + }, + errChecker: NotNil, + podChecker: NotNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + genString: "some string", + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), gomock.Any()).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("create pod error")), + ) + }, + errChecker: NotNil, + podChecker: IsNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + genString: "some string", + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("create pvc error")), + ) + }, + errChecker: NotNil, + podChecker: IsNil, + pvcChecker: IsNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + createAppOps: mocks.NewMockApplicationCreator(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + createAppOps: f.createAppOps, + } + pod, pvc, err := stepper.CreateApplication(ctx, tc.args, tc.genString) + c.Check(err, tc.errChecker) + c.Check(pod, tc.podChecker) + c.Check(pvc, tc.pvcChecker) + } +} + +func (s *CSITestSuite) TestSnapshotApplication(c *C) { + ctx := context.Background() + snapshotter := &fakeSnapshotter{name: "snapshotter"} + groupversion := &metav1.GroupVersionForDiscovery{ + GroupVersion: "gv", + Version: "v", + } + type fields struct { + snapshotOps *mocks.MockSnapshotCreator + } + for _, tc := range []struct { + args *types.CSISnapshotRestoreArgs + pvc *v1.PersistentVolumeClaim + snapshotName string + prepare func(f *fields) + errChecker Checker + snapChecker Checker + }{ + { + args: &types.CSISnapshotRestoreArgs{ + Namespace: "ns", + VolumeSnapshotClass: "vsc", + }, + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, + snapshotName: "snap1", + prepare: func(f *fields) { + gomock.InOrder( + f.snapshotOps.EXPECT().NewSnapshotter().Return(snapshotter, nil), + f.snapshotOps.EXPECT().CreateSnapshot(gomock.Any(), snapshotter, &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc1", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }).Return(&snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "createdName", + }, + }, nil), + f.snapshotOps.EXPECT().CreateFromSourceCheck(gomock.Any(), snapshotter, &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "createdName", + Namespace: "ns", + }, groupversion).Return(nil), + ) + }, + errChecker: IsNil, + snapChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + Namespace: "ns", + VolumeSnapshotClass: "vsc", + SkipCFSCheck: true, + }, + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, + snapshotName: "snap1", + prepare: func(f *fields) { + gomock.InOrder( + f.snapshotOps.EXPECT().NewSnapshotter().Return(snapshotter, nil), + f.snapshotOps.EXPECT().CreateSnapshot(gomock.Any(), snapshotter, &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc1", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }).Return(&snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "createdName", + }, + }, nil), + ) + }, + errChecker: IsNil, + snapChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + Namespace: "ns", + VolumeSnapshotClass: "vsc", + }, + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, + snapshotName: "snap1", + prepare: func(f *fields) { + gomock.InOrder( + f.snapshotOps.EXPECT().NewSnapshotter().Return(snapshotter, nil), + f.snapshotOps.EXPECT().CreateSnapshot(gomock.Any(), snapshotter, &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc1", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }).Return(&snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "createdName", + }, + }, nil), + f.snapshotOps.EXPECT().CreateFromSourceCheck(gomock.Any(), snapshotter, &types.CreateFromSourceCheckArgs{ + VolumeSnapshotClass: "vsc", + SnapshotName: "createdName", + Namespace: "ns", + }, groupversion).Return(fmt.Errorf("cfs error")), + ) + }, + errChecker: NotNil, + snapChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + Namespace: "ns", + VolumeSnapshotClass: "vsc", + }, + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, + snapshotName: "snap1", + prepare: func(f *fields) { + gomock.InOrder( + f.snapshotOps.EXPECT().NewSnapshotter().Return(snapshotter, nil), + f.snapshotOps.EXPECT().CreateSnapshot(gomock.Any(), snapshotter, &types.CreateSnapshotArgs{ + Namespace: "ns", + PVCName: "pvc1", + VolumeSnapshotClass: "vsc", + SnapshotName: "snap1", + }).Return(nil, fmt.Errorf("create snapshot error")), + ) + }, + errChecker: NotNil, + snapChecker: IsNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + Namespace: "ns", + VolumeSnapshotClass: "vsc", + }, + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, + snapshotName: "snap1", + prepare: func(f *fields) { + gomock.InOrder( + f.snapshotOps.EXPECT().NewSnapshotter().Return(nil, fmt.Errorf("snapshotter error")), + ) + }, + errChecker: NotNil, + snapChecker: IsNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + snapshotOps: mocks.NewMockSnapshotCreator(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + snapshotCreateOps: f.snapshotOps, + SnapshotGroupVersion: groupversion, + } + snapshot, err := stepper.SnapshotApplication(ctx, tc.args, tc.pvc, tc.snapshotName) + c.Check(err, tc.errChecker) + c.Check(snapshot, tc.snapChecker) + } +} + +func (s *CSITestSuite) TestRestoreApplication(c *C) { + ctx := context.Background() + resourceQuantity := resource.MustParse("1Gi") + snapshotAPIGroup := "snapshot.storage.k8s.io" + type fields struct { + createAppOps *mocks.MockApplicationCreator + } + for _, tc := range []struct { + args *types.CSISnapshotRestoreArgs + snapshot *snapv1.VolumeSnapshot + prepare func(f *fields) + errChecker Checker + podChecker Checker + pvcChecker Checker + }{ + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap1", + }, + Status: &snapv1.VolumeSnapshotStatus{ + RestoreSize: &resourceQuantity, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), &types.CreatePVCArgs{ + GenerateName: clonedPVCGenerateName, + StorageClass: "sc", + Namespace: "ns", + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: "VolumeSnapshot", + Name: "snap1", + }, + RestoreSize: &resourceQuantity, + }).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), &types.CreatePodArgs{ + GenerateName: clonedPodGenerateName, + PVCName: "pvc1", + Namespace: "ns", + Cmd: "tail -f /dev/null", + RunAsUser: 100, + ContainerImage: "image", + }).Return(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + }, nil), + f.createAppOps.EXPECT().WaitForPodReady(gomock.Any(), "ns", "pod1").Return(nil), + ) + }, + errChecker: IsNil, + podChecker: NotNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap1", + }, + Status: &snapv1.VolumeSnapshotStatus{ + RestoreSize: &resourceQuantity, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), &types.CreatePVCArgs{ + GenerateName: clonedPVCGenerateName, + StorageClass: "sc", + Namespace: "ns", + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: "VolumeSnapshot", + Name: "snap1", + }, + RestoreSize: &resourceQuantity, + }).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), &types.CreatePodArgs{ + GenerateName: clonedPodGenerateName, + PVCName: "pvc1", + Namespace: "ns", + Cmd: "tail -f /dev/null", + RunAsUser: 100, + ContainerImage: "image", + }).Return(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + }, nil), + f.createAppOps.EXPECT().WaitForPodReady(gomock.Any(), "ns", "pod1").Return(fmt.Errorf("pod ready error")), + ) + }, + errChecker: NotNil, + podChecker: NotNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap1", + }, + Status: &snapv1.VolumeSnapshotStatus{ + RestoreSize: &resourceQuantity, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), gomock.Any()).Return(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + }, + }, nil), + f.createAppOps.EXPECT().CreatePod(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("create pod error")), + ) + }, + errChecker: NotNil, + podChecker: IsNil, + pvcChecker: NotNil, + }, + { + args: &types.CSISnapshotRestoreArgs{ + StorageClass: "sc", + Namespace: "ns", + RunAsUser: 100, + ContainerImage: "image", + }, + snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap1", + }, + Status: &snapv1.VolumeSnapshotStatus{ + RestoreSize: &resourceQuantity, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.createAppOps.EXPECT().CreatePVC(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("create pvc error")), + ) + }, + errChecker: NotNil, + podChecker: IsNil, + pvcChecker: IsNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + createAppOps: mocks.NewMockApplicationCreator(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + createAppOps: f.createAppOps, + } + pod, pvc, err := stepper.RestoreApplication(ctx, tc.args, tc.snapshot) + c.Check(err, tc.errChecker) + c.Check(pod, tc.podChecker) + c.Check(pvc, tc.pvcChecker) + } +} + +func (s *CSITestSuite) TestCleanup(c *C) { + ctx := context.Background() + groupversion := &metav1.GroupVersionForDiscovery{ + GroupVersion: "gv", + Version: "v", + } + type fields struct { + cleanerOps *mocks.MockCleaner + } + for _, tc := range []struct { + results *types.CSISnapshotRestoreResults + prepare func(f *fields) + }{ + { + results: nil, + }, + { + results: &types.CSISnapshotRestoreResults{ + OriginalPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + OriginalPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + ClonedPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + ClonedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + Snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + Namespace: "ns", + }, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.cleanerOps.EXPECT().DeletePVC(ctx, "pvc1", "ns").Return(nil), + f.cleanerOps.EXPECT().DeletePod(ctx, "pod1", "ns").Return(nil), + f.cleanerOps.EXPECT().DeletePVC(ctx, "pvc2", "ns").Return(nil), + f.cleanerOps.EXPECT().DeletePod(ctx, "pod2", "ns").Return(nil), + f.cleanerOps.EXPECT().DeleteSnapshot(ctx, "snapshot", "ns", groupversion).Return(nil), + ) + }, + }, + { + results: &types.CSISnapshotRestoreResults{ + OriginalPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + OriginalPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + ClonedPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + ClonedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + Snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + Namespace: "ns", + }, + }, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.cleanerOps.EXPECT().DeletePVC(ctx, "pvc1", "ns").Return(fmt.Errorf("err")), + f.cleanerOps.EXPECT().DeletePod(ctx, "pod1", "ns").Return(fmt.Errorf("err")), + f.cleanerOps.EXPECT().DeletePVC(ctx, "pvc2", "ns").Return(fmt.Errorf("err")), + f.cleanerOps.EXPECT().DeletePod(ctx, "pod2", "ns").Return(fmt.Errorf("err")), + f.cleanerOps.EXPECT().DeleteSnapshot(ctx, "snapshot", "ns", groupversion).Return(fmt.Errorf("err")), + ) + }, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + cleanerOps: mocks.NewMockCleaner(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + cleanerOps: f.cleanerOps, + SnapshotGroupVersion: groupversion, + } + stepper.Cleanup(ctx, tc.results) + } +} + +func (s *CSITestSuite) TestValidateData(c *C) { + ctx := context.Background() + type fields struct { + validatorOps *mocks.MockDataValidator + } + for _, tc := range []struct { + prepare func(f *fields) + pod *v1.Pod + data string + errChecker Checker + }{ + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "ns", + }, + }, + data: "somedata", + prepare: func(f *fields) { + gomock.InOrder( + f.validatorOps.EXPECT().FetchPodData("pod", "ns").Return("somedata", nil), + ) + }, + errChecker: IsNil, + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "ns", + }, + }, + data: "somedata", + prepare: func(f *fields) { + gomock.InOrder( + f.validatorOps.EXPECT().FetchPodData("pod", "ns").Return("someotherdata", nil), + ) + }, + errChecker: NotNil, + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "ns", + }, + }, + data: "somedata", + prepare: func(f *fields) { + gomock.InOrder( + f.validatorOps.EXPECT().FetchPodData("pod", "ns").Return("", fmt.Errorf("error")), + ) + }, + errChecker: NotNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + validatorOps: mocks.NewMockDataValidator(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + stepper := &snapshotRestoreSteps{ + dataValidatorOps: f.validatorOps, + } + err := stepper.ValidateData(ctx, tc.pod, tc.data) + c.Check(err, tc.errChecker) + } +} diff --git a/pkg/csi/snapshot_restore_test.go b/pkg/csi/snapshot_restore_test.go new file mode 100644 index 0000000..0d7d7aa --- /dev/null +++ b/pkg/csi/snapshot_restore_test.go @@ -0,0 +1,421 @@ +package csi + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/kastenhq/kubestr/pkg/csi/mocks" + "github.com/kastenhq/kubestr/pkg/csi/types" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + . "gopkg.in/check.v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + fakedynamic "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +func Test(t *testing.T) { TestingT(t) } + +type CSITestSuite struct{} + +var _ = Suite(&CSITestSuite{}) + +func (s *CSITestSuite) TestRunSnapshotRestoreHelper(c *C) { + ctx := context.Background() + type fields struct { + stepperOps *mocks.MockSnapshotRestoreStepper + } + for _, tc := range []struct { + kubeCli kubernetes.Interface + dynCli dynamic.Interface + args *types.CSISnapshotRestoreArgs + prepare func(f *fields) + result *types.CSISnapshotRestoreResults + errChecker Checker + }{ + { // success + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: true, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + nil, + ), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, gomock.Any(), + ).Return( + &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + }, + }, + nil, + ), + f.stepperOps.EXPECT().RestoreApplication(gomock.Any(), gomock.Any(), + &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + }, + }, + ).Return( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + nil, + ), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, gomock.Any()).Return(nil), + f.stepperOps.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Return(), + ) + }, + result: &types.CSISnapshotRestoreResults{ + OriginalPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + OriginalPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + ClonedPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + ClonedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + Snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + }, + }, + }, + errChecker: IsNil, + }, + { // no cleanup + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil), + f.stepperOps.EXPECT().RestoreApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: IsNil, + }, + { // restored data validation fails + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil), + f.stepperOps.EXPECT().RestoreApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("validation error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // restore error, objects still returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil), + f.stepperOps.EXPECT().RestoreApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + fmt.Errorf("restore error"), + ), + ) + }, + result: &types.CSISnapshotRestoreResults{ + ClonedPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc2", + Namespace: "ns", + }, + }, + ClonedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns", + }, + }, + }, + errChecker: NotNil, + }, + { // restore error, no objects returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil), + f.stepperOps.EXPECT().RestoreApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, fmt.Errorf("restore error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // snapshot error, object still returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return( + &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + }, + }, + fmt.Errorf("snapshot error"), + ), + ) + }, + result: &types.CSISnapshotRestoreResults{ + Snapshot: &snapv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshot", + }, + }, + }, + errChecker: NotNil, + }, + { // snapshot error, object not returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().SnapshotApplication(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("snapshot error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // created data validation error + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, nil), + f.stepperOps.EXPECT().ValidateData(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("validation error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // create error, objects still returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + fmt.Errorf("create error"), + ), + ) + }, + result: &types.CSISnapshotRestoreResults{ + OriginalPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: "ns", + }, + }, + OriginalPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns", + }, + }, + }, + errChecker: NotNil, + }, + { // create error, objects not returned + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(nil), + f.stepperOps.EXPECT().CreateApplication(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, fmt.Errorf("create error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // args validate error + kubeCli: fake.NewSimpleClientset(), + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + args: &types.CSISnapshotRestoreArgs{ + Cleanup: false, + }, + prepare: func(f *fields) { + gomock.InOrder( + f.stepperOps.EXPECT().ValidateArgs(gomock.Any(), gomock.Any()).Return(fmt.Errorf("create error")), + ) + }, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // empty cli + kubeCli: nil, + dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + { // empty dyncli + kubeCli: fake.NewSimpleClientset(), + dynCli: nil, + result: &types.CSISnapshotRestoreResults{}, + errChecker: NotNil, + }, + } { + ctrl := gomock.NewController(c) + defer ctrl.Finish() + f := fields{ + stepperOps: mocks.NewMockSnapshotRestoreStepper(ctrl), + } + if tc.prepare != nil { + tc.prepare(&f) + } + runner := &SnapshotRestoreRunner{ + KubeCli: tc.kubeCli, + DynCli: tc.dynCli, + srSteps: f.stepperOps, + } + result, err := runner.RunSnapshotRestoreHelper(ctx, tc.args) + c.Check(err, tc.errChecker) + c.Assert(result, DeepEquals, tc.result) + } +} + +func (s *CSITestSuite) TestRunSnapshotRestoreRunner(c *C) { + ctx := context.Background() + r := &SnapshotRestoreRunner{} + _, err := r.RunSnapshotRestore(ctx, nil) + c.Check(err, NotNil) +} diff --git a/pkg/csi/types/csi_types.go b/pkg/csi/types/csi_types.go new file mode 100644 index 0000000..1dd5061 --- /dev/null +++ b/pkg/csi/types/csi_types.go @@ -0,0 +1,92 @@ +package types + +import ( + "fmt" + + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +type CSISnapshotRestoreArgs struct { + StorageClass string + VolumeSnapshotClass string + Namespace string + RunAsUser int64 + ContainerImage string + Cleanup bool + SkipCFSCheck bool +} + +func (a *CSISnapshotRestoreArgs) Validate() error { + if a.StorageClass == "" || a.VolumeSnapshotClass == "" || a.Namespace == "" { + return fmt.Errorf("Require fields are missing. (StorageClass, VolumeSnapshotClass, Namespace)") + } + return nil +} + +type CSISnapshotRestoreResults struct { + OriginalPVC *v1.PersistentVolumeClaim + OriginalPod *v1.Pod + Snapshot *snapv1.VolumeSnapshot + ClonedPVC *v1.PersistentVolumeClaim + ClonedPod *v1.Pod +} + +type CreatePVCArgs struct { + GenerateName string + StorageClass string + Namespace string + DataSource *v1.TypedLocalObjectReference + RestoreSize *resource.Quantity +} + +func (c *CreatePVCArgs) Validate() error { + if c.GenerateName == "" || c.StorageClass == "" || c.Namespace == "" { + return fmt.Errorf("Invalid CreatePVCArgs (%v)", c) + } + return nil +} + +type CreatePodArgs struct { + GenerateName string + PVCName string + Namespace string + Cmd string + RunAsUser int64 + ContainerImage string +} + +func (c *CreatePodArgs) Validate() error { + if c.GenerateName == "" || c.PVCName == "" || c.Namespace == "" || c.Cmd == "" { + return fmt.Errorf("Invalid CreatePodArgs (%v)", c) + } + return nil +} + +type CreateSnapshotArgs struct { + Namespace string + PVCName string + VolumeSnapshotClass string + SnapshotName string +} + +func (c *CreateSnapshotArgs) Validate() error { + if c.Namespace == "" || c.PVCName == "" || c.VolumeSnapshotClass == "" || c.SnapshotName == "" { + return fmt.Errorf("Invalid CreateSnapshotArgs (%v)", c) + } + return nil +} + +type CreateFromSourceCheckArgs struct { + VolumeSnapshotClass string + SnapshotName string + Namespace string +} + +func (c *CreateFromSourceCheckArgs) Validate() error { + if c.VolumeSnapshotClass == "" || c.SnapshotName == "" || c.Namespace == "" { + return fmt.Errorf("Invalid CreateFromSourceCheckArgs (%v)", c) + } + return nil +} diff --git a/pkg/fio/_config.yml b/pkg/fio/_config.yml new file mode 100644 index 0000000..37d9f1b --- /dev/null +++ b/pkg/fio/_config.yml @@ -0,0 +1 @@ +baseurl: "/fio" diff --git a/pkg/fio/dbench_license b/pkg/fio/dbench_license new file mode 100644 index 0000000..96c4a5a --- /dev/null +++ b/pkg/fio/dbench_license @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 LogDNA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pkg/fio/fio.go b/pkg/fio/fio.go new file mode 100644 index 0000000..69e4e7b --- /dev/null +++ b/pkg/fio/fio.go @@ -0,0 +1,382 @@ +package fio + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "time" + + "github.com/briandowns/spinner" + kankube "github.com/kanisterio/kanister/pkg/kube" + "github.com/kastenhq/kubestr/pkg/common" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + // DefaultNS describes the default namespace + DefaultNS = "default" + // PodNamespaceEnvKey describes the pod namespace env variable + PodNamespaceEnvKey = "POD_NAMESPACE" + // DefaultFIOJob describes the default FIO job + DefaultFIOJob = "default-fio" + // KubestrFIOJobGenName describes the generate name + KubestrFIOJobGenName = "kubestr-fio" + // ConfigMapJobKey is the default fio job key + ConfigMapJobKey = "fiojob" + // DefaultPVCSize is the default PVC size + DefaultPVCSize = "100Gi" + // PVCGenerateName is the name to generate for the PVC + PVCGenerateName = "kubestr-fio-pvc-" + // PodGenerateName is the name to generate for the POD + PodGenerateName = "kubestr-fio-pod-" + // ContainerName is the name of the container that runs the job + ContainerName = "kubestr-fio" + // PodNameEnvKey is the name of the variable used to get the current pod name + PodNameEnvKey = "HOSTNAME" + // ConfigMapMountPath is the path where we mount the configmap + ConfigMapMountPath = "/etc/fio-config" + // VolumeMountPath is the path where we mount the volume + VolumeMountPath = "/dataset" + // CreatedByFIOLabel is the key that desrcibes the label used to mark configmaps + CreatedByFIOLabel = "createdbyfio" +) + +// FIO is an interface that represents FIO related commands +type FIO interface { + RunFio(ctx context.Context, args *RunFIOArgs) (*RunFIOResult, error) // , test config +} + +// FIOrunner implments FIO +type FIOrunner struct { + Cli kubernetes.Interface + fioSteps fioSteps +} + +type RunFIOArgs struct { + StorageClass string + Size string + Namespace string + FIOJobFilepath string + FIOJobName string + Image string +} + +func (a *RunFIOArgs) Validate() error { + if a.StorageClass == "" || a.Size == "" || a.Namespace == "" { + return fmt.Errorf("Require fields are missing. (StorageClass, Size, Namespace)") + } + return nil +} + +type RunFIOResult struct { + Size string `json:"size,omitempty"` + StorageClass *sv1.StorageClass `json:"storageClass,omitempty"` + FioConfig string `json:"fioConfig,omitempty"` + Result FioResult `json:"result,omitempty"` +} + +func (f *FIOrunner) RunFio(ctx context.Context, args *RunFIOArgs) (*RunFIOResult, error) { + f.fioSteps = &fioStepper{ + cli: f.Cli, + podReady: &podReadyChecker{cli: f.Cli}, + kubeExecutor: &kubeExecutor{cli: f.Cli}, + } + return f.RunFioHelper(ctx, args) + +} + +func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIOResult, error) { + // create a configmap with test parameters + if f.Cli == nil { // for UT purposes + return nil, fmt.Errorf("cli uninitialized") + } + + if err := args.Validate(); err != nil { + return nil, err + } + + if err := f.fioSteps.validateNamespace(ctx, args.Namespace); err != nil { + return nil, errors.Wrapf(err, "Unable to find namespace (%s)", args.Namespace) + } + + sc, err := f.fioSteps.storageClassExists(ctx, args.StorageClass) + if err != nil { + return nil, errors.Wrap(err, "Cannot find StorageClass") + } + + configMap, err := f.fioSteps.loadConfigMap(ctx, args) + if err != nil { + return nil, errors.Wrap(err, "Unable to create a ConfigMap") + } + defer func() { + _ = f.fioSteps.deleteConfigMap(context.TODO(), configMap, args.Namespace) + }() + + testFileName, err := fioTestFilename(configMap.Data) + if err != nil { + return nil, errors.Wrap(err, "Failed to get test file name.") + } + + pvc, err := f.fioSteps.createPVC(ctx, args.StorageClass, args.Size, args.Namespace) + if err != nil { + return nil, errors.Wrap(err, "Failed to create PVC") + } + defer func() { + _ = f.fioSteps.deletePVC(context.TODO(), pvc.Name, args.Namespace) + }() + fmt.Println("PVC created", pvc.Name) + + pod, err := f.fioSteps.createPod(ctx, pvc.Name, configMap.Name, testFileName, args.Namespace, args.Image) + defer func() { + _ = f.fioSteps.deletePod(context.TODO(), pod.Name, args.Namespace) + }() + if err != nil { + return nil, errors.Wrap(err, "Failed to create POD") + } + fmt.Println("Pod created", pod.Name) + fmt.Printf("Running FIO test (%s) on StorageClass (%s) with a PVC of Size (%s)\n", testFileName, args.StorageClass, args.Size) + fioOutput, err := f.fioSteps.runFIOCommand(ctx, pod.Name, ContainerName, testFileName, args.Namespace) + if err != nil { + return nil, errors.Wrap(err, "Failed while running FIO test.") + } + return &RunFIOResult{ + Size: args.Size, + StorageClass: sc, + FioConfig: configMap.Data[testFileName], + Result: fioOutput, + }, nil +} + +type fioSteps interface { + validateNamespace(ctx context.Context, namespace string) error + storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) + loadConfigMap(ctx context.Context, args *RunFIOArgs) (*v1.ConfigMap, error) + createPVC(ctx context.Context, storageclass, size, namespace string) (*v1.PersistentVolumeClaim, error) + deletePVC(ctx context.Context, pvcName, namespace string) error + createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) + deletePod(ctx context.Context, podName, namespace string) error + runFIOCommand(ctx context.Context, podName, containerName, testFileName, namespace string) (FioResult, error) + deleteConfigMap(ctx context.Context, configMap *v1.ConfigMap, namespace string) error +} + +type fioStepper struct { + cli kubernetes.Interface + podReady waitForPodReadyInterface + kubeExecutor kubeExecInterface +} + +func (s *fioStepper) validateNamespace(ctx context.Context, namespace string) error { + if _, err := s.cli.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}); err != nil { + return err + } + return nil +} + +func (s *fioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) { + return s.cli.StorageV1().StorageClasses().Get(ctx, storageClass, metav1.GetOptions{}) +} + +func (s *fioStepper) loadConfigMap(ctx context.Context, args *RunFIOArgs) (*v1.ConfigMap, error) { + configMap := &v1.ConfigMap{ + Data: make(map[string]string), + } + switch { + case args.FIOJobFilepath != "": + data, err := ioutil.ReadFile(args.FIOJobFilepath) + if err != nil { + return nil, errors.Wrap(err, "File reading error") + } + configMap.Data[filepath.Base(args.FIOJobFilepath)] = string(data) + case args.FIOJobName != "": + if _, ok := fioJobs[args.FIOJobName]; !ok { + return nil, fmt.Errorf("FIO job not found- (%s)", args.FIOJobName) + } + configMap.Data[args.FIOJobName] = fioJobs[args.FIOJobName] + default: + configMap.Data[DefaultFIOJob] = fioJobs[DefaultFIOJob] + } + // create + configMap.GenerateName = KubestrFIOJobGenName + configMap.Labels = map[string]string{CreatedByFIOLabel: "true"} + return s.cli.CoreV1().ConfigMaps(args.Namespace).Create(ctx, configMap, metav1.CreateOptions{}) +} + +func (s *fioStepper) createPVC(ctx context.Context, storageclass, size, namespace string) (*v1.PersistentVolumeClaim, error) { + sizeResource, err := resource.ParseQuantity(size) + if err != nil { + return nil, errors.Wrapf(err, "Unable to parse PVC size (%s)", size) + } + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: PVCGenerateName, + }, + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: &storageclass, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): sizeResource, + }, + }, + }, + } + return s.cli.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) +} + +func (s *fioStepper) deletePVC(ctx context.Context, pvcName, namespace string) error { + return s.cli.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) +} + +func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) { + if pvcName == "" || configMapName == "" || testFileName == "" { + return nil, fmt.Errorf("Create pod missing required arguments.") + } + + if image == "" { + image = common.DefaultPodImage + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: PodGenerateName, + Namespace: namespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: ContainerName, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "tail -f /dev/null"}, + VolumeMounts: []v1.VolumeMount{ + {Name: "persistent-storage", MountPath: VolumeMountPath}, + {Name: "config-map", MountPath: ConfigMapMountPath}, + }, + Image: image, + }}, + Volumes: []v1.Volume{ + { + Name: "persistent-storage", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}, + }, + }, + { + Name: "config-map", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + } + podRes, err := s.cli.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return podRes, err + } + + err = s.podReady.waitForPodReady(ctx, namespace, podRes.Name) + if err != nil { + return nil, err + } + + podRes, err = s.cli.CoreV1().Pods(namespace).Get(ctx, podRes.Name, metav1.GetOptions{}) + if err != nil { + return podRes, err + } + + return podRes, nil +} + +func (s *fioStepper) deletePod(ctx context.Context, podName, namespace string) error { + return s.cli.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) +} + +func (s *fioStepper) runFIOCommand(ctx context.Context, podName, containerName, testFileName, namespace string) (FioResult, error) { + jobFilePath := fmt.Sprintf("%s/%s", ConfigMapMountPath, testFileName) + command := []string{"fio", "--directory", VolumeMountPath, jobFilePath, "--output-format=json"} + done := make(chan bool, 1) + var fioOut FioResult + var stdout string + var stderr string + var err error + timestart := time.Now() + go func() { + stdout, stderr, err = s.kubeExecutor.exec(namespace, podName, containerName, command) + if err != nil || stderr != "" { + if err == nil { + err = fmt.Errorf("stderr when running FIO") + } + err = errors.Wrapf(err, "Error running command:(%v), stderr:(%s)", command, stderr) + } + done <- true + }() + spin := spinner.New(spinner.CharSets[9], 100*time.Millisecond) + spin.Start() + <-done + spin.Stop() + elapsed := time.Since(timestart) + fmt.Println("Elapsed time-", elapsed) + if err != nil { + return fioOut, err + } + + err = json.Unmarshal([]byte(stdout), &fioOut) + if err != nil { + return fioOut, errors.Wrapf(err, "Unable to parse fio output into json.") + } + + return fioOut, nil +} + +// deleteConfigMap only deletes a config map if it has the label +func (s *fioStepper) deleteConfigMap(ctx context.Context, configMap *v1.ConfigMap, namespace string) error { + if val, ok := configMap.Labels[CreatedByFIOLabel]; ok && val == "true" { + return s.cli.CoreV1().ConfigMaps(namespace).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) + } + return nil +} + +func fioTestFilename(configMap map[string]string) (string, error) { + if len(configMap) != 1 { + return "", fmt.Errorf("Unable to find fio file in configmap/more than one found %v", configMap) + } + var fileName string + for key := range configMap { + fileName = key + } + return fileName, nil +} + +type waitForPodReadyInterface interface { + waitForPodReady(ctx context.Context, namespace string, name string) error +} + +type podReadyChecker struct { + cli kubernetes.Interface +} + +func (p *podReadyChecker) waitForPodReady(ctx context.Context, namespace, name string) error { + return kankube.WaitForPodReady(ctx, p.cli, namespace, name) +} + +type kubeExecInterface interface { + exec(namespace, podName, containerName string, command []string) (string, string, error) +} + +type kubeExecutor struct { + cli kubernetes.Interface +} + +func (k *kubeExecutor) exec(namespace, podName, containerName string, command []string) (string, string, error) { + return kankube.Exec(k.cli, namespace, podName, containerName, command, nil) +} diff --git a/pkg/fio/fio_jobs.go b/pkg/fio/fio_jobs.go new file mode 100644 index 0000000..ae534e3 --- /dev/null +++ b/pkg/fio/fio_jobs.go @@ -0,0 +1,68 @@ +package fio + +var fioJobs = map[string]string{ + DefaultFIOJob: testJob1, + "randrw": randReadWrite, +} + +var testJob1 = `[global] +randrepeat=0 +verify=0 +ioengine=libaio +direct=1 +gtod_reduce=1 +[job1] +name=read_iops +bs=4K +iodepth=64 +size=2G +readwrite=randread +time_based +ramp_time=2s +runtime=15s +[job2] +name=write_iops +bs=4K +iodepth=64 +size=2G +readwrite=randwrite +time_based +ramp_time=2s +runtime=15s +[job3] +name=read_bw +bs=128K +iodepth=64 +size=2G +readwrite=randread +time_based +ramp_time=2s +runtime=15s +[job4] +name=write_bw +bs=128k +iodepth=64 +size=2G +readwrite=randwrite +time_based +ramp_time=2s +runtime=15s +` + +var randReadWrite = `[global] +randrepeat=0 +verify=0 +ioengine=libaio +direct=1 +gtod_reduce=1 +[job1] +name=rand_readwrite +bs=4K +iodepth=64 +size=4G +readwrite=randrw +rwmixread=75 +time_based +ramp_time=2s +runtime=15s +` diff --git a/pkg/fio/fio_test.go b/pkg/fio/fio_test.go new file mode 100644 index 0000000..7a36a8e --- /dev/null +++ b/pkg/fio/fio_test.go @@ -0,0 +1,889 @@ +package fio + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/kastenhq/kubestr/pkg/common" + "github.com/pkg/errors" + . "gopkg.in/check.v1" + v1 "k8s.io/api/core/v1" + scv1 "k8s.io/api/storage/v1" + sv1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func Test(t *testing.T) { TestingT(t) } + +type FIOTestSuite struct{} + +var _ = Suite(&FIOTestSuite{}) + +func (s *FIOTestSuite) TestRunner(c *C) { + ctx := context.Background() + runner := &FIOrunner{ + Cli: nil, + } + _, err := runner.RunFio(ctx, nil) + c.Check(err, NotNil) +} + +func (s *FIOTestSuite) TestRunFioHelper(c *C) { + ctx := context.Background() + for i, tc := range []struct { + cli kubernetes.Interface + stepper *fakeFioStepper + args *RunFIOArgs + expectedSteps []string + checker Checker + expectedCM string + expectedSC string + expectedSize string + expectedTFN string + expectedPVC string + }{ + { // invalid args (storageclass) + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{}, + args: &RunFIOArgs{}, + checker: NotNil, + }, + { // invalid args (size) + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{}, + args: &RunFIOArgs{ + StorageClass: "sc", + }, + checker: NotNil, + }, + { // invalid args (namespace) + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{}, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + }, + checker: NotNil, + }, + { // namespace doesn't exist + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + vnErr: fmt.Errorf("namespace Err"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN"}, + }, + { // storageclass not found + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + sceErr: fmt.Errorf("storageclass Err"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE"}, + }, + { // success + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmConfigMap: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM1", + }, + Data: map[string]string{ + "testfile.fio": "testfiledata", + }, + }, + cPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "PVC", + }, + }, + cPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "Pod", + }, + }, + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: IsNil, + expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, + expectedSC: "sc", + expectedSize: DefaultPVCSize, + expectedTFN: "testfile.fio", + expectedCM: "CM1", + expectedPVC: "PVC", + }, + { // fio test error + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmConfigMap: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM1", + }, + Data: map[string]string{ + "testfile.fio": "testfiledata", + }, + }, + cPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "PVC", + }, + }, + cPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "Pod", + }, + }, + rFIOErr: fmt.Errorf("run fio error"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, + }, + { // create pod error + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmConfigMap: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM1", + }, + Data: map[string]string{ + "testfile.fio": "testfiledata", + }, + }, + cPVC: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "PVC", + }, + }, + cPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "Pod", + }, + }, + cPodErr: fmt.Errorf("pod create error"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "DPOD", "DPVC", "DCM"}, + }, + { // create PVC error + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmConfigMap: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM1", + }, + Data: map[string]string{ + "testfile.fio": "testfiledata", + }, + }, + cPVCErr: fmt.Errorf("pvc create error"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "DCM"}, + }, + { // testfilename retrieval error, more than one provided + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmConfigMap: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM1", + }, + Data: map[string]string{ + "testfile.fio": "testfiledata", + "testfile.fio2": "testfiledata", + }, + }, + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE", "LCM", "DCM"}, + }, + { // load configmap error + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + lcmErr: fmt.Errorf("failed to load configmap"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "SCE", "LCM"}, + }, + } { + c.Log(i) + fio := &FIOrunner{ + Cli: tc.cli, + fioSteps: tc.stepper, + } + _, err := fio.RunFioHelper(ctx, tc.args) + c.Check(err, tc.checker) + c.Assert(tc.stepper.steps, DeepEquals, tc.expectedSteps) + if err == nil { + c.Assert(tc.expectedSC, Equals, tc.stepper.cPVCExpSC) + c.Assert(tc.expectedSize, Equals, tc.stepper.cPVCExpSize) + c.Assert(tc.expectedTFN, Equals, tc.stepper.cPodExpFN) + c.Assert(tc.expectedCM, Equals, tc.stepper.cPodExpCM) + c.Assert(tc.expectedPVC, Equals, tc.stepper.cPodExpPVC) + } + } +} + +type fakeFioStepper struct { + steps []string + + vnErr error + + sceSC *sv1.StorageClass + sceErr error + + lcmConfigMap *v1.ConfigMap + lcmErr error + + cPVCExpSC string + cPVCExpSize string + cPVC *v1.PersistentVolumeClaim + cPVCErr error + + dPVCErr error + + cPodExpFN string + cPodExpCM string + cPodExpPVC string + cPod *v1.Pod + cPodErr error + + dPodErr error + + rFIOout FioResult + rFIOErr error +} + +func (f *fakeFioStepper) validateNamespace(ctx context.Context, namespace string) error { + f.steps = append(f.steps, "VN") + return f.vnErr +} +func (f *fakeFioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) { + f.steps = append(f.steps, "SCE") + return f.sceSC, f.sceErr +} +func (f *fakeFioStepper) loadConfigMap(ctx context.Context, args *RunFIOArgs) (*v1.ConfigMap, error) { + f.steps = append(f.steps, "LCM") + return f.lcmConfigMap, f.lcmErr +} +func (f *fakeFioStepper) createPVC(ctx context.Context, storageclass, size, namespace string) (*v1.PersistentVolumeClaim, error) { + f.steps = append(f.steps, "CPVC") + f.cPVCExpSC = storageclass + f.cPVCExpSize = size + return f.cPVC, f.cPVCErr +} +func (f *fakeFioStepper) deletePVC(ctx context.Context, pvcName, namespace string) error { + f.steps = append(f.steps, "DPVC") + return f.dPVCErr +} +func (f *fakeFioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) { + f.steps = append(f.steps, "CPOD") + f.cPodExpCM = configMapName + f.cPodExpFN = testFileName + f.cPodExpPVC = pvcName + return f.cPod, f.cPodErr +} +func (f *fakeFioStepper) deletePod(ctx context.Context, podName, namespace string) error { + f.steps = append(f.steps, "DPOD") + return f.dPodErr +} +func (f *fakeFioStepper) runFIOCommand(ctx context.Context, podName, containerName, testFileName, namespace string) (FioResult, error) { + f.steps = append(f.steps, "RFIOC") + return f.rFIOout, f.rFIOErr +} +func (f *fakeFioStepper) deleteConfigMap(ctx context.Context, configMap *v1.ConfigMap, namespace string) error { + f.steps = append(f.steps, "DCM") + return nil +} + +func (s *FIOTestSuite) TestStorageClassExists(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + storageClass string + checker Checker + }{ + { + cli: fake.NewSimpleClientset(), + storageClass: "sc", + checker: NotNil, + }, + { + cli: fake.NewSimpleClientset(&scv1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "sc"}}), + storageClass: "sc", + checker: IsNil, + }, + } { + stepper := &fioStepper{cli: tc.cli} + _, err := stepper.storageClassExists(ctx, tc.storageClass) + c.Check(err, tc.checker) + } +} + +func (s *FIOTestSuite) TestValidateNamespace(c *C) { + ctx := context.Background() + stepper := &fioStepper{cli: fake.NewSimpleClientset()} + err := stepper.validateNamespace(ctx, "ns") + c.Assert(err, NotNil) + stepper = &fioStepper{cli: fake.NewSimpleClientset(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ns", + }, + })} + err = stepper.validateNamespace(ctx, "ns") + c.Assert(err, IsNil) +} + +func (s *FIOTestSuite) TestLoadConfigMap(c *C) { + ctx := context.Background() + file, err := ioutil.TempFile("", "tempTLCfile") + c.Check(err, IsNil) + defer os.Remove(file.Name()) + for i, tc := range []struct { + cli kubernetes.Interface + configMapName string + jobName string + args *RunFIOArgs + cmChecker Checker + errChecker Checker + failCreates bool + hasLabel bool + }{ + { // provided file name not found + cli: fake.NewSimpleClientset(), + args: &RunFIOArgs{ + FIOJobFilepath: "nonexistantfile", + }, + cmChecker: IsNil, + errChecker: NotNil, + }, + { // specified config map found + cli: fake.NewSimpleClientset(), + args: &RunFIOArgs{ + FIOJobFilepath: file.Name(), + FIOJobName: "random", // won't use this case + }, + cmChecker: NotNil, + errChecker: IsNil, + }, + { // specified job name, not found + cli: fake.NewSimpleClientset(), + args: &RunFIOArgs{ + FIOJobName: "random", + }, + cmChecker: IsNil, + errChecker: NotNil, + }, + { // specified job name, found + cli: fake.NewSimpleClientset(), + args: &RunFIOArgs{ + FIOJobName: DefaultFIOJob, + }, + cmChecker: NotNil, + errChecker: IsNil, + }, + { // use default job + cli: fake.NewSimpleClientset(), + args: &RunFIOArgs{}, + cmChecker: NotNil, + errChecker: IsNil, + }, + { // Fails to create configMap + cli: fake.NewSimpleClientset(), + cmChecker: IsNil, + errChecker: NotNil, + args: &RunFIOArgs{}, + failCreates: true, + }, + } { + c.Log(i) + stepper := &fioStepper{cli: tc.cli} + if tc.failCreates { + stepper.cli.(*fake.Clientset).Fake.PrependReactor("create", "configmaps", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("Error creating object") + }) + } + cm, err := stepper.loadConfigMap(ctx, tc.args) + c.Check(err, tc.errChecker) + c.Check(cm, tc.cmChecker) + if cm != nil { + _, ok := cm.Labels[CreatedByFIOLabel] + c.Assert(ok, Equals, true) + } + } +} + +func (s *FIOTestSuite) TestCreatePVC(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + storageclass string + size string + errChecker Checker + pvcChecker Checker + failCreates bool + }{ + { + cli: fake.NewSimpleClientset(), + storageclass: "fakesc", + size: "20Gi", + errChecker: IsNil, + pvcChecker: NotNil, + }, + { // Fails to create pvc + cli: fake.NewSimpleClientset(), + storageclass: "fakesc", + size: "10Gi", + pvcChecker: IsNil, + errChecker: NotNil, + failCreates: true, + }, + { // parse error + cli: fake.NewSimpleClientset(), + storageclass: "fakesc", + size: "Not a quantity", + pvcChecker: IsNil, + errChecker: NotNil, + }, + } { + stepper := &fioStepper{cli: tc.cli} + if tc.failCreates { + stepper.cli.(*fake.Clientset).Fake.PrependReactor("create", "*", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("Error creating object") + }) + } + pvc, err := stepper.createPVC(ctx, tc.storageclass, tc.size, DefaultNS) + c.Check(err, tc.errChecker) + c.Check(pvc, tc.pvcChecker) + if pvc != nil { + c.Assert(pvc.GenerateName, Equals, PVCGenerateName) + c.Assert(*pvc.Spec.StorageClassName, Equals, tc.storageclass) + value, ok := pvc.Spec.Resources.Requests.Storage().AsInt64() + c.Assert(ok, Equals, true) + c.Assert(value, Equals, int64(21474836480)) + } + } +} + +func (s *FIOTestSuite) TestDeletePVC(c *C) { + ctx := context.Background() + stepper := &fioStepper{cli: fake.NewSimpleClientset(&v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc", + Namespace: DefaultNS, + }})} + err := stepper.deletePVC(ctx, "pvc", DefaultNS) + c.Assert(err, IsNil) + err = stepper.deletePVC(ctx, "pvc", DefaultNS) + c.Assert(err, NotNil) +} + +func (s *FIOTestSuite) TestCreatPod(c *C) { + ctx := context.Background() + for _, tc := range []struct { + pvcName string + configMapName string + testFileName string + image string + reactor []k8stesting.Reactor + podReadyErr error + errChecker Checker + }{ + { + pvcName: "pvc", + configMapName: "cm", + testFileName: "testfile", + errChecker: IsNil, + }, + { + pvcName: "pvc", + configMapName: "cm", + testFileName: "testfile", + errChecker: NotNil, + reactor: []k8stesting.Reactor{ + &k8stesting.SimpleReactor{ + Verb: "create", + Resource: "*", + Reaction: func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod"}}, nil + }, + }, + &k8stesting.SimpleReactor{ + Verb: "get", + Resource: "*", + Reaction: func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("Error getting object") + }, + }, + }, + }, + { + pvcName: "pvc", + configMapName: "cm", + testFileName: "testfile", + errChecker: NotNil, + reactor: []k8stesting.Reactor{ + &k8stesting.SimpleReactor{ + Verb: "create", + Resource: "*", + Reaction: func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod"}}, nil + }, + }, + }, + podReadyErr: fmt.Errorf("pod ready error"), + }, + { + pvcName: "pvc", + configMapName: "cm", + testFileName: "testfile", + errChecker: NotNil, + reactor: []k8stesting.Reactor{ + &k8stesting.SimpleReactor{ + Verb: "create", + Resource: "*", + Reaction: func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("pod create error") + }, + }, + }, + }, + { + pvcName: "pvc", + configMapName: "cm", + testFileName: "", + image: "someotherimage", + errChecker: NotNil, + }, + { + pvcName: "", + configMapName: "cm", + testFileName: "asdf", + errChecker: NotNil, + }, + { + pvcName: "pvc", + configMapName: "", + testFileName: "asd", + errChecker: NotNil, + }, + } { + stepper := &fioStepper{ + cli: fake.NewSimpleClientset(), + podReady: &fakePodReadyChecker{prcErr: tc.podReadyErr}, + } + if tc.reactor != nil { + stepper.cli.(*fake.Clientset).Fake.ReactionChain = tc.reactor + } + pod, err := stepper.createPod(ctx, tc.pvcName, tc.configMapName, tc.testFileName, DefaultNS, tc.image) + c.Check(err, tc.errChecker) + if err == nil { + c.Assert(pod.GenerateName, Equals, PodGenerateName) + c.Assert(len(pod.Spec.Volumes), Equals, 2) + for _, vol := range pod.Spec.Volumes { + switch vol.Name { + case "persistent-storage": + c.Assert(vol.VolumeSource.PersistentVolumeClaim.ClaimName, Equals, tc.pvcName) + case "config-map": + c.Assert(vol.VolumeSource.ConfigMap.Name, Equals, tc.configMapName) + } + } + c.Assert(len(pod.Spec.Containers), Equals, 1) + c.Assert(pod.Spec.Containers[0].Name, Equals, ContainerName) + c.Assert(pod.Spec.Containers[0].Command, DeepEquals, []string{"/bin/sh"}) + c.Assert(pod.Spec.Containers[0].Args, DeepEquals, []string{"-c", "tail -f /dev/null"}) + c.Assert(pod.Spec.Containers[0].VolumeMounts, DeepEquals, []v1.VolumeMount{ + {Name: "persistent-storage", MountPath: VolumeMountPath}, + {Name: "config-map", MountPath: ConfigMapMountPath}, + }) + if tc.image == "" { + c.Assert(pod.Spec.Containers[0].Image, Equals, common.DefaultPodImage) + } else { + c.Assert(pod.Spec.Containers[0].Image, Equals, tc.image) + } + } + } +} + +func (s *FIOTestSuite) TestDeletePod(c *C) { + ctx := context.Background() + stepper := &fioStepper{cli: fake.NewSimpleClientset(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: DefaultNS, + }})} + err := stepper.deletePod(ctx, "pod", DefaultNS) + c.Assert(err, IsNil) + err = stepper.deletePod(ctx, "pod", DefaultNS) + c.Assert(err, NotNil) +} + +func (s *FIOTestSuite) TestFioTestFileName(c *C) { + for _, tc := range []struct { + configMap map[string]string + retVal string + errChecker Checker + }{ + { + configMap: map[string]string{ + "testfile.fio": "some test data", + }, + retVal: "testfile.fio", + errChecker: IsNil, + }, + { + configMap: map[string]string{ + "ConfigMapSCKey": "storageclass", + "ConfigMapSizeKey": "10Gi", + "testfile.fio": "some test data", + }, + retVal: "", + errChecker: NotNil, + }, + } { + ret, err := fioTestFilename(tc.configMap) + c.Check(err, tc.errChecker) + c.Assert(ret, Equals, tc.retVal) + } +} + +func (s *FIOTestSuite) TestRunFioCommand(c *C) { + var parsedout FioResult + err := json.Unmarshal([]byte(parsableFioOutput), &parsedout) + c.Assert(err, IsNil) + + ctx := context.Background() + for _, tc := range []struct { + executor *fakeKubeExecutor + errChecker Checker + podName string + containerName string + testFileName string + out FioResult + }{ + { + executor: &fakeKubeExecutor{ + keErr: nil, + keStrErr: "", + keStdOut: parsableFioOutput, + }, + errChecker: IsNil, + podName: "pod", + containerName: "container", + testFileName: "tfName", + out: parsedout, + }, + { + executor: &fakeKubeExecutor{ + keErr: nil, + keStrErr: "", + keStdOut: "unparsable string", + }, + errChecker: NotNil, + podName: "pod", + containerName: "container", + testFileName: "tfName", + out: FioResult{}, + }, + { + executor: &fakeKubeExecutor{ + keErr: fmt.Errorf("kubeexec err"), + keStrErr: "", + keStdOut: "unparsable string", + }, + errChecker: NotNil, + podName: "pod", + containerName: "container", + testFileName: "tfName", + out: FioResult{}, + }, + { + executor: &fakeKubeExecutor{ + keErr: nil, + keStrErr: "execution error", + keStdOut: "unparsable string", + }, + errChecker: NotNil, + podName: "pod", + containerName: "container", + testFileName: "tfName", + out: FioResult{}, + }, + } { + stepper := &fioStepper{ + kubeExecutor: tc.executor, + } + out, err := stepper.runFIOCommand(ctx, tc.podName, tc.containerName, tc.testFileName, DefaultNS) + c.Check(err, tc.errChecker) + c.Assert(out, DeepEquals, tc.out) + c.Assert(tc.executor.keInPodName, Equals, tc.podName) + c.Assert(tc.executor.keInContainerName, Equals, tc.containerName) + c.Assert(len(tc.executor.keInCommand), Equals, 5) + c.Assert(tc.executor.keInCommand[0], Equals, "fio") + c.Assert(tc.executor.keInCommand[1], Equals, "--directory") + c.Assert(tc.executor.keInCommand[2], Equals, VolumeMountPath) + jobFilePath := fmt.Sprintf("%s/%s", ConfigMapMountPath, tc.testFileName) + c.Assert(tc.executor.keInCommand[3], Equals, jobFilePath) + } +} + +func (s *FIOTestSuite) TestDeleteConfigMap(c *C) { + ctx := context.Background() + defaultNS := "default" + os.Setenv(PodNamespaceEnvKey, defaultNS) + for _, tc := range []struct { + cli kubernetes.Interface + cm *v1.ConfigMap + errChecker Checker + lenCMList int + }{ + { // Don't delete it unless it has the label + cli: fake.NewSimpleClientset(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: defaultNS, + }, + }), + cm: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: defaultNS, + }, + }, + errChecker: IsNil, + lenCMList: 1, + }, + { // Has label delete + cli: fake.NewSimpleClientset(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: defaultNS, + }, + }), + cm: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: defaultNS, + Labels: map[string]string{ + CreatedByFIOLabel: "true", + }, + }, + }, + errChecker: IsNil, + lenCMList: 0, + }, + { // No cm exists + cli: fake.NewSimpleClientset(), + cm: &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: defaultNS, + Labels: map[string]string{ + CreatedByFIOLabel: "true", + }, + }, + }, + errChecker: NotNil, + }, + } { + stepper := &fioStepper{cli: tc.cli} + err := stepper.deleteConfigMap(ctx, tc.cm, DefaultNS) + c.Check(err, tc.errChecker) + if err == nil { + list, err := stepper.cli.CoreV1().ConfigMaps(defaultNS).List(ctx, metav1.ListOptions{}) + c.Check(err, IsNil) + c.Assert(len(list.Items), Equals, tc.lenCMList) + } + } + os.Unsetenv(PodNamespaceEnvKey) +} + +func (s *FIOTestSuite) TestWaitForPodReady(c *C) { + ctx := context.Background() + prChecker := &podReadyChecker{ + cli: fake.NewSimpleClientset(), + } + err := prChecker.waitForPodReady(ctx, "somens", "somePod") + c.Check(err, NotNil) + prChecker.cli = fake.NewSimpleClientset(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somePod", + Namespace: "somens", + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + }, + }) +} + +type fakePodReadyChecker struct { + prcErr error +} + +func (f *fakePodReadyChecker) waitForPodReady(ctx context.Context, namespace, name string) error { + return f.prcErr +} + +type fakeKubeExecutor struct { + keErr error + keStdOut string + keStrErr string + keInNS string + keInPodName string + keInContainerName string + keInCommand []string +} + +func (fk *fakeKubeExecutor) exec(namespace, podName, containerName string, command []string) (string, string, error) { + fk.keInNS = namespace + fk.keInPodName = podName + fk.keInContainerName = containerName + fk.keInCommand = command + return fk.keStdOut, fk.keStrErr, fk.keErr +} diff --git a/pkg/fio/fio_types.go b/pkg/fio/fio_types.go new file mode 100644 index 0000000..d252da6 --- /dev/null +++ b/pkg/fio/fio_types.go @@ -0,0 +1,186 @@ +package fio + +import "fmt" + +type FioResult struct { + FioVersion string `json:"fio version,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + TimestampMS int64 `json:"timestamp_ms,omitempty"` + Time string `json:"time,omitempty"` + GlobalOptions FioGlobalOptions `json:"global options,omitempty"` + Jobs []FioJobs `json:"jobs,omitempty"` + DiskUtil []FioDiskUtil `json:"disk_util,omitempty"` +} + +func (f FioResult) Print() string { + var res string + res += fmt.Sprintf("FIO version - %s\n", f.FioVersion) + res += fmt.Sprintf("Global options - %s\n\n", f.GlobalOptions.Print()) + for _, job := range f.Jobs { + res += fmt.Sprintf("%s\n", job.Print()) + } + res += "Disk stats (read/write):\n" + for _, du := range f.DiskUtil { + res += fmt.Sprintf("%s\n", du.Print()) + } + + return res +} + +type FioGlobalOptions struct { + Directory string `json:"directory,omitempty"` + RandRepeat string `json:"randrepeat,omitempty"` + Verify string `json:"verify,omitempty"` + IOEngine string `json:"ioengine,omitempty"` + Direct string `json:"direct,omitempty"` + GtodReduce string `json:"gtod_reduce,omitempty"` +} + +func (g FioGlobalOptions) Print() string { + return fmt.Sprintf("ioengine=%s verify=%s direct=%s gtod_reduce=%s", g.IOEngine, g.Verify, g.Direct, g.GtodReduce) +} + +type FioJobs struct { + JobName string `json:"jobname,omitempty"` + GroupID int `json:"groupid,omitempty"` + Error int `json:"error,omitempty"` + Eta int `json:"eta,omitempty"` + Elapsed int `json:"elapsed,omitempty"` + JobOptions FioJobOptions `json:"job options,omitempty"` + Read FioStats `json:"read,omitempty"` + Write FioStats `json:"write,omitempty"` + Trim FioStats `json:"trim,omitempty"` + Sync FioStats `json:"sync,omitempty"` + JobRuntime int32 `json:"job_runtime,omitempty"` + UsrCpu float32 `json:"usr_cpu,omitempty"` + SysCpu float32 `json:"sys_cpu,omitempty"` + Ctx int32 `json:"ctx,omitempty"` + MajF int32 `json:"majf,omitempty"` + MinF int32 `json:"minf,omitempty"` + IoDepthLevel FioDepth `json:"iodepth_level,omitempty"` + IoDepthSubmit FioDepth `json:"iodepth_submit,omitempty"` + IoDepthComplete FioDepth `json:"iodepth_complete,omitempty"` + LatencyNs FioLatency `json:"latency_ns,omitempty"` + LatencyUs FioLatency `json:"latency_us,omitempty"` + LatencyMs FioLatency `json:"latency_ms,omitempty"` + LatencyDepth int32 `json:"latency_depth,omitempty"` + LatencyTarget int32 `json:"latency_target,omitempty"` + LatencyPercentile float32 `json:"latency_percentile,omitempty"` + LatencyWindow int32 `json:"latency_window,omitempty"` +} + +func (j FioJobs) Print() string { + var job string + job += fmt.Sprintf("%s\n", j.JobOptions.Print()) + if j.Read.Iops != 0 || j.Read.BW != 0 { + job += fmt.Sprintf("read:\n%s\n", j.Read.Print()) + } + if j.Write.Iops != 0 || j.Write.BW != 0 { + job += fmt.Sprintf("write:\n%s\n", j.Write.Print()) + } + return job +} + +type FioJobOptions struct { + Name string `json:"name,omitempty"` + BS string `json:"bs,omitempty"` + IoDepth string `json:"iodepth,omitempty"` + Size string `json:"size,omitempty"` + RW string `json:"rw,omitempty"` + RampTime string `json:"ramp_time,omitempty"` + RunTime string `json:"runtime,omitempty"` +} + +func (o FioJobOptions) Print() string { + return fmt.Sprintf("JobName: %s\n blocksize=%s filesize=%s iodepth=%s rw=%s", o.Name, o.BS, o.Size, o.IoDepth, o.RW) +} + +type FioStats struct { + IOBytes int64 `json:"io_bytes,omitempty"` + IOKBytes int64 `json:"io_kbytes,omitempty"` + BWBytes int64 `json:"bw_bytes,omitempty"` + BW int64 `json:"bw,omitempty"` + Iops float32 `json:"iops,omitempty"` + Runtime int64 `json:"runtime,omitempty"` + TotalIos int64 `json:"total_ios,omitempty"` + ShortIos int64 `json:"short_ios,omitempty"` + DropIos int64 `json:"drop_ios,omitempty"` + SlatNs FioNS `json:"slat_ns,omitempty"` + ClatNs FioNS `json:"clat_ns,omitempty"` + LatNs FioNS `json:"lat_ns,omitempty"` + BwMin int64 `json:"bw_min,omitempty"` + BwMax int64 `json:"bw_max,omitempty"` + BwAgg float32 `json:"bw_agg,omitempty"` + BwMean float32 `json:"bw_mean,omitempty"` + BwDev float32 `json:"bw_dev,omitempty"` + BwSamples int32 `json:"bw_samples,omitempty"` + IopsMin int32 `json:"iops_min,omitempty"` + IopsMax int32 `json:"iops_max,omitempty"` + IopsMean float32 `json:"iops_mean,omitempty"` + IopsStdDev float32 `json:"iops_stddev,omitempty"` + IopsSamples int32 `json:"iops_samples,omitempty"` +} + +func (s FioStats) Print() string { + var stats string + stats += fmt.Sprintf(" IOPS=%f BW(KiB/s)=%d\n", s.Iops, s.BW) + stats += fmt.Sprintf(" iops: min=%d max=%d avg=%f\n", s.IopsMin, s.IopsMax, s.IopsMean) + stats += fmt.Sprintf(" bw(KiB/s): min=%d max=%d avg=%f", s.BwMin, s.BwMax, s.BwMean) + return stats +} + +type FioNS struct { + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + Mean float32 `json:"mean,omitempty"` + StdDev float32 `json:"stddev,omitempty"` + N int64 `json:"N,omitempty"` +} + +type FioDepth struct { + FioDepth0 float32 `json:"0,omitempty"` + FioDepth1 float32 `json:"1,omitempty"` + FioDepth2 float32 `json:"2,omitempty"` + FioDepth4 float32 `json:"4,omitempty"` + FioDepth8 float32 `json:"8,omitempty"` + FioDepth16 float32 `json:"16,omitempty"` + FioDepth32 float32 `json:"32,omitempty"` + FioDepth64 float32 `json:"64,omitempty"` + FioDepthGE64 float32 `json:">=64,omitempty"` +} + +type FioLatency struct { + FioLat2 float32 `json:"2,omitempty"` + FioLat4 float32 `json:"4,omitempty"` + FioLat10 float32 `json:"10,omitempty"` + FioLat20 float32 `json:"20,omitempty"` + FioLat50 float32 `json:"50,omitempty"` + FioLat100 float32 `json:"100,omitempty"` + FioLat250 float32 `json:"250,omitempty"` + FioLat500 float32 `json:"500,omitempty"` + FioLat750 float32 `json:"750,omitempty"` + FioLat1000 float32 `json:"1000,omitempty"` + FioLat2000 float32 `json:"2000,omitempty"` + FioLatGE2000 float32 `json:">=2000,omitempty"` +} + +type FioDiskUtil struct { + Name string `json:"name,omitempty"` + ReadIos int64 `json:"read_ios,omitempty"` + WriteIos int64 `json:"write_ios,omitempty"` + ReadMerges int64 `json:"read_merges,omitempty"` + WriteMerges int64 `json:"write_merges,omitempty"` + ReadTicks int64 `json:"read_ticks,omitempty"` + WriteTicks int64 `json:"write_ticks,omitempty"` + InQueue int64 `json:"in_queue,omitempty"` + Util float32 `json:"util,omitempty"` +} + +func (d FioDiskUtil) Print() string { + //Disk stats (read/write): + //rbd4: ios=30022/11982, merge=0/313, ticks=1028675/1022768, in_queue=2063740, util=99.67% + var du string + du += fmt.Sprintf(" %s: ios=%d/%d merge=%d/%d ticks=%d/%d in_queue=%d, util=%f%%", d.Name, d.ReadIos, + d.WriteIos, d.ReadMerges, d.WriteMerges, d.ReadTicks, d.WriteTicks, d.InQueue, d.Util) + return du +} diff --git a/pkg/fio/parsable_fio_output.go b/pkg/fio/parsable_fio_output.go new file mode 100644 index 0000000..c9590cf --- /dev/null +++ b/pkg/fio/parsable_fio_output.go @@ -0,0 +1,491 @@ +package fio + +const parsableFioOutput = `{ + "fio version" : "fio-3.20", + "timestamp" : 1611952282, + "timestamp_ms" : 1611952282240, + "time" : "Fri Jan 29 20:31:22 2021", + "global options" : { + "directory" : "/dataset", + "randrepeat" : "0", + "verify" : "0", + "ioengine" : "libaio", + "direct" : "1", + "gtod_reduce" : "1" + }, + "jobs" : [ + { + "jobname" : "read_iops", + "groupid" : 0, + "error" : 0, + "eta" : 0, + "elapsed" : 18, + "job options" : { + "name" : "read_iops", + "bs" : "4K", + "iodepth" : "64", + "size" : "2G", + "rw" : "randread", + "ramp_time" : "2s", + "runtime" : "15s" + }, + "read" : { + "io_bytes" : 61886464, + "io_kbytes" : 60436, + "bw_bytes" : 4039322, + "bw" : 3944, + "iops" : 982.050780, + "runtime" : 15321, + "total_ios" : 15046, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 1919, + "bw_max" : 7664, + "bw_agg" : 100.000000, + "bw_mean" : 3995.000000, + "bw_dev" : 1200.820783, + "bw_samples" : 30, + "iops_min" : 479, + "iops_max" : 1916, + "iops_mean" : 998.566667, + "iops_stddev" : 300.247677, + "iops_samples" : 30 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 15322, + "usr_cpu" : 1.109516, + "sys_cpu" : 3.648349, + "ctx" : 17991, + "majf" : 1, + "minf" : 62, + "iodepth_level" : { + "1" : 0.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 100.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 99.993354, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.100000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 64, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + }, + { + "jobname" : "write_iops", + "groupid" : 0, + "error" : 0, + "eta" : 0, + "elapsed" : 18, + "job options" : { + "name" : "write_iops", + "bs" : "4K", + "iodepth" : "64", + "size" : "2G", + "rw" : "randwrite", + "ramp_time" : "2s", + "runtime" : "15s" + }, + "read" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "write" : { + "io_bytes" : 24805376, + "io_kbytes" : 24224, + "bw_bytes" : 1616406, + "bw" : 1578, + "iops" : 390.525218, + "runtime" : 15346, + "total_ios" : 5993, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 512, + "bw_max" : 2706, + "bw_agg" : 100.000000, + "bw_mean" : 1581.066667, + "bw_dev" : 476.641189, + "bw_samples" : 30, + "iops_min" : 128, + "iops_max" : 676, + "iops_mean" : 395.033333, + "iops_stddev" : 119.151738, + "iops_samples" : 30 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 15345, + "usr_cpu" : 0.508309, + "sys_cpu" : 2.280873, + "ctx" : 7411, + "majf" : 1, + "minf" : 63, + "iodepth_level" : { + "1" : 0.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 100.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 99.983317, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.100000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 64, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ], + "disk_util" : [ + { + "name" : "rbd4", + "read_ios" : 16957, + "write_ios" : 6896, + "read_merges" : 0, + "write_merges" : 207, + "read_ticks" : 1072290, + "write_ticks" : 1043421, + "in_queue" : 2119036, + "util" : 99.712875 + } + ] + }` diff --git a/pkg/kubestr/csi-drivers.go b/pkg/kubestr/csi-drivers.go new file mode 100644 index 0000000..784068c --- /dev/null +++ b/pkg/kubestr/csi-drivers.go @@ -0,0 +1,113 @@ +package kubestr + +// THIS FILE IS AUTO_GENERATED. +// To generate file run "go generate" at the top level +// This file must be checked in. + +var CSIDriverList = []*CSIDriver{ + {NameUrl: "[Alicloud Disk](https://github.com/AliyunContainerService/csi-plugin)", DriverName: "diskplugin.csi.alibabacloud.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Alicloud Disk", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot"}, + {NameUrl: "[Alicloud NAS](https://github.com/AliyunContainerService/csi-plugin)", DriverName: "nasplugin.csi.alibabacloud.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Alicloud Network Attached Storage (NAS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "No", Features: ""}, + {NameUrl: "[Alicloud OSS](https://github.com/AliyunContainerService/csi-plugin)", DriverName: "ossplugin.csi.alibabacloud.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Alicloud Object Storage Service (OSS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "No", Features: ""}, + {NameUrl: "[ArStor CSI](https://github.com/huayun-docs/csi-driver-arstor)", DriverName: "arstor.csi.huayun.io", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Huayun Storage Service (ArStor)", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[AWS Elastic Block Storage](https://github.com/kubernetes-sigs/aws-ebs-csi-driver)", DriverName: "ebs.csi.aws.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for AWS Elastic Block Storage (EBS)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[AWS Elastic File System](https://github.com/aws/aws-efs-csi-driver)", DriverName: "efs.csi.aws.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for AWS Elastic File System (EFS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "No", Features: ""}, + {NameUrl: "[AWS FSx for Lustre](https://github.com/aws/aws-fsx-csi-driver)", DriverName: "fsx.csi.aws.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for AWS FSx for Lustre (EBS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Azure disk](https://github.com/kubernetes-sigs/azuredisk-csi-driver)", DriverName: "disk.csi.azure.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Azure disk", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Azure file](https://github.com/kubernetes-sigs/azurefile-csi-driver)", DriverName: "file.csi.azure.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Azure file", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Bigtera VirtualStor (block)](https://github.com/bigtera-ce/ceph-csi)", DriverName: "csi.block.bigtera.com", Versions: "v0.3, v1.0.0, v1.1.0", Description: "A Container Storage Interface (CSI) Driver for Bigtera VirtualStor block storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[Bigtera VirtualStor (filesystem)](https://github.com/bigtera-ce/ceph-csi)", DriverName: "csi.fs.bigtera.com", Versions: "v0.3, v1.0.0, v1.1.0", Description: "A Container Storage Interface (CSI) Driver for Bigtera VirtualStor filesystem", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion"}, + {NameUrl: "[CephFS](https://github.com/ceph/ceph-csi)", DriverName: "cephfs.csi.ceph.com", Versions: "v0.3, v1.0.0, v1.1.0, v1.2.0", Description: "A Container Storage Interface (CSI) Driver for CephFS", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion, Snapshot, Clone"}, + {NameUrl: "[Ceph RBD](https://github.com/ceph/ceph-csi)", DriverName: "rbd.csi.ceph.com", Versions: "v0.3, v1.0.0, v1.1.0, v1.2.0", Description: "A Container Storage Interface (CSI) Driver for Ceph RBD", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Topology, Cloning"}, + {NameUrl: "[ChubaoFS](https://github.com/chubaofs/chubaofs-csi)", DriverName: "csi.chubaofs.com", Versions: "v1.0.0", Description: "A Container Storage Interface (CSI) Driver for ChubaoFS Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Cinder](https://github.com/kubernetes/cloud-provider-openstack/tree/master/pkg/csi/cinder)", DriverName: "cinder.csi.openstack.org", Versions: "v0.3, v1.0, v1.1", Description: "A Container Storage Interface (CSI) Driver for OpenStack Cinder", Persistence: "Persistent and Ephemeral", AccessModes: "Depends on the storage backend used", DynamicProvisioning: "Yes, if storage backend supports it", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[cloudscale.ch](https://github.com/cloudscale-ch/csi-cloudscale)", DriverName: "csi.cloudscale.ch", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for the [cloudscale.ch](https://www.cloudscale.ch/) IaaS platform", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[Datatom-InfinityCSI](https://github.com/datatom-infinity/infinity-csi)", DriverName: "csi-infiblock-plugin", Versions: "v0.3, v1.0.0, v1.1.0", Description: "A Container Storage Interface (CSI) Driver for DATATOM Infinity storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Topology"}, + {NameUrl: "[Datatom-InfinityCSI (filesystem)](https://github.com/datatom-infinity/infinity-csi)", DriverName: "csi-infifs-plugin", Versions: "v0.3, v1.0.0, v1.1.0", Description: "A Container Storage Interface (CSI) Driver for DATATOM Infinity filesystem storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion"}, + {NameUrl: "[Datera](https://github.com/Datera/datera-csi)", DriverName: "dsp.csi.daterainc.io", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Datera Data Services Platform (DSP)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[DDN EXAScaler](https://github.com/DDNStorage/exa-csi-driver)", DriverName: "exa.csi.ddn.com", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) Driver for DDN EXAScaler filesystems", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion"}, + {NameUrl: "[Dell EMC PowerScale](https://github.com/dell/csi-powerscale)", DriverName: "csi-isilon.dellemc.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Dell EMC PowerScale](https://www.delltechnologies.com/en-us/storage/powerscale.htm)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion, Cloning"}, + {NameUrl: "[Dell EMC PowerMax](https://github.com/dell/csi-powermax)", DriverName: "csi-powermax.dellemc.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Dell EMC PowerMax](https://www.delltechnologies.com/en-us/storage/powermax.htm)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Dell EMC PowerStore](https://github.com/dell/csi-powerstore)", DriverName: "csi-powerstore.dellemc.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Dell EMC PowerStore](https://www.delltechnologies.com/en-us/storage/powerstore-storage-appliance.htm)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Dell EMC Unity](https://github.com/dell/csi-unity)", DriverName: "csi-unity.dellemc.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Dell EMC Unity](https://www.delltechnologies.com/en-us/storage/unity.htm)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Dell EMC VxFlexOS](https://github.com/dell/csi-vxflexos)", DriverName: "csi-vxflexos.dellemc.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Dell EMC VxFlexOS](https://www.delltechnologies.com/en-us/hyperconverged-infrastructure/vxflex.htm)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Topology"}, + {NameUrl: "[democratic-csi](https://github.com/democratic-csi/democratic-csi)", DriverName: "org.democratic-csi.[X]", Versions: "v1.0,v1.1,v1.2", Description: "Generic CSI plugin supporting zfs based solutions ([FreeNAS](https://www.freenas.org/) / [TrueNAS](https://www.truenas.com/) and [ZoL](https://zfsonlinux.org/) solutions such as [Ubuntu](https://ubuntu.com/))", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Single Pod (Block Volume)

Read/Write Multiple Pods (File Volume)", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Diamanti-CSI](https://diamanti.com/use-cases/io-acceleration/#csi)", DriverName: "dcx.csi.diamanti.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Diamanti DCX Platform", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[DigitalOcean Block Storage](https://github.com/digitalocean/csi-digitalocean)", DriverName: "dobs.csi.digitalocean.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for DigitalOcean Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[DriveScale](https://github.com/DriveScale/k8s-plugins)", DriverName: "csi.drivescale.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for DriveScale software composable infrastructure solution", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Ember CSI](https://ember-csi.io)", DriverName: "[x].ember-csi.io", Versions: "v0.2, v0.3, v1.0", Description: "Multi-vendor CSI plugin supporting over 80 Drivers to provide block and mount storage to Container Orchestration systems.", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot"}, + {NameUrl: "[Excelero NVMesh](https://github.com/Excelero/nvmesh-csi-driver)", DriverName: "nvmesh-csi.excelero.com", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) Driver for Excelero NVMesh", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Expansion"}, + {NameUrl: "[GCE Persistent Disk](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver)", DriverName: "pd.csi.storage.gke.io", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Google Compute Engine Persistent Disk (GCE PD)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Topology"}, + {NameUrl: "[Google Cloud Filestore](https://github.com/kubernetes-sigs/gcp-filestore-csi-driver)", DriverName: "com.google.csi.filestore", Versions: "v0.3", Description: "A Container Storage Interface (CSI) Driver for Google Cloud Filestore", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Google Cloud Storage](https://github.com/ofek/csi-gcs)", DriverName: "gcs.csi.ofek.dev", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Google Cloud Storage", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion"}, + {NameUrl: "[GlusterFS](https://github.com/gluster/gluster-csi-driver)", DriverName: "org.gluster.glusterfs", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for GlusterFS", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[Gluster VirtBlock](https://github.com/gluster/gluster-csi-driver)", DriverName: "org.gluster.glustervirtblock", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Gluster Virtual Block volumes", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Hammerspace CSI](https://github.com/hammer-space/csi-plugin)", DriverName: "com.hammerspace.csi", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Hammerspace Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot"}, + {NameUrl: "[Hedvig](https://documentation.commvault.com/commvault/hedvig/others/pdf/Hedvig_CSI_User_Guide.pdf)", DriverName: "io.hedvig.csi", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Hedvig", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion"}, + {NameUrl: "[Hetzner Cloud Volumes CSI](https://github.com/hetznercloud/csi-driver)", DriverName: "csi.hetzner.cloud", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for Hetzner Cloud Volumes", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Expansion"}, + {NameUrl: "[Hitachi Vantara](https://knowledge.hitachivantara.com/Documents/Adapters_and_Drivers/Storage_Adapters_and_Drivers/Containers)", DriverName: "hspc.csi.hitachi.com", Versions: "v1.2", Description: "A Container Storage Interface (CSI) Driver for VSP series Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[HPE](https://github.com/hpe-storage/csi-driver)", DriverName: "csi.hpe.com", Versions: "v1.0, v1.1, v1.2", Description: "A [multi-platform](https://scod.hpedev.io/csi_driver) Container Storage Interface (CSI) driver. Supports [HPE Nimble Storage](https://hpe.com/storage/nimble), [HPE Primera](https://hpe.com/storage/primera) and [HPE 3PAR](https://hpe.com/storage/3par)", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Huawei Storage CSI](https://github.com/Huawei/eSDK_K8S_Plugin)", DriverName: "csi.huawei.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for FusionStorage, OceanStor 100D, OceanStor Dorado V3, OceanStor Dorado V6, OceanStor V3, OceanStor V5", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pod", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion, Cloning"}, + {NameUrl: "[HyperV CSI](https://github.com/Zetanova/hyperv-csi-driver)", DriverName: "eu.zetanova.csi.hyperv", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) driver to manage hyperv hosts", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[IBM Block Storage](https://github.com/ibm/ibm-block-csi-driver)", DriverName: "block.csi.ibm.com", Versions: "v1.0, v1.1, v1.2", Description: "A Container Storage Interface (CSI) [Driver](https://www.ibm.com/support/knowledgecenter/SSRQ8T) for IBM Spectrum Virtualize Family, IBM FlashSystem A9000 and A9000R, IBM DS8880 and DS8900.", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[IBM Spectrum Scale](https://github.com/IBM/ibm-spectrum-scale-csi)", DriverName: "spectrumscale.csi.ibm.com", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) [Driver](https://www.ibm.com/support/knowledgecenter/STXKQY_CSI_SHR) for the IBM Spectrum Scale File System", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[IBM Cloud Block Storage VPC CSI Driver](https://cloud.ibm.com/docs/containers?topic=containers-vpc-block)", DriverName: "vpc.block.csi.ibm.io", Versions: "v1.0", Description: "A Container Storage Interface (CSI) [Driver](https://cloud.ibm.com/docs/containers?topic=containers-vpc-block) for IBM Cloud Kubernetes Service and Red Hat OpenShift on IBM Cloud", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block"}, + {NameUrl: "[Infinidat](https://github.com/Infinidat/infinibox-csi-driver)", DriverName: "infinibox-csi-driver", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) Driver for Infinidat [InfiniBox](https://infinidat.com/en/products-technology/infinibox)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Inspur InStorage CSI](https://github.com/OpenInspur/instorage-k8s)", DriverName: "csi-instorage", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for inspur AS/HF/CS/CF Series Primary Storage, inspur AS13000 Series SDS Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Intel PMEM-CSI](https://github.com/intel/pmem-csi)", DriverName: "pmem-csi.intel.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) driver for [PMEM](https://pmem.io/) from Intel", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block"}, + {NameUrl: "[JuiceFS](https://github.com/juicedata/juicefs-csi-driver)", DriverName: "csi.juicefs.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for JuiceFS File System", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[kaDalu](https://github.com/kadalu/kadalu)", DriverName: "org.kadalu.gluster", Versions: "v0.3", Description: "A CSI Driver (and operator) for GlusterFS", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[KumoScale Block Storage](https://github.com/KioxiaAmerica/kumoscale-csi)", DriverName: "kumoscale.kioxia.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for KumoScale Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Topology"}, + {NameUrl: "[Linode Block Storage](https://github.com/linode/linode-blockstorage-csi-driver)", DriverName: "linodebs.csi.linode.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Linode Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[LINSTOR](https://github.com/LINBIT/linstor-csi)", DriverName: "io.drbd.linstor-csi", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [LINSTOR](https://www.linbit.com/en/linstor/) volumes", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[Longhorn](https://github.com/longhorn/longhorn)", DriverName: "driver.longhorn.io", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [Longhorn](https://longhorn.io/) volumes", Persistence: "Persistent", AccessModes: "Read/Write Single Node", DynamicProvisioning: "Yes", Features: "Raw Block"}, + {NameUrl: "[MacroSAN](https://github.com/macrosan-csi/macrosan-csi-driver)", DriverName: "csi-macrosan", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for MacroSAN Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Manila](https://github.com/kubernetes/cloud-provider-openstack/tree/master/pkg/csi/manila)", DriverName: "manila.csi.openstack.org", Versions: "v1.1, v1.2", Description: "A Container Storage Interface (CSI) Driver for OpenStack Shared File System Service (Manila)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Topology"}, + {NameUrl: "[MapR](https://github.com/mapr/mapr-csi)", DriverName: "com.mapr.csi-kdf", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for MapR Data Platform", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[MooseFS](https://github.com/moosefs/moosefs-csi)", DriverName: "com.tuxera.csi.moosefs", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for [MooseFS](https://moosefs.com/) clusters.", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[NetApp](https://github.com/NetApp/trident)", DriverName: "csi.trident.netapp.io", Versions: "v1.0, v1.1, v1.2", Description: "A Container Storage Interface (CSI) Driver for NetApp's [Trident](https://netapp-trident.readthedocs.io/) container storage orchestrator", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning, Topology"}, + {NameUrl: "[NexentaStor File Storage](https://github.com/Nexenta/nexentastor-csi-driver)", DriverName: "nexentastor-csi-driver.nexenta.com", Versions: "v1.0, v1.1, v1.2", Description: "A Container Storage Interface (CSI) Driver for NexentaStor File Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion, Cloning, Topology"}, + {NameUrl: "[NexentaStor Block Storage](https://github.com/Nexenta/nexentastor-csi-driver-block)", DriverName: "nexentastor-block-csi-driver.nexenta.com", Versions: "v1.0, v1.1, v1.2", Description: "A Container Storage Interface (CSI) Driver for NexentaStor over iSCSI protocol", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion, Cloning, Topology, Raw block"}, + {NameUrl: "[Nutanix](https://github.com/nutanix/csi-plugin)", DriverName: "com.nutanix.csi", Versions: "v0.3, v1.0, v1.2", Description: "A Container Storage Interface (CSI) Driver for Nutanix", Persistence: "Persistent", AccessModes: "Read/Write Single Pod with Nutanix Volumes and Read/Write Multiple Pods with Nutanix Files", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[OpenEBS](https://github.com/openebs/csi)", DriverName: "cstor.csi.openebs.io", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for [OpenEBS](https://www.openebs.io/)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Expansion, Snapshot, Cloning"}, + {NameUrl: "[OpenSDS](https://github.com/opensds/nbp/tree/master/csi)", DriverName: "csi-opensdsplugin", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for [OpenSDS]((https://www.opensds.io/))", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot"}, + {NameUrl: "[Open-E](https://github.com/open-e/JovianDSS-KubernetesCSI)", DriverName: "com.open-e.joviandss.csi", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Open-E JovianDSS Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Snapshot, Cloning"}, + {NameUrl: "[oVirt](https://github.com/openshift/ovirt-csi-driver)", DriverName: "csi.ovirt.org", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for [oVirt](https://ovirt.org)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Block, File Storage"}, + {NameUrl: "[Portworx](https://github.com/libopenstorage/openstorage/tree/master/csi)", DriverName: "pxd.openstorage.org", Versions: "v0.3, v1.1", Description: "A Container Storage Interface (CSI) Driver for [Portworx](https://docs.portworx.com/portworx-install-with-kubernetes/storage-operations/csi/)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion"}, + {NameUrl: "[Pure Storage CSI](https://github.com/purestorage/pso-csi)", DriverName: "pure-csi", Versions: "v1.0, v1.1, v1.2, v1.3", Description: "A Container Storage Interface (CSI) Driver for Pure Storage's [Pure Service Orchestrator](https://purestorage.com/containers)", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Cloning, Raw Block, Topology, Expansion"}, + {NameUrl: "[QingCloud CSI](https://github.com/yunify/qingcloud-csi)", DriverName: "disk.csi.qingcloud.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for QingCloud Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[QingStor CSI](https://github.com/yunify/qingstor-csi)", DriverName: "neonsan.csi.qingstor.com", Versions: "v0.3, v1.1", Description: "A Container Storage Interface (CSI) Driver for NeonSAN storage system", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Quobyte](https://github.com/quobyte/quobyte-csi)", DriverName: "quobyte-csi", Versions: "v0.2", Description: "A Container Storage Interface (CSI) Driver for Quobyte", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[ROBIN](https://get.robin.io/)", DriverName: "robin", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for [ROBIN](https://docs.robin.io)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[SandStone](https://github.com/sandstone-storage/sandstone-csi-driver)", DriverName: "csi-sandstone-plugin", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for SandStone USP", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Sangfor-EDS-File-Storage](https://github.com/evan37717/sangfor-eds-csi)", DriverName: "eds.csi.file.sangfor.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Sangfor Distributed File Storage(EDS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Sangfor-EDS-Block-Storage](https://github.com/eds-wzc/sangfor-eds-csi)", DriverName: "eds.csi.block.sangfor.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Sangfor Block Storage(EDS)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[SeaweedFS](https://github.com/seaweedfs/seaweedfs-csi-driver)", DriverName: "seaweedfs-csi-driver", Versions: "v1.0", Description: "A Container Storage Interface (CSI Driver for [SeaweedFS](https://github.com/chrislusf/seaweedfs))", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Secrets Store CSI Driver](https://github.com/kubernetes-sigs/secrets-store-csi-driver)", DriverName: "secrets-store.csi.k8s.io", Versions: "v0.0.10", Description: "A Container Storage Interface (CSI) Driver for mounting secrets, keys, and certs stored in enterprise-grade external secrets stores as volumes.", Persistence: "Ephemeral", AccessModes: "N/A", DynamicProvisioning: "N/A", Features: ""}, + {NameUrl: "[SmartX](http://www.smartx.com/?locale=en)", DriverName: "csi-smtx-plugin", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for SmartX ZBS Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion"}, + {NameUrl: "[SPDK-CSI](https://github.com/spdk/spdk-csi)", DriverName: "csi.spdk.io", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for [SPDK](https://spdk.io/)", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[StorageOS](https://docs.storageos.com/docs/platforms/kubernetes/install/)", DriverName: "storageos", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for [StorageOS](https://storageos.com/)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Storidge](https://docs.storidge.com/kubernetes_storage/overview.html)", DriverName: "csi.cio.storidge.com", Versions: "v0.3, v1.0", Description: "A Container Storage Interface (CSI) Driver for [Storidge CIO](https://storidge.com/)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion"}, + {NameUrl: "[StorPool](https://kb.storpool.com/storpool_integrations/github/kubernetes.html)", DriverName: "csi-driver.storpool.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for [StorPool](https://storpool.com/)", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Expansion"}, + {NameUrl: "[Tencent Cloud Block Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)", DriverName: "com.tencent.cloud.csi.cbs", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Tencent Cloud Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Snapshot"}, + {NameUrl: "[Tencent Cloud File Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)", DriverName: "com.tencent.cloud.csi.cfs", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Tencent Cloud File Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Tencent Cloud Object Storage](https://github.com/TencentCloud/kubernetes-csi-tencentcloud)", DriverName: "com.tencent.cloud.csi.cosfs", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for Tencent Cloud Object Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "No", Features: ""}, + {NameUrl: "[TopoLVM](https://github.com/cybozu-go/topolvm)", DriverName: "topolvm.cybozu.com", Versions: "v1.1", Description: "A Container Storage Interface (CSI) Driver for LVM", Persistence: "Persistent and Ephemeral", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Expansion, Topology Aware"}, + {NameUrl: "[VAST Data](https://github.com/vast-data/vast-csi)", DriverName: "csi.vastdata.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for VAST Data", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[XSKY-EBS](https://xsky-storage.github.io/xsky-csi-driver/csi-block.html)", DriverName: "csi.block.xsky.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for XSKY Distributed Block Storage (X-EBS)", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[XSKY-EUS](https://xsky-storage.github.io/xsky-csi-driver/csi-fs.html)", DriverName: "csi.fs.xsky.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for XSKY Distributed File Storage (X-EUS)", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Vault](https://github.com/kubevault/csi-driver)", DriverName: "secrets.csi.kubevault.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for mounting HashiCorp Vault secrets as volumes.", Persistence: "Ephemeral", AccessModes: "N/A", DynamicProvisioning: "N/A", Features: ""}, + {NameUrl: "[Veritas InfoScale Volumes](https://www.veritas.com/solution/virtualization/containers.html)", DriverName: "org.veritas.infoscale", Versions: "v1.2", Description: "A Container Storage Interface (CSI) Driver for Veritas InfoScale volumes", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Snapshot, Expansion, Cloning"}, + {NameUrl: "[vSphere](https://github.com/kubernetes-sigs/vsphere-csi-driver)", DriverName: "csi.vsphere.vmware.com", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for VMware vSphere", Persistence: "Persistent", AccessModes: "Read/Write Single Pod (Block Volume)

Read/Write Multiple Pods (File Volume)", DynamicProvisioning: "Yes", Features: "Raw Block,

Expansion (Block Volume),

Topology Aware (Block Volume)"}, + {NameUrl: "[Vultr Block Storage](https://github.com/vultr/vultr-csi)", DriverName: "block.csi.vultr.com", Versions: "v1.2", Description: "A Container Storage Interface (CSI) Driver for Vultr Block Storage", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[WekaIO](https://github.com/weka/csi-wekafs)", DriverName: "csi.weka.io", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for mounting WekaIO WekaFS filesystem as volumes", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Yandex.Cloud](https://github.com/flant/yandex-csi-driver)", DriverName: "yandex.csi.flant.com", Versions: "v1.2", Description: "A Container Storage Interface (CSI) plugin for Yandex.Cloud Compute Disks", Persistence: "Persistent", AccessModes: "Read/Write Single Pod", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[YanRongYun](http://www.yanrongyun.com/)", DriverName: "?", Versions: "v1.0", Description: "A Container Storage Interface (CSI) Driver for YanRong YRCloudFile Storage", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: ""}, + {NameUrl: "[Zadara-CSI](https://github.com/zadarastorage/zadara-csi)", DriverName: "csi.zadara.com", Versions: "v1.0, v1.1", Description: "A Container Storage Interface (CSI) plugin for Zadara VPSA Storage Array & VPSA All-Flash", Persistence: "Persistent", AccessModes: "Read/Write Multiple Pods", DynamicProvisioning: "Yes", Features: "Raw Block, Snapshot, Expansion, Cloning"}, + {NameUrl: "[Flexvolume](https://github.com/kubernetes-csi/csi-driver-flex)", DriverName: "Sample", Versions: "", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[HostPath](https://github.com/kubernetes-csi/csi-driver-host-path)", DriverName: "v1.2.0", Versions: "Only use for a single node tests. See the [Example](example.html) page for Kubernetes-specific instructions.", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[ImagePopulator](https://github.com/kubernetes-csi/csi-driver-image-populator)", DriverName: "Prototype", Versions: "Driver that lets you use a container image as an ephemeral volume.", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[In-memory Sample Mock Driver](https://github.com/kubernetes-csi/csi-test/tree/master/mock/service)", DriverName: "v0.3.0", Versions: "The sample mock driver used for [csi-sanity](https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity)", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[NFS](https://github.com/kubernetes-csi/csi-driver-nfs)", DriverName: "Sample", Versions: "", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[Synology NAS](https://github.com/jparklab/synology-csi)", DriverName: "v1.0.0", Versions: "An unofficial (and unsupported) Container Storage Interface Driver for Synology NAS.", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, + {NameUrl: "[VFS Driver](https://github.com/thecodeteam/csi-vfs)", DriverName: "Released", Versions: "A CSI plugin that provides a virtual file system.", Description: "", Persistence: "", AccessModes: "", DynamicProvisioning: "", Features: ""}, +} diff --git a/pkg/kubestr/kubernetes_checks.go b/pkg/kubestr/kubernetes_checks.go new file mode 100644 index 0000000..c7b5d34 --- /dev/null +++ b/pkg/kubestr/kubernetes_checks.go @@ -0,0 +1,118 @@ +package kubestr + +import ( + "fmt" + "strconv" + + "github.com/pkg/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + version "k8s.io/apimachinery/pkg/version" +) + +const ( + // MinK8sMajorVersion is the minimum supported Major version + MinK8sMajorVersion = 1 + // MinK8sMinorVersion is the minimum supported Minor version + MinK8sMinorVersion = 12 + // MinK8sGitVersion is the minimum supported k8s version + MinK8sGitVersion = "v1.12.0" + // RbacGroupName describe hte rbac group name + RbacGroupName = "rbac.authorization.k8s.io" +) + +// KubernetesChecks runs all the baseline checks on the cluster +func (p *Kubestr) KubernetesChecks() []*TestOutput { + var result []*TestOutput + result = append(result, p.validateK8sVersion()) + result = append(result, p.validateRBAC()) + result = append(result, p.validateAggregatedLayer()) + return result +} + +// validateK8sVersion validates the clusters K8s version +func (p *Kubestr) validateK8sVersion() *TestOutput { + testName := "Kubernetes Version Check" + version, err := p.validateK8sVersionHelper() + if err != nil { + return MakeTestOutput(testName, StatusError, err.Error(), nil) + } + return MakeTestOutput(testName, StatusOK, fmt.Sprintf("Valid kubernetes version (%s)", version.String()), version) +} + +// getK8sVersion fetches the k8s vesion +func (p *Kubestr) validateK8sVersionHelper() (*version.Info, error) { + version, err := p.cli.Discovery().ServerVersion() + if err != nil { + return nil, err + } + + majorStr := version.Major + if len(majorStr) > 1 && string(majorStr[len(majorStr)-1]) == "+" { + majorStr = majorStr[:len(majorStr)-1] + } + major, err := strconv.Atoi(majorStr) + if err != nil { + return nil, errors.Wrap(err, "Unable to derive kubernetes major version") + } + + minorStr := version.Minor + if len(minorStr) > 1 && string(minorStr[len(minorStr)-1]) == "+" { + minorStr = minorStr[:len(minorStr)-1] + } + minor, err := strconv.Atoi(minorStr) + if err != nil { + return nil, errors.Wrap(err, "Unable to derive kubernetes minor version") + } + if (major < MinK8sMajorVersion) || + (major == MinK8sMajorVersion && minor < MinK8sMinorVersion) { + return version, fmt.Errorf("Current kubernetes version (%s) is not supported. Minimum version is %s", version.String(), MinK8sGitVersion) + } + return version, nil +} + +func (p *Kubestr) validateRBAC() *TestOutput { + testName := "RBAC Check" + //fmt.Println(" Checking if Kubernetes RBAC is enabled:") + group, err := p.validateRBACHelper() + if err != nil { + return MakeTestOutput(testName, StatusError, err.Error(), nil) + } + return MakeTestOutput(testName, StatusOK, "Kubernetes RBAC is enabled", *group) +} + +// getRBAC runs the Rbac test +func (p *Kubestr) validateRBACHelper() (*v1.APIGroup, error) { + serverGroups, err := p.cli.Discovery().ServerGroups() + if err != nil { + return nil, err + } + for _, group := range serverGroups.Groups { + if group.Name == RbacGroupName { + return &group, nil + } + } + return nil, fmt.Errorf("Kubernetes RBAC is not enabled") +} + +func (p *Kubestr) validateAggregatedLayer() *TestOutput { + testName := "Aggregated Layer Check" + resourceList, err := p.validateAggregatedLayerHelper() + if err != nil { + MakeTestOutput(testName, StatusError, err.Error(), nil) + } + return MakeTestOutput(testName, StatusOK, "The Kubernetes Aggregated Layer is enabled", resourceList) +} + +// getAggregatedLayer checks the aggregated API layer +func (p *Kubestr) validateAggregatedLayerHelper() (*v1.APIResourceList, error) { + _, serverResources, err := p.cli.Discovery().ServerGroupsAndResources() + if err != nil { + return nil, err + } + for _, resourceList := range serverResources { + if resourceList.GroupVersion == "apiregistration.k8s.io/v1" || resourceList.GroupVersion == "apiregistration.k8s.io/v1beta1" { + return resourceList, nil + } + } + return nil, fmt.Errorf("Can not detect the Aggregated Layer. Is it enabled?") +} diff --git a/pkg/kubestr/kubernetes_checks_test.go b/pkg/kubestr/kubernetes_checks_test.go new file mode 100644 index 0000000..58bf79b --- /dev/null +++ b/pkg/kubestr/kubernetes_checks_test.go @@ -0,0 +1,160 @@ +package kubestr + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + version "k8s.io/apimachinery/pkg/version" + discoveryfake "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type K8sChecksTestSuite struct{} + +var _ = Suite(&K8sChecksTestSuite{}) + +func (s *K8sChecksTestSuite) TestGetK8sVersion(c *C) { + for _, tc := range []struct { + ver *version.Info + checker Checker + out *version.Info + }{ + { + ver: &version.Info{Major: "1", Minor: "17", GitVersion: "v1.17"}, + checker: IsNil, + out: &version.Info{Major: "1", Minor: "17", GitVersion: "v1.17"}, + }, + { + ver: &version.Info{Major: "1", Minor: "11", GitVersion: "v1.11"}, + checker: NotNil, + out: &version.Info{Major: "1", Minor: "11", GitVersion: "v1.11"}, + }, + { + ver: &version.Info{Major: "1", Minor: "", GitVersion: "v1."}, + checker: NotNil, + out: nil, + }, + { + ver: &version.Info{Major: "", Minor: "11", GitVersion: "v."}, + checker: NotNil, + out: nil, + }, + } { + cli := fake.NewSimpleClientset() + cli.Discovery().(*discoveryfake.FakeDiscovery).FakedServerVersion = tc.ver + p := &Kubestr{cli: cli} + out, err := p.validateK8sVersionHelper() + c.Assert(out, DeepEquals, tc.out) + c.Check(err, tc.checker) + } +} + +func (s *K8sChecksTestSuite) TestValidateRBAC(c *C) { + for _, tc := range []struct { + resources []*metav1.APIResourceList + checker Checker + out *v1.APIGroup + }{ + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "/////", + }, + }, + checker: NotNil, + out: nil, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "rbac.authorization.k8s.io/v1", + }, + }, + checker: IsNil, + out: &v1.APIGroup{ + Name: "rbac.authorization.k8s.io", + Versions: []v1.GroupVersionForDiscovery{ + {GroupVersion: "rbac.authorization.k8s.io/v1", Version: "v1"}, + }, + PreferredVersion: v1.GroupVersionForDiscovery{GroupVersion: "rbac.authorization.k8s.io/v1", Version: "v1"}, + }, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "notrbac.authorization.k8s.io/v1", + }, + }, + checker: NotNil, + out: nil, + }, + } { + cli := fake.NewSimpleClientset() + cli.Discovery().(*discoveryfake.FakeDiscovery).Resources = tc.resources + p := &Kubestr{cli: cli} + out, err := p.validateRBACHelper() + c.Assert(out, DeepEquals, tc.out) + c.Check(err, tc.checker) + } +} + +func (s *K8sChecksTestSuite) TestValidateAggregatedLayer(c *C) { + for _, tc := range []struct { + resources []*metav1.APIResourceList + checker Checker + out *metav1.APIResourceList + }{ + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "/////", + }, + }, + checker: NotNil, + out: nil, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "apiregistration.k8s.io/v1", + }, + }, + checker: IsNil, + out: &metav1.APIResourceList{ + GroupVersion: "apiregistration.k8s.io/v1", + }, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "apiregistration.k8s.io/v1beta1", + }, + }, + checker: IsNil, + out: &metav1.APIResourceList{ + GroupVersion: "apiregistration.k8s.io/v1beta1", + }, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "notapiregistration.k8s.io/v1", + }, + }, + checker: NotNil, + out: nil, + }, + } { + cli := fake.NewSimpleClientset() + cli.Discovery().(*discoveryfake.FakeDiscovery).Resources = tc.resources + p := &Kubestr{cli: cli} + out, err := p.validateAggregatedLayerHelper() + c.Assert(out, DeepEquals, tc.out) + c.Check(err, tc.checker) + } +} diff --git a/pkg/kubestr/kubestr.go b/pkg/kubestr/kubestr.go new file mode 100644 index 0000000..f389938 --- /dev/null +++ b/pkg/kubestr/kubestr.go @@ -0,0 +1,91 @@ +package kubestr + +import ( + "github.com/kanisterio/kanister/pkg/kube" + "github.com/kastenhq/kubestr/pkg/fio" + "github.com/pkg/errors" + sv1 "k8s.io/api/storage/v1" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// Kubestr is the primary object for running the kubestr tool. It holds all the cluster state information +// as well. +type Kubestr struct { + cli kubernetes.Interface + dynCli dynamic.Interface + sdsfgValidator snapshotDataSourceFG + storageClassList *sv1.StorageClassList + volumeSnapshotClassList *unstructured.UnstructuredList + Fio fio.FIO +} + +const Logo = ` +************************************** + _ ___ _ ___ ___ ___ _____ ___ + | |/ / | | | _ ) __/ __|_ _| _ \ + | ' <| |_| | _ \ _|\__ \ | | | / + |_|\_\\___/|___/___|___/ |_| |_|_\ + +Explore your kuberntes storage options +************************************** +` + +var ( + DefaultQPS = float32(50) + DefaultBurst = 100 +) + +// NewKubestr initializes a new kubestr object to run preflight tests +func NewKubestr() (*Kubestr, error) { + cli, err := LoadKubeCli() + if err != nil { + return nil, err + } + dynCli, err := LoadDynCli() + if err != nil { + return nil, err + } + return &Kubestr{ + cli: cli, + dynCli: dynCli, + sdsfgValidator: &snapshotDataSourceFGValidator{ + cli: cli, + dynCli: dynCli, + }, + Fio: &fio.FIOrunner{ + Cli: cli, + }, + }, nil +} + +// LoadDynCli loads the config and returns a dynamic CLI +func LoadDynCli() (dynamic.Interface, error) { + cfg, err := kube.LoadConfig() + if err != nil { + return nil, errors.Wrap(err, "Failed to load config for Dynamic client") + } + clientset, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "Failed to create Dynamic client") + } + return clientset, nil +} + +// LoadKubeCli load the config and returns a kubernetes client +// NewClient returns a k8 client configured by the kanister environment. +func LoadKubeCli() (kubernetes.Interface, error) { + config, err := kube.LoadConfig() + if err != nil { + return nil, err + } + config.QPS = DefaultQPS + config.Burst = DefaultBurst + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientset, nil +} diff --git a/pkg/kubestr/storage_provisioners.go b/pkg/kubestr/storage_provisioners.go new file mode 100644 index 0000000..66367ec --- /dev/null +++ b/pkg/kubestr/storage_provisioners.go @@ -0,0 +1,429 @@ +package kubestr + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + + kanvolume "github.com/kanisterio/kanister/pkg/kube/volume" + "github.com/kastenhq/kubestr/pkg/common" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +const ( + // APIVersionKey describes the APIVersion key + APIVersionKey = "apiVersion" + // FeatureGateTestPVCName is the name of the pvc created by the feature gate + // validation test + FeatureGateTestPVCName = "kubestr-featuregate-test" + // DefaultNS describes the default namespace + DefaultNS = "default" + // PodNamespaceEnvKey describes the pod namespace env variable + PodNamespaceEnvKey = "POD_NAMESPACE" +) + +// Provisioner holds the important information of a provisioner +type Provisioner struct { + ProvisionerName string + CSIDriver *CSIDriver + URL string + StorageClasses []*SCInfo + VolumeSnapshotClasses []*VSCInfo + StatusList []Status +} + +type CSIDriver struct { + NameUrl string + DriverName string + Versions string + Description string + Persistence string + AccessModes string + DynamicProvisioning string + Features string +} + +func (c *CSIDriver) Provider() string { + re := regexp.MustCompile(`\[(.*?)\]`) + match := re.FindStringSubmatch(c.NameUrl) // find the left most match + if len(match) < 2 { + return "" + } + return match[1] +} + +func (c *CSIDriver) URL() string { + re := regexp.MustCompile(`\((.*?)\)`) + match := re.FindAllStringSubmatch(c.NameUrl, -1) + if len(match) < 1 { + return "" + } + url := match[len(match)-1] // find the right most match + if len(url) < 2 { + return "" + } + return url[1] +} + +func (c *CSIDriver) Print(prefix string) { + fmt.Printf(prefix+" Provider: %s\n", c.Provider()) + fmt.Printf(prefix+" Website: %s\n", c.URL()) + fmt.Printf(prefix+" Description: %s\n", c.Description) + fmt.Printf(prefix+" Additional Features: %s\n", c.Features) +} + +func (c *CSIDriver) SupportsSnapshots() bool { + return strings.Contains(c.Features, "Snapshot") +} + +// SCInfo stores the info of a StorageClass +type SCInfo struct { + Name string + StatusList []Status + Raw interface{} `json:",omitempty"` +} + +// VSCInfo stores the info of a VolumeSnapshotClass +type VSCInfo struct { + Name string + StatusList []Status + HasAnnotation bool + Raw interface{} `json:",omitempty"` +} + +// Print prints the provionsioner specific details +func (v *Provisioner) Print() { + printSuccessColor(" " + v.ProvisionerName + ":") + for _, status := range v.StatusList { + status.Print(" ") + } + switch { + case v.CSIDriver != nil: + fmt.Println(" This is a CSI driver!") + fmt.Println(" (The following info may not be up to date. Please check with the provider for more information.)") + v.CSIDriver.Print(" ") + case strings.HasPrefix(v.ProvisionerName, "kubernetes.io"): + fmt.Println(" This is an in tree provisioner.") + case strings.Contains(v.ProvisionerName, "csi"): + fmt.Println(" This might be a CSI Driver. But it is not publicly listed.") + default: + fmt.Println(" Unknown driver type.") + } + fmt.Println() + if len(v.StorageClasses) > 0 { + fmt.Printf(" Storage Classes:\n") + for _, sc := range v.StorageClasses { + fmt.Printf(" * %s\n", sc.Name) + for _, status := range sc.StatusList { + status.Print(" ") + } + } + } + + if len(v.VolumeSnapshotClasses) > 0 { + fmt.Printf(" Volume Snapshot Classes:\n") + for _, vsc := range v.VolumeSnapshotClasses { + fmt.Printf(" * %s\n", vsc.Name) + for _, status := range vsc.StatusList { + status.Print(" ") + } + } + } + + if len(v.StorageClasses) > 0 { + fmt.Println() + fmt.Println(" To perform a FIO test, run-") + fmt.Println(" ./kubestr fio -s ") + switch { + case len(v.VolumeSnapshotClasses) == 0 && v.CSIDriver != nil && v.CSIDriver.SupportsSnapshots(): + fmt.Println() + fmt.Println(" This provisioner supports snapshots, however no Volume Snaphsot Classes were found.") + case len(v.VolumeSnapshotClasses) > 0: + fmt.Println() + fmt.Println(" To test CSI snapshot/restore functionality, run-") + fmt.Println(" ./kubestr csicheck -s -v ") + } + } +} + +// ValidateProvisioners validates the provisioners in a cluster +func (p *Kubestr) ValidateProvisioners(ctx context.Context) ([]*Provisioner, error) { + provisionerList, err := p.provisionerList(ctx) + if err != nil { + return nil, fmt.Errorf("Error listing provisioners: %s", err.Error()) + } + var validateProvisionersOutput []*Provisioner + for _, provisioner := range provisionerList { + processedProvisioner, err := p.processProvisioner(ctx, provisioner) + if err != nil { + return nil, err + } + validateProvisionersOutput = append(validateProvisionersOutput, processedProvisioner) + } + return validateProvisionersOutput, nil +} + +func (p *Kubestr) processProvisioner(ctx context.Context, provisioner string) (*Provisioner, error) { + retProvisioner := &Provisioner{ + ProvisionerName: provisioner, + } + + storageClassList, err := p.loadStorageClasses(ctx) + if err != nil { + return nil, err + } + for _, storageClass := range storageClassList.Items { + if storageClass.Provisioner == provisioner { + retProvisioner.StorageClasses = append(retProvisioner.StorageClasses, + p.validateStorageClass(provisioner, storageClass)) // review this + } + } + + for _, csiDriver := range CSIDriverList { + if strings.Contains(provisioner, csiDriver.DriverName) { + retProvisioner.CSIDriver = csiDriver + } + } + + if retProvisioner.CSIDriver != nil { + if !p.hasCSIDriverObject(ctx, provisioner) { + retProvisioner.StatusList = append(retProvisioner.StatusList, + makeStatus(StatusWarning, "Missing CSIDriver Object. Required by some provisioners.", nil)) + } + if clusterCsiSnapshotCapable, err := p.isK8sVersionCSISnapshotCapable(ctx); err != nil || !clusterCsiSnapshotCapable { + retProvisioner.StatusList = append(retProvisioner.StatusList, + makeStatus(StatusInfo, "Cluster is not CSI snapshot capable. Requires VolumeSnapshotDataSource feature gate.", nil)) + return retProvisioner, errors.Wrap(err, "Failed to validate if Kubernetes version was CSI capable") + } + csiSnapshotGroupVersion := p.getCSIGroupVersion() + if csiSnapshotGroupVersion == nil { + retProvisioner.StatusList = append(retProvisioner.StatusList, + makeStatus(StatusInfo, "Can't find the CSI snapshot group api version.", nil)) + return retProvisioner, nil + } + // load volumeSnapshotClass + vscs, err := p.loadVolumeSnapshotClasses(ctx, csiSnapshotGroupVersion.Version) + if err != nil { + return nil, errors.Wrap(err, "Failed to load volume snapshot classes") + } + for _, vsc := range vscs.Items { + if p.getDriverNameFromUVSC(vsc, csiSnapshotGroupVersion.GroupVersion) == provisioner { + retProvisioner.VolumeSnapshotClasses = append(retProvisioner.VolumeSnapshotClasses, + p.validateVolumeSnapshotClass(vsc, csiSnapshotGroupVersion.GroupVersion)) + } + } + } + return retProvisioner, nil +} + +// hasCSIDriverObject sees if a provisioner has a CSIDriver Object +func (p *Kubestr) hasCSIDriverObject(ctx context.Context, provisioner string) bool { + csiDrivers, err := p.cli.StorageV1beta1().CSIDrivers().List(ctx, metav1.ListOptions{}) + if err != nil { + return false + } + for _, driver := range csiDrivers.Items { + if driver.Name == provisioner { + return true + } + } + return false +} + +func (p *Kubestr) isK8sVersionCSISnapshotCapable(ctx context.Context) (bool, error) { + k8sVersion, err := p.validateK8sVersionHelper() + if err != nil { + return false, err + } + minorStr := k8sVersion.Minor + if string(minorStr[len(minorStr)-1]) == "+" { + minorStr = minorStr[:len(minorStr)-1] + } + minor, err := strconv.Atoi(minorStr) + if err != nil { + return false, err + } + if minor < 17 && k8sVersion.Major == "1" { + return p.sdsfgValidator.validate(ctx) + } + return true, nil +} + +// validateStorageClass validates a storageclass +func (p *Kubestr) validateStorageClass(provisioner string, storageClass sv1.StorageClass) *SCInfo { + scStatus := &SCInfo{ + Name: storageClass.Name, + Raw: storageClass, + } + return scStatus +} + +// validateVolumeSnapshotClass validates the VolumeSnapshotClass +func (p *Kubestr) validateVolumeSnapshotClass(vsc unstructured.Unstructured, groupVersion string) *VSCInfo { + retVSC := &VSCInfo{ + Name: vsc.GetName(), + Raw: vsc, + } + switch groupVersion { + case common.SnapshotAlphaVersion: + _, ok := vsc.Object[common.VolSnapClassAlphaDriverKey] + if !ok { + retVSC.StatusList = append(retVSC.StatusList, + makeStatus(StatusError, fmt.Sprintf("VolumeSnapshotClass (%s) missing 'snapshotter' field", vsc.GetName()), nil)) + } + case common.SnapshotBetaVersion: + _, ok := vsc.Object[common.VolSnapClassBetaDriverKey] + if !ok { + retVSC.StatusList = append(retVSC.StatusList, + makeStatus(StatusError, fmt.Sprintf("VolumeSnapshotClass (%s) missing 'driver' field", vsc.GetName()), nil)) + } + case common.SnapshotStableVersion: + _, ok := vsc.Object[common.VolSnapClassStableDriverKey] + if !ok { + retVSC.StatusList = append(retVSC.StatusList, + makeStatus(StatusError, fmt.Sprintf("VolumeSnapshotClass (%s) missing 'driver' field", vsc.GetName()), nil)) + } + } + return retVSC +} + +func (p *Kubestr) provisionerList(ctx context.Context) ([]string, error) { + storageClassList, err := p.loadStorageClasses(ctx) + if err != nil { + return nil, err + } + provisionerSet := make(map[string]struct{}) + for _, storageClass := range storageClassList.Items { + provisionerSet[storageClass.Provisioner] = struct{}{} + } + return convertSetToSlice(provisionerSet), nil +} + +func (p *Kubestr) loadStorageClasses(ctx context.Context) (*sv1.StorageClassList, error) { + if p.storageClassList == nil { + sc, err := p.cli.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + p.storageClassList = sc + } + return p.storageClassList, nil +} + +func (p *Kubestr) loadVolumeSnapshotClasses(ctx context.Context, version string) (*unstructured.UnstructuredList, error) { + if p.volumeSnapshotClassList == nil { + VolSnapClassGVR := schema.GroupVersionResource{Group: common.SnapGroupName, Version: version, Resource: common.VolumeSnapshotClassResourcePlural} + us, err := p.dynCli.Resource(VolSnapClassGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + p.volumeSnapshotClassList = us + } + return p.volumeSnapshotClassList, nil +} + +// getDriverNameFromUVSC get the driver name from an unstructured VSC +func (p *Kubestr) getDriverNameFromUVSC(vsc unstructured.Unstructured, version string) string { + var driverName interface{} + var ok bool + switch version { + case common.SnapshotAlphaVersion: + driverName, ok = vsc.Object[common.VolSnapClassAlphaDriverKey] + if !ok { + return "" + } + case common.SnapshotBetaVersion: + driverName, ok = vsc.Object[common.VolSnapClassBetaDriverKey] + if !ok { + return "" + } + case common.SnapshotStableVersion: + driverName, ok = vsc.Object[common.VolSnapClassStableDriverKey] + if !ok { + return "" + } + } + driver, ok := driverName.(string) + if !ok { + return "" + } + return driver +} + +// getCSIGroupVersion fetches the CSI Group Version +func (p *Kubestr) getCSIGroupVersion() *metav1.GroupVersionForDiscovery { + groups, _, err := p.cli.Discovery().ServerGroupsAndResources() + if err != nil { + return nil + } + for _, group := range groups { + if group.Name == common.SnapGroupName { + return &group.PreferredVersion + } + } + return nil +} + +type snapshotDataSourceFG interface { + validate(ctx context.Context) (bool, error) +} + +type snapshotDataSourceFGValidator struct { + cli kubernetes.Interface + dynCli dynamic.Interface +} + +func (s *snapshotDataSourceFGValidator) validate(ctx context.Context) (bool, error) { + ns := getPodNamespace() + + // deletes if exists. If it doesn't exist, this is a noop + err := kanvolume.DeletePVC(s.cli, ns, FeatureGateTestPVCName) + if err != nil { + return false, errors.Wrap(err, "Error deleting VolumeSnapshotDataSource feature-gate validation pvc") + } + // defer delete + defer func() { + _ = kanvolume.DeletePVC(s.cli, ns, FeatureGateTestPVCName) + }() + + // create PVC + snapshotKind := "VolumeSnapshot" + snapshotAPIGroup := "snapshot.storage.k8s.io" + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: FeatureGateTestPVCName, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: snapshotKind, + Name: "fakeSnap", + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + + pvcRes, err := s.cli.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return false, errors.Wrap(err, "Error creating VolumeSnapshotDataSource feature-gate validation pvc") + } + if pvcRes.Spec.DataSource == nil { + return false, nil + } + return true, nil +} diff --git a/pkg/kubestr/storage_provisioners_test.go b/pkg/kubestr/storage_provisioners_test.go new file mode 100644 index 0000000..4cc73fc --- /dev/null +++ b/pkg/kubestr/storage_provisioners_test.go @@ -0,0 +1,352 @@ +package kubestr + +import ( + "context" + "fmt" + + "github.com/kanisterio/kanister/pkg/kube/snapshot/apis/v1alpha1" + . "gopkg.in/check.v1" + scv1 "k8s.io/api/storage/v1" + "k8s.io/api/storage/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + version "k8s.io/apimachinery/pkg/version" + discoveryfake "k8s.io/client-go/discovery/fake" + fakedynamic "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +type ProvisionerTestSuite struct{} + +var _ = Suite(&ProvisionerTestSuite{}) + +func (s *ProvisionerTestSuite) TestHasCSIDriverObject(c *C) { + ctx := context.Background() + for _, tc := range []struct { + cli kubernetes.Interface + provisionerName string + hasDriver bool + }{ + { + cli: fake.NewSimpleClientset(), + provisionerName: "provisioner", + hasDriver: false, + }, + { + cli: fake.NewSimpleClientset(&v1beta1.CSIDriverList{ + Items: []v1beta1.CSIDriver{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "drivername", + }, + }, + }}), + provisionerName: "drivername", + hasDriver: true, + }, + } { + p := &Kubestr{cli: tc.cli} + hasDriver := p.hasCSIDriverObject(ctx, tc.provisionerName) + c.Assert(hasDriver, Equals, tc.hasDriver) + } +} + +func (s *ProvisionerTestSuite) TestIsK8sVersionCSISnapshotCapable(c *C) { + ctx := context.Background() + for _, tc := range []struct { + ver *version.Info + checker Checker + capable bool + sdsfg snapshotDataSourceFG + }{ + { + ver: &version.Info{Major: "1", Minor: "", GitVersion: "v1.17"}, + checker: NotNil, + capable: false, + }, + { + ver: &version.Info{Major: "1", Minor: "15+", GitVersion: "v1.15+"}, + checker: NotNil, + capable: false, + sdsfg: &fakeSDSFGValidator{err: fmt.Errorf("someerror"), cap: false}, + }, + { + ver: &version.Info{Major: "1", Minor: "15+", GitVersion: "v1.15+"}, + checker: IsNil, + capable: true, + sdsfg: &fakeSDSFGValidator{err: nil, cap: true}, + }, + { + ver: &version.Info{Major: "1", Minor: "17", GitVersion: "v1.17"}, + checker: IsNil, + capable: true, + }, + } { + cli := fake.NewSimpleClientset() + cli.Discovery().(*discoveryfake.FakeDiscovery).FakedServerVersion = tc.ver + p := &Kubestr{cli: cli, sdsfgValidator: tc.sdsfg} + cap, err := p.isK8sVersionCSISnapshotCapable(ctx) + c.Check(err, tc.checker) + c.Assert(cap, Equals, tc.capable) + } +} + +type fakeSDSFGValidator struct { + err error + cap bool +} + +func (f *fakeSDSFGValidator) validate(ctx context.Context) (bool, error) { + return f.cap, f.err +} + +func (s *ProvisionerTestSuite) TestValidateVolumeSnapshotClass(c *C) { + for _, tc := range []struct { + vsc unstructured.Unstructured + groupVersion string + out *VSCInfo + }{ + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "vsc1", + }, + "snapshotter": "something", + }, + }, + groupVersion: "snapshot.storage.k8s.io/v1alpha1", + out: &VSCInfo{ + Name: "vsc1", + }, + }, + { // failure + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "vsc1", + }, + "notsnapshotter": "something", + }, + }, + groupVersion: "snapshot.storage.k8s.io/v1alpha1", + out: &VSCInfo{ + Name: "vsc1", + StatusList: []Status{ + makeStatus(StatusError, fmt.Sprintf("VolumeSnapshotClass (%s) missing 'snapshotter' field", "vsc1"), nil), + }, + }, + }, + { + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "vsc1", + }, + "driver": "something", + }, + }, + groupVersion: "snapshot.storage.k8s.io/v1beta1", + out: &VSCInfo{ + Name: "vsc1", + }, + }, + { // failure + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "vsc1", + }, + "notdriver": "something", + }, + }, + groupVersion: "snapshot.storage.k8s.io/v1beta1", + out: &VSCInfo{ + Name: "vsc1", + StatusList: []Status{ + makeStatus(StatusError, fmt.Sprintf("VolumeSnapshotClass (%s) missing 'driver' field", "vsc1"), nil), + }, + }, + }, + } { + p := &Kubestr{} + out := p.validateVolumeSnapshotClass(tc.vsc, tc.groupVersion) + c.Assert(out.Name, Equals, tc.out.Name) + c.Assert(len(out.StatusList), Equals, len(tc.out.StatusList)) + } +} + +func (s *ProvisionerTestSuite) TestLoadStorageClassesAndProvisioners(c *C) { + ctx := context.Background() + p := &Kubestr{cli: fake.NewSimpleClientset( + &scv1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "sc1"}, Provisioner: "provisioner1"}, + &scv1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "sc2"}, Provisioner: "provisioner2"}, + )} + scs, err := p.loadStorageClasses(ctx) + c.Assert(err, IsNil) + c.Assert(len(scs.Items), Equals, 2) + c.Assert(scs, Equals, p.storageClassList) + + // reload has the same + p.cli = fake.NewSimpleClientset() + scs, err = p.loadStorageClasses(ctx) + c.Assert(err, IsNil) + c.Assert(len(scs.Items), Equals, 2) + c.Assert(scs, Equals, p.storageClassList) + + // proviosners uses loaded list + provisioners, err := p.provisionerList(ctx) + c.Assert(err, IsNil) + c.Assert(len(provisioners), Equals, 2) +} + +func (s *ProvisionerTestSuite) TestLoadVolumeSnaphsotClasses(c *C) { + ctx := context.Background() + p := &Kubestr{dynCli: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", v1alpha1.GroupName, v1alpha1.Version), + "kind": "VolumeSnapshotClass", + "metadata": map[string]interface{}{ + "name": "theVSC", + }, + "snapshotter": "somesnapshotter", + "deletionPolicy": "Delete", + }, + })} + vsc, err := p.loadVolumeSnapshotClasses(ctx, v1alpha1.Version) + c.Assert(err, IsNil) + c.Assert(len(vsc.Items), Equals, 1) + c.Assert(vsc, Equals, p.volumeSnapshotClassList) + + // reload has the same + p.dynCli = fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()) + vsc, err = p.loadVolumeSnapshotClasses(ctx, v1alpha1.Version) + c.Assert(err, IsNil) + c.Assert(len(vsc.Items), Equals, 1) + c.Assert(vsc, Equals, p.volumeSnapshotClassList) +} + +func (s *ProvisionerTestSuite) TestGetCSIGroupVersion(c *C) { + for _, tc := range []struct { + resources []*metav1.APIResourceList + out *metav1.GroupVersionForDiscovery + }{ + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "/////", + }, + }, + out: nil, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "snapshot.storage.k8s.io/v1beta1", + }, + { + GroupVersion: "snapshot.storage.k8s.io/v1apha1", + }, + }, + out: &metav1.GroupVersionForDiscovery{ + GroupVersion: "snapshot.storage.k8s.io/v1beta1", + Version: "v1beta1", + }, + }, + { + resources: []*metav1.APIResourceList{ + { + GroupVersion: "NOTsnapshot.storage.k8s.io/v1beta1", + }, + }, + out: nil, + }, + } { + cli := fake.NewSimpleClientset() + cli.Discovery().(*discoveryfake.FakeDiscovery).Resources = tc.resources + p := &Kubestr{cli: cli} + out := p.getCSIGroupVersion() + c.Assert(out, DeepEquals, tc.out) + } +} + +func (s *ProvisionerTestSuite) TestGetDriverNameFromUVSC(c *C) { + for _, tc := range []struct { + vsc unstructured.Unstructured + version string + out string + }{ + { // alpha success + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "snapshotter": "drivername", + }, + }, + version: "snapshot.storage.k8s.io/v1alpha1", + out: "drivername", + }, + { // key missing + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + version: "snapshot.storage.k8s.io/v1alpha1", + out: "", + }, + { // beta success + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "driver": "drivername", + }, + }, + version: "snapshot.storage.k8s.io/v1beta1", + out: "drivername", + }, + { // key missing + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + version: "snapshot.storage.k8s.io/v1beta1", + out: "", + }, + { // type conversion + vsc: unstructured.Unstructured{ + Object: map[string]interface{}{ + "driver": int64(1), + }, + }, + version: "snapshot.storage.k8s.io/v1beta1", + out: "", + }, + } { + p := &Kubestr{} + out := p.getDriverNameFromUVSC(tc.vsc, tc.version) + c.Assert(out, Equals, tc.out) + } +} + +// func (s *ProvisionerTestSuite) TestGetDriverStats(c *C) { +// var snapshotCount int +// var expansionCount int +// var cloningCount int +// featureMap := make(map[string]struct{}) +// for _, driver := range CSIDriverList { +// if strings.Contains("Snapshot", driver.Features) { +// snapshotCount++ +// } +// if strings.Contains("Expansion", driver.Features) { +// expansionCount++ +// } +// if strings.Contains("Cloning", driver.Features) { +// cloningCount++ +// } +// featureMap[driver.Features] = struct{}{} +// } +// c.Log("totalcsidrivers: ", len(CSIDriverList)) +// c.Log("snapshotCount: ", snapshotCount) +// c.Log("expansionCount: ", expansionCount) +// c.Log("cloningCount: ", cloningCount) +// c.Log("unique combinations: ", len(featureMap)) +// c.Assert(true, Equals, false) +// } diff --git a/pkg/kubestr/utils.go b/pkg/kubestr/utils.go new file mode 100644 index 0000000..1b3764d --- /dev/null +++ b/pkg/kubestr/utils.go @@ -0,0 +1,126 @@ +package kubestr + +import ( + "fmt" + "os" +) + +const ( + // ErrorColor formatted color red + ErrorColor = "\033[1;31m%s\033[0m" + // SuccessColor formatted color green + SuccessColor = "\033[1;32m%s\033[0m" + // YellowColor formatted color yellow + YellowColor = "\033[1;33m%s\033[0m" +) + +// Status is a generic structure to return a status +type Status struct { + StatusCode StatusCode + StatusMessage string + Raw interface{} `json:",omitempty"` +} + +// StatusCode type definition +type StatusCode string + +const ( + // StatusOK is the success status code + StatusOK = StatusCode("OK") + // StatusWarning is the informational status code + StatusWarning = StatusCode("Warning") + // StatusError is the failure status code + StatusError = StatusCode("Error") + // StatusInfo is the Info status code + StatusInfo = StatusCode("Info") +) + +// Print prints a status message with a given prefix +func (s *Status) Print(prefix string) { + switch s.StatusCode { + case StatusOK: + printSuccessMessage(prefix + s.StatusMessage) + case StatusError: + printErrorMessage(prefix + s.StatusMessage) + case StatusWarning: + printWarningMessage(prefix + s.StatusMessage) + default: + printInfoMessage(prefix + s.StatusMessage) + } +} + +// printErrorMessage prints the error message +func printErrorMessage(errorMesg string) { + fmt.Printf("%s - ", errorMesg) + fmt.Printf(ErrorColor, "Error") + fmt.Println() +} + +// printSuccessMessage prints the success message +func printSuccessMessage(message string) { + fmt.Printf("%s - ", message) + fmt.Printf(SuccessColor, "OK") + fmt.Println() +} + +func printSuccessColor(message string) { + fmt.Printf(SuccessColor, message) + fmt.Println() +} + +// printInfoMessage prints a warning +func printInfoMessage(message string) { + fmt.Println(message) +} + +// printWarningMessage prints a warning +func printWarningMessage(message string) { + fmt.Printf(YellowColor+"\n", message) +} + +// TestOutput is the generic return value for tests +type TestOutput struct { + TestName string + Status []Status + Raw interface{} `json:",omitempty"` +} + +// Print prints a TestRetVal as a string output +func (t *TestOutput) Print() { + fmt.Println(t.TestName + ":") + for _, status := range t.Status { + status.Print(" ") + } +} + +func MakeTestOutput(testname string, code StatusCode, mesg string, raw interface{}) *TestOutput { + return &TestOutput{ + TestName: testname, + Status: []Status{makeStatus(code, mesg, nil)}, + Raw: raw, + } +} + +func makeStatus(code StatusCode, mesg string, raw interface{}) Status { + return Status{ + StatusCode: code, + StatusMessage: mesg, + Raw: raw, + } +} + +func convertSetToSlice(set map[string]struct{}) []string { + var slice []string + for i := range set { + slice = append(slice, i) + } + return slice +} + +// getPodNamespace gets the pods namespace or returns default +func getPodNamespace() string { + if val, ok := os.LookupEnv(PodNamespaceEnvKey); ok { + return val + } + return DefaultNS +} diff --git a/scripts/load_csi_provisioners.sh b/scripts/load_csi_provisioners.sh new file mode 100755 index 0000000..1f2d7c4 --- /dev/null +++ b/scripts/load_csi_provisioners.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +current_directory=$(dirname "$0") +curl https://raw.githubusercontent.com/kubernetes-csi/docs/master/book/src/drivers.md -o ${current_directory}/../extra/csi-drivers + +cat <> ${current_directory}/../extra/csi-drivers-temp.go +package kubestr + +// THIS FILE IS AUTO_GENERATED. +// To generate file run "go generate" at the top level +// This file must be checked in. + +EOT + + +echo "var CSIDriverList = []*CSIDriver{" >> ${current_directory}/../extra/csi-drivers-temp.go +while read p; do + if [[ $p == [* ]]; + then + IFS='|' + read -a fields <<< "$p" + name_url=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[0]}) + driver_name=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[1]} | sed 's/`//g') + versions=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[2]}) + description=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[3]}) + persistence=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[4]}) + access_modes=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[5]}| sed 's/"//g') + dynamic_provisioning=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[6]}) + features=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<${fields[7]}) + echo "{NameUrl: \"$name_url\", DriverName: \"$driver_name\", Versions: \"$versions\", Description: \"$description\", Persistence: \"$persistence\", AccessModes: \"$access_modes\", DynamicProvisioning: \"$dynamic_provisioning\", Features: \"$features\"}," >> ${current_directory}/../extra/csi-drivers-temp.go + fi +done <${current_directory}/../extra/csi-drivers +echo "}" >> ${current_directory}/../extra/csi-drivers-temp.go + +gofmt ${current_directory}/../extra/csi-drivers-temp.go > ${current_directory}/../pkg/kubestr/csi-drivers.go +rm ${current_directory}/../extra/csi-drivers-temp.go