mirrors/kube-arangodb
1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00
Code Activity

[Feature] Improve JWT rotation (#587)

Browse source
This commit is contained in:
Adam Janikowski 2020-06-26 08:53:24 +02:00 committed by GitHub
parent b7114fa3d2
commit 490e8b80dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
60 changed files with 2571 additions and 481 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -8,6 +8,7 @@
- Improve Helm 3 support
- Allow to customize ID Pod selectors
- Add Label and Envs Pod customization
- Improved JWT Rotation
## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25)
- Prevent deletion of not known PVC's

1
lifecycle.go
View file

@ -62,6 +62,7 @@ func init() {
cmdMain.AddCommand(cmdLifecycle)
cmdLifecycle.AddCommand(cmdLifecyclePreStop)
cmdLifecycle.AddCommand(cmdLifecycleCopy)
cmdLifecycle.AddCommand(cmdLifecycleProbe)
cmdLifecycleCopy.Flags().StringVar(&lifecycleCopyOptions.TargetDir, "target", "", "Target directory to copy the executable to")
}

192
lifecycle_probes.go Normal file
View file

@ -0,0 +1,192 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"github.com/arangodb/go-driver/jwt"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)
var (
cmdLifecycleProbe = &cobra.Command{
Use: "probe",
Run: cmdLifecycleProbeCheck,
}
probeInput struct {
SSL bool
Auth bool
Endpoint string
JWTPath string
}
)
func init() {
f := cmdLifecycleProbe.PersistentFlags()
f.BoolVarP(&probeInput.SSL, "ssl", "", false, "Determines if SSL is enabled")
f.BoolVarP(&probeInput.Auth, "auth", "", false, "Determines if authentication is enabled")
f.StringVarP(&probeInput.Endpoint, "endpoint", "", "/_api/version", "Determines if SSL is enabled")
f.StringVarP(&probeInput.JWTPath, "jwt", "", k8sutil.ClusterJWTSecretVolumeMountDir, "Path to the JWT tokens")
}
func probeClient() *http.Client {
tr := &http.Transport{}
if probeInput.SSL {
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
client := &http.Client{
Transport: tr,
}
return client
}
func probeEndpoint(endpoint string) string {
proto := "http"
if probeInput.SSL {
proto = "https"
}
return fmt.Sprintf("%s://%s:%d%s", proto, "127.0.0.1", k8sutil.ArangoPort, endpoint)
}
func readJWTFile(file string) ([]byte, error) {
p := path.Join(probeInput.JWTPath, file)
log.Info().Str("path", p).Msgf("Try to use file")
f, err := os.Open(p)
if err != nil {
return nil, err
}
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
return data, nil
}
func getJWTToken() ([]byte, error) {
// Try read default one
if token, err := readJWTFile(constants.SecretKeyToken); err == nil {
log.Info().Str("token", constants.SecretKeyToken).Msgf("Using JWT Token")
return token, nil
}
// Try read active one
if token, err := readJWTFile(pod.ActiveJWTKey); err == nil {
log.Info().Str("token", pod.ActiveJWTKey).Msgf("Using JWT Token")
return token, nil
}
if files, err := ioutil.ReadDir(probeInput.JWTPath); err == nil {
for _, file := range files {
if token, err := readJWTFile(file.Name()); err == nil {
log.Info().Str("token", file.Name()).Msgf("Using JWT Token")
return token, nil
}
}
}
return nil, errors.Errorf("Unable to find any token")
}
func addAuthHeader(req *http.Request) error {
if !probeInput.Auth {
return nil
}
token, err := getJWTToken()
if err != nil {
return err
}
header, err := jwt.CreateArangodJwtAuthorizationHeader(string(token), "probe")
if err != nil {
return err
}
req.Header.Add("Authorization", header)
return nil
}
func doRequest() (*http.Response, error) {
client := probeClient()
req, err := http.NewRequest(http.MethodGet, probeEndpoint(probeInput.Endpoint), nil)
if err != nil {
return nil, err
}
if err := addAuthHeader(req); err != nil {
return nil, err
}
return client.Do(req)
}
func cmdLifecycleProbeCheck(cmd *cobra.Command, args []string) {
if err := cmdLifecycleProbeCheckE(); err != nil {
log.Error().Err(err).Msgf("Fatal")
os.Exit(1)
}
}
func cmdLifecycleProbeCheckE() error {
resp, err := doRequest()
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
if resp.Body != nil {
defer resp.Body.Close()
if data, err := ioutil.ReadAll(resp.Body); err == nil {
return errors.Errorf("Unexpected code: %d - %s", resp.StatusCode, string(data))
}
}
return errors.Errorf("Unexpected code: %d", resp.StatusCode)
}
log.Info().Msgf("Check passed")
return nil
}

18
pkg/apis/deployment/v1/hashes.go
View file

@ -22,12 +22,26 @@
package v1
import shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
type DeploymentStatusHashes struct {
Encryption DeploymentStatusHashList `json:"encryption,omitempty"`
Encryption DeploymentStatusHashesEncryption `json:"rocksDBEncryption,omitempty"`
TLS DeploymentStatusHashesTLS `json:"tls,omitempty"`
JWT DeploymentStatusHashesJWT `json:"jwt,omitempty"`
}
type DeploymentStatusHashesEncryption struct {
Keys shared.HashList `json:"keys,omitempty"`
}
type DeploymentStatusHashesTLS struct {
CA *string `json:"ca,omitempty"`
Truststore DeploymentStatusHashList `json:"truststore,omitempty"`
Truststore shared.HashList `json:"truststore,omitempty"`
}
type DeploymentStatusHashesJWT struct {
Active string `json:"active,omitempty"`
Passive shared.HashList `json:"passive,omitempty"`
Propagated bool `json:"propagated,omitempty"`
}

12
pkg/apis/deployment/v1/plan.go
View file

@ -101,6 +101,18 @@ const (
ActionTypeEncryptionKeyRefresh ActionType = "EncryptionKeyRefresh"
// ActionTypeEncryptionKeyStatusUpdate update status object with current encryption keys
ActionTypeEncryptionKeyStatusUpdate ActionType = "EncryptionKeyStatusUpdate"
// ActionTypeJWTStatusUpdate update status of JWT Secret
ActionTypeJWTStatusUpdate ActionType = "JWTStatusUpdate"
// ActionTypeJWTSetActive change active JWT key
ActionTypeJWTSetActive ActionType = "JWTSetActive"
// ActionTypeJWTAdd add new JWT key
ActionTypeJWTAdd ActionType = "JWTAdd"
// ActionTypeJWTClean Clean old JWT key
ActionTypeJWTClean ActionType = "JWTClean"
// ActionTypeJWTRefresh refresh jwt tokens
ActionTypeJWTRefresh ActionType = "JWTRefresh"
// ActionTypeJWTPropagated change propagated flag
ActionTypeJWTPropagated ActionType = "JWTPropagated"
)
const (

72
pkg/apis/deployment/v1/zz_generated.deepcopy.go generated
View file

@ -27,6 +27,7 @@ package v1
import (
time "time"
sharedv1 "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
@ -426,35 +427,12 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in DeploymentStatusHashList) DeepCopyInto(out *DeploymentStatusHashList) {
{
in := &in
*out = make(DeploymentStatusHashList, len(*in))
copy(*out, *in)
return
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashList.
func (in DeploymentStatusHashList) DeepCopy() DeploymentStatusHashList {
if in == nil {
return nil
}
out := new(DeploymentStatusHashList)
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) {
*out = *in
if in.Encryption != nil {
in, out := &in.Encryption, &out.Encryption
*out = make(DeploymentStatusHashList, len(*in))
copy(*out, *in)
}
in.Encryption.DeepCopyInto(&out.Encryption)
in.TLS.DeepCopyInto(&out.TLS)
in.JWT.DeepCopyInto(&out.JWT)
return
}
@ -468,6 +446,48 @@ func (in *DeploymentStatusHashes) DeepCopy() *DeploymentStatusHashes {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DeploymentStatusHashesEncryption) DeepCopyInto(out *DeploymentStatusHashesEncryption) {
*out = *in
if in.Keys != nil {
in, out := &in.Keys, &out.Keys
*out = make(sharedv1.HashList, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesEncryption.
func (in *DeploymentStatusHashesEncryption) DeepCopy() *DeploymentStatusHashesEncryption {
if in == nil {
return nil
}
out := new(DeploymentStatusHashesEncryption)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DeploymentStatusHashesJWT) DeepCopyInto(out *DeploymentStatusHashesJWT) {
*out = *in
if in.Passive != nil {
in, out := &in.Passive, &out.Passive
*out = make(sharedv1.HashList, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesJWT.
func (in *DeploymentStatusHashesJWT) DeepCopy() *DeploymentStatusHashesJWT {
if in == nil {
return nil
}
out := new(DeploymentStatusHashesJWT)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DeploymentStatusHashesTLS) DeepCopyInto(out *DeploymentStatusHashesTLS) {
*out = *in
@ -478,7 +498,7 @@ func (in *DeploymentStatusHashesTLS) DeepCopyInto(out *DeploymentStatusHashesTLS
}
if in.Truststore != nil {
in, out := &in.Truststore, &out.Truststore
*out = make(DeploymentStatusHashList, len(*in))
*out = make(sharedv1.HashList, len(*in))
copy(*out, *in)
}
return

55
pkg/deployment/client/client.go
View file

@ -41,6 +41,9 @@ type Client interface {
GetEncryption(ctx context.Context) (EncryptionDetails, error)
RefreshEncryption(ctx context.Context) (EncryptionDetails, error)
GetJWT(ctx context.Context) (JWTDetails, error)
RefreshJWT(ctx context.Context) (JWTDetails, error)
}
type client struct {
@ -75,6 +78,20 @@ func (c *client) parseEncryptionResponse(response driver.Response) (EncryptionDe
return d, nil
}
func (c *client) parseJWTResponse(response driver.Response) (JWTDetails, error) {
if err := response.CheckStatus(http.StatusOK); err != nil {
return JWTDetails{}, err
}
var d JWTDetails
if err := response.ParseBody("", &d); err != nil {
return JWTDetails{}, err
}
return d, nil
}
func (c *client) GetTLS(ctx context.Context) (TLSDetails, error) {
r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/tls")
if err != nil {
@ -150,3 +167,41 @@ func (c *client) RefreshEncryption(ctx context.Context) (EncryptionDetails, erro
return d, nil
}
func (c *client) GetJWT(ctx context.Context) (JWTDetails, error) {
r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/jwt")
if err != nil {
return JWTDetails{}, err
}
response, err := c.c.Do(ctx, r)
if err != nil {
return JWTDetails{}, err
}
d, err := c.parseJWTResponse(response)
if err != nil {
return JWTDetails{}, err
}
return d, nil
}
func (c *client) RefreshJWT(ctx context.Context) (JWTDetails, error) {
r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/jwt")
if err != nil {
return JWTDetails{}, err
}
response, err := c.c.Do(ctx, r)
if err != nil {
return JWTDetails{}, err
}
d, err := c.parseJWTResponse(response)
if err != nil {
return JWTDetails{}, err
}
return d, nil
}

25
pkg/deployment/client/encryption.go
View file

@ -22,33 +22,12 @@
package client
type EncryptionKeyEntry struct {
Sha string `json:"sha256,omitempty"`
}
type EncryptionDetailsResult struct {
Keys []EncryptionKeyEntry `json:"encryption-keys,omitempty"`
Keys Entries `json:"encryption-keys,omitempty"`
}
func (e EncryptionDetailsResult) KeysPresent(m map[string][]byte) bool {
if len(e.Keys) != len(m) {
return false
}
for key := range m {
ok := false
for _, entry := range e.Keys {
if entry.Sha == key {
ok = true
break
}
}
if !ok {
return false
}
}
return true
return e.Keys.KeysPresent(m)
}
type EncryptionDetails struct {

13
pkg/apis/deployment/v1/key_hashes.go → pkg/deployment/client/jwt.go
View file

@ -20,10 +20,13 @@
// Author Adam Janikowski
//
package v1
package client
import (
shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
)
type JWTDetailsResult struct {
Active *Entry `json:"active,omitempty"`
Passive Entries `json:"passive,omitempty"`
}
type DeploymentStatusHashList shared.HashList
type JWTDetails struct {
Result JWTDetailsResult `json:"result,omitempty"`
}

101
pkg/deployment/client/sha.go Normal file
View file

@ -0,0 +1,101 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package client
import (
"strings"
)
type Sha string
func (s Sha) String() string {
return string(s)
}
func (s Sha) Type() string {
z := strings.Split(s.String(), ":")
if len(z) < 2 {
return "sha256"
}
return z[0]
}
func (s Sha) Checksum() string {
z := strings.Split(s.String(), ":")
if len(z) < 2 {
return z[0]
}
return z[1]
}
type Entry struct {
Sha256 *Sha `json:"sha256,omitempty"`
Sha256Old *Sha `json:"SHA256,omitempty"`
}
func (e *Entry) GetSHA() Sha {
if e == nil {
return ""
}
if e.Sha256 != nil {
return *e.Sha256
}
if e.Sha256Old != nil {
return *e.Sha256Old
}
return ""
}
type Entries []Entry
func (e Entries) KeysPresent(m map[string][]byte) bool {
if len(e) != len(m) {
return false
}
for key := range m {
ok := false
for _, entry := range e {
if entry.GetSHA().Checksum() == key {
ok = true
break
}
}
if !ok {
return false
}
}
return true
}
func (e Entries) Contains(s string) bool {
for _, entry := range e {
if entry.GetSHA().String() == s {
return true
}
}
return false
}

3
pkg/deployment/client/tls.go
View file

@ -23,8 +23,9 @@
package client
type TLSKeyFile struct {
*Entry `json:",inline"`
PrivateKeyHash string `json:"privateKeySHA256,omitempty"`
Checksum string `json:"SHA256,omitempty"`
Certificates []string `json:"certificates,omitempty"`
}

118
pkg/deployment/client_cache.go
View file

@ -25,29 +25,76 @@ package deployment
import (
"context"
"fmt"
"net"
"strconv"
"sync"
"k8s.io/client-go/kubernetes"
"github.com/pkg/errors"
"github.com/arangodb/go-driver/agency"
"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
driver "github.com/arangodb/go-driver"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/arangod"
)
type clientCache struct {
mutex sync.Mutex
clients map[string]driver.Client
kubecli kubernetes.Interface
apiObject *api.ArangoDeployment
apiObjectGetter func() *api.ArangoDeployment
databaseClient driver.Client
factory conn.Factory
}
// newClientCache creates a new client cache
func newClientCache(kubecli kubernetes.Interface, apiObject *api.ArangoDeployment) *clientCache {
func newClientCache(apiObjectGetter func() *api.ArangoDeployment, factory conn.Factory) *clientCache {
return &clientCache{
clients: make(map[string]driver.Client),
kubecli: kubecli,
apiObject: apiObject,
apiObjectGetter: apiObjectGetter,
factory: factory,
}
}
func (cc *clientCache) extendHost(host string) string {
scheme := "http"
if cc.apiObjectGetter().Spec.TLS.IsSecure() {
scheme = "https"
}
return scheme + "://" + net.JoinHostPort(host, strconv.Itoa(k8sutil.ArangoPort))
}
func (cc *clientCache) getClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) {
key := fmt.Sprintf("%d-%s", group, id)
c, found := cc.clients[key]
if found {
return c, nil
}
// Not found, create a new client
c, err := cc.factory.Client(cc.extendHost(k8sutil.CreatePodDNSName(cc.apiObjectGetter(), group.AsRole(), id)))
if err != nil {
return nil, maskAny(err)
}
cc.clients[key] = c
return c, nil
}
func (cc *clientCache) get(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) {
client, err := cc.getClient(ctx, group, id)
if err != nil {
return nil, maskAny(err)
}
if _, err := client.Version(ctx); err == nil {
return client, nil
} else if driver.IsUnauthorized(err) {
delete(cc.clients, fmt.Sprintf("%d-%s", group, id))
return cc.getClient(ctx, group, id)
} else {
return client, nil
}
}
@ -57,37 +104,70 @@ func (cc *clientCache) Get(ctx context.Context, group api.ServerGroup, id string
cc.mutex.Lock()
defer cc.mutex.Unlock()
key := fmt.Sprintf("%d-%s", group, id)
c, found := cc.clients[key]
if found {
return cc.get(ctx, group, id)
}
func (cc *clientCache) getDatabaseClient() (driver.Client, error) {
if c := cc.databaseClient; c != nil {
return c, nil
}
// Not found, create a new client
c, err := arangod.CreateArangodClient(ctx, cc.kubecli.CoreV1(), cc.apiObject, group, id)
c, err := cc.factory.Client(cc.extendHost(k8sutil.CreateDatabaseClientServiceDNSName(cc.apiObjectGetter())))
if err != nil {
return nil, maskAny(err)
}
cc.clients[key] = c
cc.databaseClient = c
return c, nil
}
func (cc *clientCache) getDatabase(ctx context.Context) (driver.Client, error) {
client, err := cc.getDatabaseClient()
if err != nil {
return nil, maskAny(err)
}
if _, err := client.Version(ctx); err == nil {
return client, nil
} else if driver.IsUnauthorized(err) {
cc.databaseClient = nil
return cc.getDatabaseClient()
} else {
return client, nil
}
}
// GetDatabase returns a cached client for the entire database (cluster coordinators or single server),
// creating one if needed.
func (cc *clientCache) GetDatabase(ctx context.Context) (driver.Client, error) {
cc.mutex.Lock()
defer cc.mutex.Unlock()
if c := cc.databaseClient; c != nil {
return c, nil
return cc.getDatabase(ctx)
}
func (cc *clientCache) getAgencyClient() (agency.Agency, error) {
// Not found, create a new client
var dnsNames []string
for _, m := range cc.apiObjectGetter().Status.Members.Agents {
dnsNames = append(dnsNames, cc.extendHost(k8sutil.CreatePodDNSName(cc.apiObjectGetter(), api.ServerGroupAgents.AsRole(), m.ID)))
}
// Not found, create a new client
shortTimeout := false
c, err := arangod.CreateArangodDatabaseClient(ctx, cc.kubecli.CoreV1(), cc.apiObject, shortTimeout)
if len(dnsNames) == 0 {
return nil, errors.Errorf("There is no DNS Name")
}
c, err := cc.factory.Agency(dnsNames...)
if err != nil {
return nil, maskAny(err)
}
cc.databaseClient = c
return c, nil
}
// GetDatabase returns a cached client for the agency
func (cc *clientCache) GetAgency(ctx context.Context) (agency.Agency, error) {
cc.mutex.Lock()
defer cc.mutex.Unlock()
return cc.getAgencyClient()
}

109
pkg/deployment/context_impl.go
View file

@ -24,9 +24,18 @@ package deployment
import (
"context"
"crypto/tls"
"fmt"
"net"
nhttp "net/http"
"strconv"
"time"
"github.com/arangodb/go-driver/http"
"github.com/arangodb/go-driver/jwt"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
goErrors "github.com/pkg/errors"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@ -37,20 +46,19 @@ import (
driver "github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/agency"
"github.com/rs/zerolog/log"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/util/arangod"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
v1 "k8s.io/api/core/v1"
)
// GetBackup receives information about a backup resource
func (d *Deployment) GetBackup(backup string) (*backupApi.ArangoBackup, error) {
return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(backup, metav1.GetOptions{})
return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(backup, meta.GetOptions{})
}
// GetAPIObject returns the deployment as k8s object.
@ -198,11 +206,84 @@ func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id str
// GetAgency returns a connection to the entire agency.
func (d *Deployment) GetAgency(ctx context.Context) (agency.Agency, error) {
result, err := arangod.CreateArangodAgencyClient(ctx, d.deps.KubeCli.CoreV1(), d.apiObject)
return d.clientCache.GetAgency(ctx)
}
func (d *Deployment) getConnConfig() (http.ConnectionConfig, error) {
transport := &nhttp.Transport{
Proxy: nhttp.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 100 * time.Millisecond,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 100 * time.Millisecond,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
if d.apiObject.Spec.TLS.IsSecure() {
transport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
}
}
connConfig := http.ConnectionConfig{
Transport: transport,
DontFollowRedirect: true,
}
return connConfig, nil
}
func (d *Deployment) getAuth() (driver.Authentication, error) {
if !d.apiObject.Spec.Authentication.IsAuthenticated() {
return nil, nil
}
secrets := d.GetKubeCli().CoreV1().Secrets(d.apiObject.GetNamespace())
var secret string
if i := d.apiObject.Status.CurrentImage; i == nil || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || !i.Enterprise {
s, err := secrets.Get(d.apiObject.Spec.Authentication.GetJWTSecretName(), meta.GetOptions{})
if err != nil {
return nil, goErrors.Errorf("JWT Secret is missing")
}
jwt, ok := s.Data[constants.SecretKeyToken]
if !ok {
return nil, goErrors.Errorf("JWT Secret is invalid")
}
secret = string(jwt)
} else {
s, err := secrets.Get(pod.JWTSecretFolder(d.apiObject.GetName()), meta.GetOptions{})
if err != nil {
d.deps.Log.Error().Err(err).Msgf("Unable to get secret")
return nil, goErrors.Errorf("JWT Folder Secret is missing")
}
if len(s.Data) == 0 {
return nil, goErrors.Errorf("JWT Folder Secret is empty")
}
if q, ok := s.Data[pod.ActiveJWTKey]; ok {
secret = string(q)
} else {
for _, q := range s.Data {
secret = string(q)
break
}
}
}
jwt, err := jwt.CreateArangodJwtAuthorizationHeader(secret, "kube-arangodb")
if err != nil {
return nil, maskAny(err)
}
return result, nil
return driver.RawAuthentication(jwt), nil
}
// GetSyncServerClient returns a cached client for a specific arangosync server.
@ -268,7 +349,7 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err
func (d *Deployment) DeletePod(podName string) error {
log := d.deps.Log
ns := d.apiObject.GetNamespace()
if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod")
return maskAny(err)
}
@ -281,8 +362,8 @@ func (d *Deployment) CleanupPod(p *v1.Pod) error {
log := d.deps.Log
podName := p.GetName()
ns := p.GetNamespace()
options := metav1.NewDeleteOptions(0)
options.Preconditions = metav1.NewUIDPreconditions(string(p.GetUID()))
options := meta.NewDeleteOptions(0)
options.Preconditions = meta.NewUIDPreconditions(string(p.GetUID()))
if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(podName, options); err != nil && !k8sutil.IsNotFound(err) {
log.Debug().Err(err).Str("pod", podName).Msg("Failed to cleanup pod")
return maskAny(err)
@ -296,7 +377,7 @@ func (d *Deployment) RemovePodFinalizers(podName string) error {
log := d.deps.Log
ns := d.GetNamespace()
kubecli := d.deps.KubeCli
p, err := kubecli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
p, err := kubecli.CoreV1().Pods(ns).Get(podName, meta.GetOptions{})
if err != nil {
if k8sutil.IsNotFound(err) {
return nil
@ -314,7 +395,7 @@ func (d *Deployment) RemovePodFinalizers(podName string) error {
func (d *Deployment) DeletePvc(pvcName string) error {
log := d.deps.Log
ns := d.apiObject.GetNamespace()
if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
log.Debug().Err(err).Str("pvc", pvcName).Msg("Failed to remove pvc")
return maskAny(err)
}
@ -338,7 +419,7 @@ func (d *Deployment) UpdatePvc(pvc *v1.PersistentVolumeClaim) error {
// GetPv returns PV info about PV with given name.
func (d *Deployment) GetPv(pvName string) (*v1.PersistentVolume, error) {
pv, err := d.GetKubeCli().CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
pv, err := d.GetKubeCli().CoreV1().PersistentVolumes().Get(pvName, meta.GetOptions{})
if err == nil {
return pv, nil
}
@ -366,7 +447,7 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) {
// GetPvc gets a PVC by the given name, in the samespace of the deployment.
func (d *Deployment) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) {
pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(pvcName, metav1.GetOptions{})
pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(pvcName, meta.GetOptions{})
if err != nil {
log.Debug().Err(err).Str("pvc-name", pvcName).Msg("Failed to get PVC")
return nil, maskAny(err)
@ -392,7 +473,7 @@ func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatu
func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error {
secretName := k8sutil.CreateTLSKeyfileSecretName(d.apiObject.GetName(), group.AsRole(), member.ID)
ns := d.apiObject.GetNamespace()
if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
return maskAny(err)
}
return nil
@ -402,7 +483,7 @@ func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberSt
// If the secret does not exist, the error is ignored.
func (d *Deployment) DeleteSecret(secretName string) error {
ns := d.apiObject.GetNamespace()
if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
return maskAny(err)
}
return nil

11
pkg/deployment/deployment.go
View file

@ -29,6 +29,8 @@ import (
"sync/atomic"
"time"
"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
"github.com/arangodb/kube-arangodb/pkg/util/arangod"
@ -123,14 +125,17 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
if err := apiObject.Spec.Validate(); err != nil {
return nil, maskAny(err)
}
d := &Deployment{
apiObject: apiObject,
config: config,
deps: deps,
eventCh: make(chan *deploymentEvent, deploymentEventQueueSize),
stopCh: make(chan struct{}),
clientCache: newClientCache(deps.KubeCli, apiObject),
}
d.clientCache = newClientCache(d.getArangoDeployment, conn.NewFactory(d.getAuth, d.getConnConfig))
d.status.last = *(apiObject.Status.DeepCopy())
d.reconciler = reconcile.NewReconciler(deps.Log, d)
d.resilience = resilience.NewResilience(deps.Log, d)
@ -491,3 +496,7 @@ func (d *Deployment) SetNumberOfServers(ctx context.Context, noCoordinators, noD
}
return nil
}
func (d *Deployment) getArangoDeployment() *api.ArangoDeployment {
return d.apiObject
}

18
pkg/deployment/deployment_affinity_test.go
View file

@ -98,7 +98,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -160,7 +160,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -225,7 +225,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -295,7 +295,7 @@ func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -374,7 +374,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -439,7 +439,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -507,7 +507,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -580,7 +580,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -661,7 +661,7 @@ func TestEnsurePod_ArangoDB_NodeAffinity(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

50
pkg/deployment/deployment_core_test.go
View file

@ -76,7 +76,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullAlways,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -127,7 +127,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullAlways,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -187,7 +187,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -244,7 +244,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -309,7 +309,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -364,7 +364,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -420,7 +420,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -477,7 +477,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -536,7 +536,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -588,7 +588,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -643,7 +643,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -696,7 +696,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.TlsKeyfileVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, true, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -735,7 +735,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(false,
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, false,
authorization, k8sutil.ArangoPort)
},
ExpectedEvent: "member agent is created",
@ -795,7 +795,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true,
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true,
authorization, k8sutil.ArangoPort)
},
ExpectedEvent: "member agent is created",
@ -875,7 +875,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.RocksdbEncryptionVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -927,7 +927,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -981,7 +981,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -1043,7 +1043,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -1116,7 +1116,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
},
Resources: emptyResources,
Lifecycle: createTestLifecycle(),
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -1188,7 +1188,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
},
Resources: emptyResources,
Lifecycle: createTestLifecycle(),
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -1238,7 +1238,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true,
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true,
authorization, k8sutil.ArangoPort)
},
config: Config{
@ -1273,7 +1273,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
},
Ports: createTestPorts(),
Lifecycle: createTestLifecycle(),
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
VolumeMounts: []core.VolumeMount{
@ -1327,7 +1327,7 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
auth, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, auth)
testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(cmd, true, auth)
},
ExpectedEvent: "member coordinator is created",
ExpectedPod: core.Pod{
@ -1391,9 +1391,9 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) {
authReadiness, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true,
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true,
authLiveness, 0)
testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, authReadiness)
testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(cmd, true, authReadiness)
},
ExpectedEvent: "member single is created",
ExpectedPod: core.Pod{

8
pkg/deployment/deployment_encryption_test.go
View file

@ -87,7 +87,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
k8sutil.RocksdbEncryptionVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -136,7 +136,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"})
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true,
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(cmd, true,
authorization, k8sutil.ArangoPort)
},
config: Config{
@ -171,7 +171,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
},
Ports: createTestPorts(),
Lifecycle: createTestLifecycle(),
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
VolumeMounts: []core.VolumeMount{
@ -245,7 +245,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
k8sutil.RocksdbEncryptionReadOnlyVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

6
pkg/deployment/deployment_image_test.go
View file

@ -91,7 +91,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullAlways,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -143,7 +143,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullAlways,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -195,7 +195,7 @@ func TestEnsurePod_ArangoDB_ImagePropagation(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullAlways,
SecurityContext: securityContext.NewSecurityContext(),
},

4
pkg/deployment/deployment_inspector.go
View file

@ -174,9 +174,9 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
}
// Ensure we have image info
if retrySoon, err := d.ensureImages(d.apiObject); err != nil {
if retrySoon, exists, err := d.ensureImages(d.apiObject); err != nil {
return minInspectionInterval, errors.Wrapf(err, "Image detection failed")
} else if retrySoon {
} else if retrySoon || !exists {
return minInspectionInterval, nil
}

12
pkg/deployment/deployment_metrics_test.go
View file

@ -81,7 +81,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -142,7 +142,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -212,7 +212,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -281,7 +281,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -343,7 +343,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -410,7 +410,7 @@ func TestEnsurePod_Metrics(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

20
pkg/deployment/deployment_pod_probe_test.go
View file

@ -69,7 +69,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -126,7 +126,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: modTestLivenessProbe(false, "", k8sutil.ArangoPort, func(probe *core.Probe) {
LivenessProbe: modTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort, func(probe *core.Probe) {
probe.TimeoutSeconds = 50
}),
ImagePullPolicy: core.PullIfNotPresent,
@ -184,8 +184,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessSimpleProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessSimpleProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -235,7 +235,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -291,8 +291,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessSimpleProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessSimpleProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -342,7 +342,7 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
ReadinessProbe: createTestReadinessProbe(false, ""),
ReadinessProbe: createTestReadinessProbe(cmd, false, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -398,8 +398,8 @@ func TestEnsurePod_ArangoDB_Probe(t *testing.T) {
k8sutil.ArangodVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessProbe(false, ""),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ReadinessProbe: createTestReadinessProbe(cmd, false, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

6
pkg/deployment/deployment_pod_resources_test.go
View file

@ -88,7 +88,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -148,7 +148,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) {
Env: []core.EnvVar{
resourceLimitAsEnv(t, resourcesUnfiltered),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -204,7 +204,7 @@ func TestEnsurePod_ArangoDB_Resources(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

6
pkg/deployment/deployment_pod_sync_test.go
View file

@ -228,7 +228,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) {
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(
true, "bearer "+auth, k8sutil.ArangoSyncMasterPort)
"", true, "bearer "+auth, k8sutil.ArangoSyncMasterPort)
},
ExpectedEvent: "member syncmaster is created",
ExpectedPod: core.Pod{
@ -307,7 +307,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) {
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(
true, "bearer "+auth, k8sutil.ArangoSyncMasterPort)
"", true, "bearer "+auth, k8sutil.ArangoSyncMasterPort)
},
ExpectedEvent: "member syncmaster is created",
ExpectedPod: core.Pod{
@ -409,7 +409,7 @@ func TestEnsurePod_Sync_Worker(t *testing.T) {
require.NoError(t, err)
testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(
true, "bearer "+auth, k8sutil.ArangoSyncWorkerPort)
"", true, "bearer "+auth, k8sutil.ArangoSyncWorkerPort)
},
ExpectedEvent: "member syncworker is created",
ExpectedPod: core.Pod{

10
pkg/deployment/deployment_pod_tls_sni_test.go
View file

@ -116,7 +116,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) {
k8sutil.TlsKeyfileVolumeMount(),
},
Resources: emptyResources,
ReadinessProbe: createTestReadinessProbe(true, ""),
ReadinessProbe: createTestReadinessProbe(cmd, true, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -188,7 +188,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) {
k8sutil.TlsKeyfileVolumeMount(),
},
Resources: emptyResources,
ReadinessProbe: createTestReadinessProbe(true, ""),
ReadinessProbe: createTestReadinessProbe(cmd, true, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -260,7 +260,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) {
k8sutil.TlsKeyfileVolumeMount(),
},
Resources: emptyResources,
ReadinessProbe: createTestReadinessProbe(true, ""),
ReadinessProbe: createTestReadinessProbe(cmd, true, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -365,7 +365,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) {
},
},
Resources: emptyResources,
ReadinessProbe: createTestReadinessProbe(true, ""),
ReadinessProbe: createTestReadinessProbe(cmd, true, ""),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -440,7 +440,7 @@ func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) {
k8sutil.TlsKeyfileVolumeMount(),
},
Resources: emptyResources,
LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, true, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

6
pkg/deployment/deployment_pod_volumes_test.go
View file

@ -96,7 +96,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -157,7 +157,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) {
VolumeMounts: []core.VolumeMount{
k8sutil.ArangodVolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},
@ -221,7 +221,7 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) {
k8sutil.ArangodVolumeMount(),
createExampleVolumeMount("volume").VolumeMount(),
},
LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort),
LivenessProbe: createTestLivenessProbe(cmd, false, "", k8sutil.ArangoPort),
ImagePullPolicy: core.PullIfNotPresent,
SecurityContext: securityContext.NewSecurityContext(),
},

97
pkg/deployment/deployment_suite_test.go
View file

@ -26,8 +26,13 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
"github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/jwt"
@ -96,16 +101,16 @@ func createTestToken(deployment *Deployment, testCase *testCaseStruct, paths []s
return jwt.CreateArangodJwtAuthorizationHeaderAllowedPaths(s, "kube-arangodb", paths)
}
func modTestLivenessProbe(secure bool, authorization string, port int, mod func(*core.Probe)) *core.Probe {
probe := createTestLivenessProbe(secure, authorization, port)
func modTestLivenessProbe(mode string, secure bool, authorization string, port int, mod func(*core.Probe)) *core.Probe {
probe := createTestLivenessProbe(mode, secure, authorization, port)
mod(probe)
return probe
}
func createTestReadinessSimpleProbe(secure bool, authorization string, port int) *core.Probe {
probe := createTestLivenessProbe(secure, authorization, port)
func createTestReadinessSimpleProbe(mode string, secure bool, authorization string, port int) *core.Probe {
probe := createTestReadinessProbe(mode, secure, authorization)
probe.InitialDelaySeconds = 15
probe.PeriodSeconds = 10
@ -113,23 +118,75 @@ func createTestReadinessSimpleProbe(secure bool, authorization string, port int)
return probe
}
func createTestLivenessProbe(secure bool, authorization string, port int) *core.Probe {
return k8sutil.HTTPProbeConfig{
LocalPath: "/_api/version",
func createTestLivenessProbe(mode string, secure bool, authorization string, port int) *core.Probe {
return getProbeCreator(mode)(secure, authorization, "/_api/version", port).Create()
}
func createTestReadinessProbe(mode string, secure bool, authorization string) *core.Probe {
p := getProbeCreator(mode)(secure, authorization, "/_admin/server/availability", k8sutil.ArangoPort).Create()
p.InitialDelaySeconds = 2
p.PeriodSeconds = 2
return p
}
type probeCreator func(secure bool, authorization, endpoint string, port int) resources.Probe
const (
cmd = "cmd"
)
func getProbeCreator(t string) probeCreator {
switch t {
case cmd:
return getCMDProbeCreator()
default:
return getHTTPProbeCreator()
}
}
func getHTTPProbeCreator() probeCreator {
return func(secure bool, authorization, endpoint string, port int) resources.Probe {
return createHTTPTestProbe(secure, authorization, endpoint, port)
}
}
func getCMDProbeCreator() probeCreator {
return func(secure bool, authorization, endpoint string, port int) resources.Probe {
return createCMDTestProbe(secure, authorization != "", endpoint)
}
}
func createCMDTestProbe(secure, authorization bool, endpoint string) resources.Probe {
bin, _ := os.Executable()
args := []string{
filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(bin)),
"lifecycle",
"probe",
fmt.Sprintf("--endpoint=%s", endpoint),
}
if secure {
args = append(args, "--ssl")
}
if authorization {
args = append(args, "--auth")
}
return &probes.CMDProbeConfig{
Command: args,
}
}
func createHTTPTestProbe(secure bool, authorization string, endpoint string, port int) resources.Probe {
return &probes.HTTPProbeConfig{
LocalPath: endpoint,
Secure: secure,
Authorization: authorization,
Port: port,
}.Create()
}
func createTestReadinessProbe(secure bool, authorization string) *core.Probe {
return k8sutil.HTTPProbeConfig{
LocalPath: "/_admin/server/availability",
Secure: secure,
Authorization: authorization,
InitialDelaySeconds: 2,
PeriodSeconds: 2,
}.Create()
}
}
func createTestCommandForDBServer(name string, tls, auth, encryptionRocksDB bool) []string {
@ -375,8 +432,8 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment)
deps: deps,
eventCh: make(chan *deploymentEvent, deploymentEventQueueSize),
stopCh: make(chan struct{}),
clientCache: newClientCache(deps.KubeCli, arangoDeployment),
}
d.clientCache = newClientCache(d.getArangoDeployment, conn.NewFactory(d.getAuth, d.getConnConfig))
arangoDeployment.Spec.SetDefaults(arangoDeployment.GetName())
d.resources = resources.NewResources(deps.Log, d)
@ -444,7 +501,7 @@ func createTestExporterCommand(secure bool, port uint16) []string {
}
func createTestExporterLivenessProbe(secure bool) *core.Probe {
return k8sutil.HTTPProbeConfig{
return probes.HTTPProbeConfig{
LocalPath: "/",
Port: k8sutil.ArangoExporterPort,
Secure: secure,

17
pkg/deployment/images.go
View file

@ -71,7 +71,7 @@ type imagesBuilder struct {
// ensureImages creates pods needed to detect ImageID for specified images.
// Returns: retrySoon, error
func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, error) {
func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, error) {
status, lastVersion := d.GetStatus()
ib := imagesBuilder{
APIObject: apiObject,
@ -87,29 +87,28 @@ func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, error)
},
}
ctx := context.Background()
retrySoon, err := ib.Run(ctx)
retrySoon, exists, err := ib.Run(ctx)
if err != nil {
return retrySoon, maskAny(err)
return retrySoon, exists, maskAny(err)
}
return retrySoon, nil
return retrySoon, exists, nil
}
// Run creates pods needed to detect ImageID for specified images and puts the found
// image ID's into the status.Images list.
// Returns: retrySoon, error
func (ib *imagesBuilder) Run(ctx context.Context) (bool, error) {
result := false
func (ib *imagesBuilder) Run(ctx context.Context) (bool, bool, error) {
// Check ArangoDB image
if _, found := ib.Status.Images.GetByImage(ib.Spec.GetImage()); !found {
// We need to find the image ID for the ArangoDB image
retrySoon, err := ib.fetchArangoDBImageIDAndVersion(ctx, ib.Spec.GetImage())
if err != nil {
return retrySoon, maskAny(err)
return retrySoon, false, maskAny(err)
}
result = result || retrySoon
return retrySoon, false, nil
}
return result, nil
return false, true, nil
}
// fetchArangoDBImageIDAndVersion checks a running pod for fetching the ID of the given image.

2
pkg/deployment/images_test.go
View file

@ -330,7 +330,7 @@ func TestEnsureImages(t *testing.T) {
require.NoError(t, err)
// Act
retrySoon, err := d.ensureImages(d.apiObject)
retrySoon, _, err := d.ensureImages(d.apiObject)
// Assert
assert.EqualValues(t, testCase.RetrySoon, retrySoon)

14
pkg/deployment/pod/encryption.go
View file

@ -95,7 +95,7 @@ func GetEncryptionKeyFromSecret(keyfile *core.Secret) (string, []byte, error) {
return sha, d, nil
}
func GetKeyfolderSecretName(name string) string {
func GetEncryptionFolderSecretName(name string) string {
n := fmt.Sprintf("%s-encryption-folder", name)
return n
@ -145,7 +145,7 @@ func (e encryption) Volumes(i Input) ([]core.Volume, []core.VolumeMount) {
vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, i.Deployment.RocksDB.Encryption.GetKeySecretName())
return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionVolumeMount()}
} else {
vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, GetKeyfolderSecretName(i.ApiObject.GetName()))
vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, GetEncryptionFolderSecretName(i.ApiObject.GetName()))
return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionReadOnlyVolumeMount()}
}
}
@ -155,17 +155,21 @@ func (e encryption) Verify(i Input, cachedStatus inspector.Inspector) error {
return nil
}
if !GroupEncryptionSupported(i.Deployment.GetMode(), i.Group) {
return nil
}
if !MultiFileMode(i) {
secret, exists := cachedStatus.Secret(i.Deployment.RocksDB.Encryption.GetKeySecretName())
if !exists {
return errors.Errorf("Encryption key secret does not exist %s", i.Deployment.RocksDB.Encryption.GetKeySecretName())
}
if !MultiFileMode(i) {
if err := k8sutil.ValidateEncryptionKeyFromSecret(secret); err != nil {
return errors.Wrapf(err, "RocksDB encryption key secret validation failed")
}
return nil
} else {
return nil
}
return nil
}

24
pkg/deployment/pod/jwt.go
View file

@ -23,6 +23,7 @@
package pod
import (
"fmt"
"path/filepath"
"github.com/arangodb/go-driver"
@ -33,6 +34,8 @@ import (
core "k8s.io/api/core/v1"
)
const ActiveJWTKey = "-"
func IsAuthenticated(i Input) bool {
return i.Deployment.IsAuthenticated()
}
@ -48,8 +51,12 @@ func VersionHasJWTSecretKeyfile(v driver.Version) bool {
return false
}
func VersionHasJWTSecretKeyfolder(i Input) bool {
return i.Enterprise && i.Version.CompareTo("3.7.0") > 0
func JWTSecretFolder(name string) string {
return fmt.Sprintf("%s-jwt-folder", name)
}
func VersionHasJWTSecretKeyfolder(v driver.Version, enterprise bool) bool {
return enterprise && v.CompareTo("3.7.0") > 0
}
func JWT() Builder {
@ -81,7 +88,9 @@ func (e jwt) Args(i Input) k8sutil.OptionPairs {
options.Add("--server.authentication", "true")
if VersionHasJWTSecretKeyfile(i.Version) {
if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) {
options.Add("--server.jwt-secret-folder", k8sutil.ClusterJWTSecretVolumeMountDir)
} else if VersionHasJWTSecretKeyfile(i.Version) {
keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken)
options.Add("--server.jwt-secret-keyfile", keyPath)
} else {
@ -96,7 +105,12 @@ func (e jwt) Volumes(i Input) ([]core.Volume, []core.VolumeMount) {
return nil, nil
}
vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName())
var vol core.Volume
if VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) {
vol = k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, JWTSecretFolder(i.ApiObject.GetName()))
} else {
vol = k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName())
}
return []core.Volume{vol}, []core.VolumeMount{k8sutil.ClusterJWTVolumeMount()}
}
@ -105,6 +119,7 @@ func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error {
return nil
}
if !VersionHasJWTSecretKeyfolder(i.Version, i.Enterprise) {
secret, exists := cachedStatus.Secret(i.Deployment.Authentication.GetJWTSecretName())
if !exists {
return errors.Errorf("Secret for JWT token is missing %s", i.Deployment.Authentication.GetJWTSecretName())
@ -113,6 +128,7 @@ func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error {
if err := k8sutil.ValidateTokenFromSecret(secret); err != nil {
return errors.Wrapf(err, "Cluster JWT secret validation failed")
}
}
return nil
}

2
pkg/deployment/reconcile/action_encryption_add.go
View file

@ -101,7 +101,7 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) {
return true, nil
}
_, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
_, err = a.actionCtx.SecretsInterface().Patch(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
if err != nil {
return false, err
}

2
pkg/deployment/reconcile/action_encryption_refresh.go
View file

@ -54,7 +54,7 @@ func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) {
}
func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) {
keyfolder, err := a.actionCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(a.actionCtx.GetName()), meta.GetOptions{})
keyfolder, err := a.actionCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{})
if err != nil {
a.log.Err(err).Msgf("Unable to fetch encryption folder")
return true, false, nil

2
pkg/deployment/reconcile/action_encryption_remove.go
View file

@ -77,7 +77,7 @@ func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) {
return true, nil
}
_, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
_, err = a.actionCtx.SecretsInterface().Patch(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
if err != nil {
if !k8sutil.IsInvalid(err) {
return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch))

10
pkg/deployment/reconcile/action_encryption_status_update.go
View file

@ -57,7 +57,7 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro
return true, nil
}
f, err := a.actionCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{})
f, err := a.actionCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{})
if err != nil {
a.log.Error().Err(err).Msgf("Unable to get folder info")
return true, nil
@ -67,16 +67,16 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro
if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
if len(keyHashes) == 0 {
if s.Hashes.Encryption != nil {
s.Hashes.Encryption = nil
if s.Hashes.Encryption.Keys != nil {
s.Hashes.Encryption.Keys = nil
return true
}
return false
}
if !util.CompareStringArray(keyHashes, s.Hashes.Encryption) {
s.Hashes.Encryption = keyHashes
if !util.CompareStringArray(keyHashes, s.Hashes.Encryption.Keys) {
s.Hashes.Encryption.Keys = keyHashes
return true
}
return false

124
pkg/deployment/reconcile/action_jwt_add.go Normal file
View file

@ -0,0 +1,124 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"encoding/base64"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
"k8s.io/apimachinery/pkg/types"
)
func init() {
registerAction(api.ActionTypeJWTAdd, newJWTAdd)
}
func newJWTAdd(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtAddAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtAddAction struct {
actionImpl
actionEmptyCheckProgress
}
func (a *jwtAddAction) Start(ctx context.Context) (bool, error) {
folder, err := ensureJWTFolderSupportFromAction(a.actionCtx)
if err != nil {
a.log.Error().Err(err).Msgf("Action not supported")
return true, nil
}
if !folder {
a.log.Error().Msgf("Action not supported")
return true, nil
}
appendToken, exists := a.action.Params[checksum]
if !exists {
a.log.Warn().Msgf("Key %s is missing in action", checksum)
return true, nil
}
s, ok := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().Authentication.GetJWTSecretName())
if !ok {
a.log.Error().Msgf("JWT Secret is missing, no rotation will take place")
return true, nil
}
jwt, ok := s.Data[constants.SecretKeyToken]
if !ok {
a.log.Error().Msgf("JWT Secret is invalid, no rotation will take place")
return true, nil
}
jwtSha := util.SHA256(jwt)
if appendToken != jwtSha {
a.log.Error().Msgf("JWT Secret changed")
return true, nil
}
f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName()))
if !ok {
a.log.Error().Msgf("Unable to get JWT folder info")
return true, nil
}
if _, ok := f.Data[jwtSha]; ok {
a.log.Info().Msgf("JWT Already exists")
return true, nil
}
p := patch.NewPatch()
p.ItemAdd(patch.NewPath("data", jwtSha), base64.StdEncoding.EncodeToString(jwt))
patch, err := p.Marshal()
if err != nil {
a.log.Error().Err(err).Msgf("Unable to encrypt patch")
return true, nil
}
_, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch)
if err != nil {
if !k8sutil.IsInvalid(err) {
return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName()))
}
}
return true, nil
}

115
pkg/deployment/reconcile/action_jwt_clean.go Normal file
View file

@ -0,0 +1,115 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
"k8s.io/apimachinery/pkg/types"
)
func init() {
registerAction(api.ActionTypeJWTClean, newJWTClean)
}
func newJWTClean(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtCleanAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtCleanAction struct {
actionImpl
actionEmptyCheckProgress
}
func (a *jwtCleanAction) Start(ctx context.Context) (bool, error) {
folder, err := ensureJWTFolderSupportFromAction(a.actionCtx)
if err != nil {
a.log.Error().Err(err).Msgf("Action not supported")
return true, nil
}
if !folder {
a.log.Error().Msgf("Action not supported")
return true, nil
}
cleanToken, exists := a.action.Params[checksum]
if !exists {
a.log.Warn().Msgf("Key %s is missing in action", checksum)
return true, nil
}
if cleanToken == pod.ActiveJWTKey {
a.log.Error().Msgf("Unable to remove active key")
return true, nil
}
f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName()))
if !ok {
a.log.Error().Msgf("Unable to get JWT folder info")
return true, nil
}
if key, ok := f.Data[pod.ActiveJWTKey]; !ok {
a.log.Info().Msgf("Active Key is required")
return true, nil
} else if util.SHA256(key) == cleanToken {
a.log.Info().Msgf("Unable to remove active key")
return true, nil
}
if _, ok := f.Data[cleanToken]; !ok {
a.log.Info().Msgf("KEy to be removed does not exist")
return true, nil
}
p := patch.NewPatch()
p.ItemRemove(patch.NewPath("data", cleanToken))
patch, err := p.Marshal()
if err != nil {
a.log.Error().Err(err).Msgf("Unable to encrypt patch")
return true, nil
}
_, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch)
if err != nil {
if !k8sutil.IsInvalid(err) {
return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName()))
}
}
return true, nil
}

77
pkg/deployment/reconcile/action_jwt_propagated.go Normal file
View file

@ -0,0 +1,77 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/rs/zerolog"
)
func init() {
registerAction(api.ActionTypeJWTPropagated, newJWTPropagated)
}
func newJWTPropagated(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtPropagatedAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtPropagatedAction struct {
actionImpl
actionEmptyCheckProgress
}
func (a *jwtPropagatedAction) Start(ctx context.Context) (bool, error) {
_, err := ensureJWTFolderSupportFromAction(a.actionCtx)
if err != nil {
a.log.Error().Err(err).Msgf("Action not supported")
return true, nil
}
propagatedFlag, exists := a.action.Params[propagated]
if !exists {
a.log.Error().Err(err).Msgf("Propagated flag is missing")
return true, nil
}
propagatedFlagBool := propagatedFlag == conditionTrue
if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
if s.Hashes.JWT.Propagated != propagatedFlagBool {
s.Hashes.JWT.Propagated = propagatedFlagBool
return true
}
return false
}); err != nil {
return false, err
}
return true, nil
}

78
pkg/deployment/reconcile/action_jwt_refresh.go Normal file
View file

@ -0,0 +1,78 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/client"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/rs/zerolog"
)
func init() {
registerAction(api.ActionTypeJWTRefresh, newJWTRefresh)
}
func newJWTRefresh(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtRefreshAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtRefreshAction struct {
actionImpl
}
func (a *jwtRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) {
if folder, err := ensureJWTFolderSupport(a.actionCtx.GetSpec(), a.actionCtx.GetStatus()); err != nil || !folder {
return true, false, nil
}
folder, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetAPIObject().GetName()))
if !ok {
a.log.Error().Msgf("Unable to get JWT folder info")
return true, false, nil
}
c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID)
if err != nil {
a.log.Warn().Err(err).Msg("Unable to get client")
return true, false, nil
}
if invalid, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, true); err != nil {
a.log.Warn().Err(err).Msg("Error while getting JWT Status")
return true, false, nil
} else if invalid {
return false, false, nil
}
return true, false, nil
}
func (a *jwtRefreshAction) Start(ctx context.Context) (bool, error) {
ready, _, err := a.CheckProgress(ctx)
return ready, err
}

116
pkg/deployment/reconcile/action_jwt_set_active.go Normal file
View file

@ -0,0 +1,116 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"encoding/base64"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
"k8s.io/apimachinery/pkg/types"
)
func init() {
registerAction(api.ActionTypeJWTSetActive, newJWTSetActive)
}
func newJWTSetActive(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtSetActiveAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtSetActiveAction struct {
actionImpl
actionEmptyCheckProgress
}
func (a *jwtSetActiveAction) Start(ctx context.Context) (bool, error) {
folder, err := ensureJWTFolderSupportFromAction(a.actionCtx)
if err != nil {
a.log.Error().Err(err).Msgf("Action not supported")
return true, nil
}
if !folder {
a.log.Error().Msgf("Action not supported")
return true, nil
}
toActiveChecksum, exists := a.action.Params[checksum]
if !exists {
a.log.Warn().Msgf("Key %s is missing in action", checksum)
return true, nil
}
f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName()))
if !ok {
a.log.Error().Msgf("Unable to get JWT folder info")
return true, nil
}
toActiveData, toActivePresent := f.Data[toActiveChecksum]
if !toActivePresent {
a.log.Error().Msgf("JWT key which is desired to be active is not anymore in secret")
return true, nil
}
activeKeyData, active := f.Data[pod.ActiveJWTKey]
if util.SHA256(activeKeyData) == toActiveChecksum {
a.log.Info().Msgf("Desired JWT is already active")
return true, nil
}
p := patch.NewPatch()
path := patch.NewPath("data", pod.ActiveJWTKey)
if !active {
p.ItemAdd(path, base64.StdEncoding.EncodeToString(toActiveData))
} else {
p.ItemReplace(path, base64.StdEncoding.EncodeToString(toActiveData))
}
patch, err := p.Marshal()
if err != nil {
a.log.Error().Err(err).Msgf("Unable to encrypt patch")
return true, nil
}
_, err = a.actionCtx.SecretsInterface().Patch(pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch)
if err != nil {
if !k8sutil.IsInvalid(err) {
return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName()))
}
}
return true, nil
}

183
pkg/deployment/reconcile/action_jwt_status_update.go Normal file
View file

@ -0,0 +1,183 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"fmt"
"sort"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
"github.com/pkg/errors"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/rs/zerolog"
)
const (
checksum = "checksum"
propagated = "propagated"
conditionTrue = "True"
conditionFalse = "False"
)
func ensureJWTFolderSupportFromAction(actionCtx ActionContext) (bool, error) {
return ensureJWTFolderSupport(actionCtx.GetSpec(), actionCtx.GetStatus())
}
func ensureJWTFolderSupport(spec api.DeploymentSpec, status api.DeploymentStatus) (bool, error) {
if !spec.IsAuthenticated() {
return false, errors.Errorf("Authentication is disabled")
}
if image := status.CurrentImage; image == nil {
return false, errors.Errorf("Missing image info")
} else {
if !image.Enterprise {
return false, nil
}
if image.ArangoDBVersion.CompareTo("3.7.0") < 0 {
return false, nil
}
}
return true, nil
}
func init() {
registerAction(api.ActionTypeJWTStatusUpdate, newJWTStatusUpdate)
}
func newJWTStatusUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &jwtStatusUpdateAction{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type jwtStatusUpdateAction struct {
actionImpl
actionEmptyCheckProgress
}
func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) {
folder, err := ensureJWTFolderSupportFromAction(a.actionCtx)
if err != nil {
a.log.Error().Err(err).Msgf("Action not supported")
return true, nil
}
if !folder {
f, ok := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().Authentication.GetJWTSecretName())
if !ok {
a.log.Error().Msgf("Unable to get JWT secret info")
return true, nil
}
key, ok := f.Data[constants.SecretKeyToken]
if !ok {
a.log.Error().Msgf("JWT Token is invalid")
return true, nil
}
keySha := fmt.Sprintf("sha256:%s", util.SHA256(key))
if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
if s.Hashes.JWT.Passive != nil {
s.Hashes.JWT.Passive = nil
return true
}
if s.Hashes.JWT.Active != keySha {
s.Hashes.JWT.Active = keySha
return true
}
return false
}); err != nil {
return false, err
}
return true, nil
}
f, ok := a.actionCtx.GetCachedStatus().Secret(pod.JWTSecretFolder(a.actionCtx.GetName()))
if !ok {
a.log.Error().Msgf("Unable to get JWT folder info")
return true, nil
}
if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) (update bool) {
activeKeyData, active := f.Data[pod.ActiveJWTKey]
activeKeyShort := util.SHA256(activeKeyData)
activeKey := fmt.Sprintf("sha256:%s", activeKeyShort)
if active {
if s.Hashes.JWT.Active != activeKey {
s.Hashes.JWT.Active = activeKey
update = true
}
}
if len(f.Data) == 0 {
if s.Hashes.JWT.Passive != nil {
s.Hashes.JWT.Passive = nil
update = true
}
}
var keys []string
for key := range f.Data {
if key == pod.ActiveJWTKey || key == activeKeyShort {
continue
}
keys = append(keys, key)
}
if len(keys) == 0 {
if s.Hashes.JWT.Passive != nil {
s.Hashes.JWT.Passive = nil
update = true
}
}
sort.Strings(keys)
keys = util.PrefixStringArray(keys, "sha256:")
if !util.CompareStringArray(keys, s.Hashes.JWT.Passive) {
s.Hashes.JWT.Passive = keys
update = true
}
return
}); err != nil {
return false, err
}
return true, nil
}

8
pkg/deployment/reconcile/action_tls_ca_append.go
View file

@ -37,10 +37,6 @@ import (
"github.com/rs/zerolog"
)
const (
actionTypeAppendTLSCACertificateChecksum = "checksum"
)
func init() {
registerAction(api.ActionTypeAppendTLSCACertificate, newAppendTLSCACertificateAction)
}
@ -64,9 +60,9 @@ func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error)
return true, nil
}
certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum]
certChecksum, exists := a.action.Params[checksum]
if !exists {
a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum)
a.log.Warn().Msgf("Key %s is missing in action", checksum)
return true, nil
}

4
pkg/deployment/reconcile/action_tls_ca_clean.go
View file

@ -61,9 +61,9 @@ func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
return true, nil
}
certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum]
certChecksum, exists := a.action.Params[checksum]
if !exists {
a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum)
a.log.Warn().Msgf("Key %s is missing in action", checksum)
return true, nil
}

2
pkg/deployment/reconcile/action_tls_keyfile_refresh.go
View file

@ -80,7 +80,7 @@ func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context)
return true, false, nil
}
if e.Result.KeyFile.Checksum == keyfileSha {
if e.Result.KeyFile.GetSHA().Checksum() == keyfileSha {
return true, false, nil
}

2
pkg/deployment/reconcile/helper_tls_sni.go
View file

@ -91,7 +91,7 @@ func compareTLSSNIConfig(ctx context.Context, c driver.Connection, m map[string]
return false, errors.Errorf("Unable to fetch TLS SNI state")
}
if value.Checksum != currentValue {
if value.GetSHA().Checksum() != currentValue {
return false, nil
}
}

18
pkg/deployment/reconcile/plan_builder.go
View file

@ -90,7 +90,7 @@ func (d *Reconciler) CreatePlan(ctx context.Context, cachedStatus inspector.Insp
func fetchAgency(ctx context.Context, log zerolog.Logger,
spec api.DeploymentSpec, status api.DeploymentStatus,
context PlanBuilderContext) (*agency.ArangoPlanDatabases, error) {
cache inspector.Inspector, context PlanBuilderContext) (*agency.ArangoPlanDatabases, error) {
if spec.GetMode() != api.DeploymentModeCluster && spec.GetMode() != api.DeploymentModeActiveFailover {
return nil, nil
} else if status.Members.Agents.MembersReady() > 0 {
@ -99,7 +99,7 @@ func fetchAgency(ctx context.Context, log zerolog.Logger,
ret := &agency.ArangoPlanDatabases{}
if err := context.GetAgencyData(agencyCtx, ret, agency.ArangoKey, agency.PlanKey, agency.PlanCollectionsKey); err != nil {
if err := context.GetAgencyData(agencyCtx, cache, agency.ArangoKey, agency.PlanKey, agency.PlanCollectionsKey); err != nil {
return nil, err
}
@ -123,7 +123,7 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
}
// Fetch agency plan
agencyPlan, agencyErr := fetchAgency(ctx, log, spec, status, builderCtx)
agencyPlan, agencyErr := fetchAgency(ctx, log, spec, status, cachedStatus, builderCtx)
// Check for various scenario's
var plan api.Plan
@ -210,6 +210,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
plan = pb.Apply(createTLSStatusUpdate)
}
if plan.IsEmpty() {
plan = pb.Apply(createJWTStatusUpdate)
}
// Check for scale up/down
if plan.IsEmpty() {
plan = pb.Apply(createScaleMemeberPlan)
@ -220,11 +224,15 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
plan = pb.Apply(createRotateOrUpgradePlan)
}
// Add encryption keys
// Add keys
if plan.IsEmpty() {
plan = pb.Apply(createEncryptionKey)
}
if plan.IsEmpty() {
plan = pb.Apply(createJWTKeyUpdate)
}
if plan.IsEmpty() {
plan = pb.Apply(createCARenewalPlan)
}
@ -256,7 +264,7 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
}
if plan.IsEmpty() {
plan = pb.Apply(cleanEncryptionKey)
plan = pb.Apply(createEncryptionKeyCleanPlan)
}
if plan.IsEmpty() {

135
pkg/deployment/reconcile/plan_builder_encryption.go
View file

@ -25,6 +25,8 @@ package reconcile
import (
"context"
core "k8s.io/api/core/v1"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@ -73,7 +75,7 @@ func createEncryptionKey(ctx context.Context,
return nil
}
keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName()))
keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName()))
if !exists {
log.Error().Msgf("Encryption key folder does not exist")
return nil
@ -88,49 +90,7 @@ func createEncryptionKey(ctx context.Context,
return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "")}
}
var plan api.Plan
status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
if !pod.GroupEncryptionSupported(spec.Mode.Get(), group) {
return nil
}
glog := log.With().Str("group", group.AsRole())
for _, m := range members {
if m.Phase != api.MemberPhaseCreated {
// Only make changes when phase is created
continue
}
if m.ArangoVersion.CompareTo("3.7.0") < 0 {
continue
}
mlog := glog.Str("member", m.ID).Logger()
c, err := context.GetServerClient(ctx, group, m.ID)
if err != nil {
mlog.Warn().Err(err).Msg("Unable to get client")
continue
}
client := client.NewClient(c.Connection())
e, err := client.GetEncryption(ctx)
if err != nil {
mlog.Error().Err(err).Msgf("Unable to fetch encryption keys")
continue
}
if !e.Result.KeysPresent(keyfolder.Data) {
plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRefresh, group, m.ID))
mlog.Info().Msgf("Refresh of encryption keys required")
continue
}
}
return nil
})
plan, _ := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder)
if !plan.IsEmpty() {
return plan
@ -163,7 +123,7 @@ func createEncryptionKeyStatusUpdateRequired(ctx context.Context,
return false
}
keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName()))
keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName()))
if !exists {
log.Error().Msgf("Encryption key folder does not exist")
return false
@ -171,14 +131,14 @@ func createEncryptionKeyStatusUpdateRequired(ctx context.Context,
keyHashes := secretKeysToListWithPrefix("sha256:", keyfolder)
if !util.CompareStringArray(keyHashes, status.Hashes.Encryption) {
if !util.CompareStringArray(keyHashes, status.Hashes.Encryption.Keys) {
return true
}
return false
}
func cleanEncryptionKey(ctx context.Context,
func createEncryptionKeyCleanPlan(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
@ -186,12 +146,24 @@ func cleanEncryptionKey(ctx context.Context,
return nil
}
keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName()))
keyfolder, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(context.GetName()))
if !exists {
log.Error().Msgf("Encryption key folder does not exist")
return nil
}
plan, failed := areEncryptionKeysUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, keyfolder)
if failed {
log.Info().Msgf("Unable to continue with encryption until all servers are ready")
return nil
}
if len(plan) != 0 {
log.Info().Msgf("Unable to continue with encryption until all servers report state or gonna be upToDate")
return nil
}
if len(keyfolder.Data) <= 1 {
return nil
}
@ -215,8 +187,6 @@ func cleanEncryptionKey(ctx context.Context,
return nil
}
var plan api.Plan
for key := range keyfolder.Data {
if key != name {
plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRemove, api.ServerGroupUnknown, "").AddParam("key", key))
@ -229,3 +199,68 @@ func cleanEncryptionKey(ctx context.Context,
return api.Plan{}
}
func areEncryptionKeysUpToDate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext,
folder *core.Secret) (plan api.Plan, failed bool) {
status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
if !pod.GroupEncryptionSupported(spec.Mode.Get(), group) {
return nil
}
for _, m := range list {
if updateRequired, failedMember := isEncryptionKeyUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, group, m, folder); failedMember {
failed = true
continue
} else if updateRequired {
plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRefresh, group, m.ID))
continue
}
}
return nil
})
return
}
func isEncryptionKeyUpToDate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext,
group api.ServerGroup, m api.MemberStatus,
folder *core.Secret) (updateRequired bool, failed bool) {
if m.Phase != api.MemberPhaseCreated {
return false, true
}
if m.ArangoVersion.CompareTo("3.7.0") < 0 {
return false, false
}
mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger()
c, err := context.GetServerClient(ctx, group, m.ID)
if err != nil {
mlog.Warn().Err(err).Msg("Unable to get client")
return false, true
}
client := client.NewClient(c.Connection())
e, err := client.GetEncryption(ctx)
if err != nil {
mlog.Error().Err(err).Msgf("Unable to fetch encryption keys")
return false, true
}
if !e.Result.KeysPresent(folder.Data) {
mlog.Info().Msgf("Refresh of encryption keys required")
return true, false
}
return false, false
}

333
pkg/deployment/reconcile/plan_builder_jwt.go Normal file
View file

@ -0,0 +1,333 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"fmt"
"sort"
"github.com/arangodb/kube-arangodb/pkg/deployment/client"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
)
func createJWTKeyUpdate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
if folder, err := ensureJWTFolderSupport(spec, status); err != nil || !folder {
return nil
}
folder, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName()))
if !ok {
log.Error().Msgf("Unable to get JWT folder info")
return nil
}
s, ok := cachedStatus.Secret(spec.Authentication.GetJWTSecretName())
if !ok {
log.Info().Msgf("JWT Secret is missing, no rotation will take place")
return nil
}
jwt, ok := s.Data[constants.SecretKeyToken]
if !ok {
log.Warn().Msgf("JWT Secret is invalid, no rotation will take place")
return addJWTPropagatedPlanAction(status)
}
jwtSha := util.SHA256(jwt)
if _, ok := folder.Data[jwtSha]; !ok {
return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTAdd, api.ServerGroupUnknown, "", "Add JWT key").AddParam(checksum, jwtSha))
}
activeKey, ok := folder.Data[pod.ActiveJWTKey]
if !ok {
return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha))
}
plan, failed := areJWTTokensUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, folder)
if len(plan) > 0 {
return plan
}
if failed {
log.Info().Msgf("JWT Failed on one pod, no rotation will take place")
return nil
}
if util.SHA256(activeKey) != jwtSha {
return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTSetActive, api.ServerGroupUnknown, "", "Set active key").AddParam(checksum, jwtSha))
}
for key := range folder.Data {
if key == pod.ActiveJWTKey {
continue
}
if key == jwtSha {
continue
}
return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTClean, api.ServerGroupUnknown, "", "Remove old key").AddParam(checksum, key))
}
return addJWTPropagatedPlanAction(status)
}
func createJWTStatusUpdate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
if _, err := ensureJWTFolderSupport(spec, status); err != nil {
return nil
}
if createJWTStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) {
return addJWTPropagatedPlanAction(status, api.NewAction(api.ActionTypeJWTStatusUpdate, api.ServerGroupUnknown, "", "Update status"))
}
return nil
}
func createJWTStatusUpdateRequired(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) bool {
folder, err := ensureJWTFolderSupport(spec, status)
if err != nil {
log.Error().Err(err).Msgf("Action not supported")
return false
}
if !folder {
if status.Hashes.JWT.Passive != nil {
return true
}
f, ok := cachedStatus.Secret(spec.Authentication.GetJWTSecretName())
if !ok {
log.Error().Msgf("Unable to get JWT secret info")
return false
}
key, ok := f.Data[constants.SecretKeyToken]
if !ok {
log.Error().Msgf("JWT Token is invalid")
return false
}
keySha := fmt.Sprintf("sha256:%s", util.SHA256(key))
if status.Hashes.JWT.Active != keySha {
log.Error().Msgf("JWT Token is invalid")
return true
}
return false
}
f, ok := cachedStatus.Secret(pod.JWTSecretFolder(apiObject.GetName()))
if !ok {
log.Error().Msgf("Unable to get JWT folder info")
return false
}
activeKeyData, active := f.Data[pod.ActiveJWTKey]
activeKeyShort := util.SHA256(activeKeyData)
activeKey := fmt.Sprintf("sha256:%s", activeKeyShort)
if active {
if status.Hashes.JWT.Active != activeKey {
return true
}
}
if len(f.Data) == 0 {
if status.Hashes.JWT.Passive != nil {
return true
}
return false
}
var keys []string
for key := range f.Data {
if key == pod.ActiveJWTKey || key == activeKeyShort {
continue
}
keys = append(keys, key)
}
if len(keys) == 0 {
if status.Hashes.JWT.Passive != nil {
return true
}
return false
}
sort.Strings(keys)
keys = util.PrefixStringArray(keys, "sha256:")
if !util.CompareStringArray(keys, status.Hashes.JWT.Passive) {
return true
}
return false
}
func areJWTTokensUpToDate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext,
folder *core.Secret) (plan api.Plan, failed bool) {
status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
for _, m := range list {
if updateRequired, failedMember := isJWTTokenUpToDate(ctx, log, apiObject, spec, status, cachedStatus, context, group, m, folder); failedMember {
failed = true
continue
} else if updateRequired {
plan = append(plan, api.NewAction(api.ActionTypeJWTRefresh, group, m.ID))
continue
}
}
return nil
})
return
}
func isJWTTokenUpToDate(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext,
group api.ServerGroup, m api.MemberStatus,
folder *core.Secret) (updateRequired bool, failed bool) {
if m.Phase != api.MemberPhaseCreated {
return false, true
}
if m.ArangoVersion.CompareTo("3.7.0") < 0 {
return false, false
}
mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger()
c, err := context.GetServerClient(ctx, group, m.ID)
if err != nil {
mlog.Warn().Err(err).Msg("Unable to get client")
return false, true
}
if updateRequired, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, false); err != nil {
mlog.Warn().Err(err).Msg("JET UpToDate Check failed")
return false, true
} else if updateRequired {
return true, false
}
return false, false
}
func addJWTPropagatedPlanAction(s api.DeploymentStatus, actions ...api.Action) api.Plan {
got := len(actions) != 0
cond := conditionFalse
if !got {
cond = conditionTrue
}
if s.Hashes.JWT.Propagated == got {
p := api.Plan{api.NewAction(api.ActionTypeJWTPropagated, api.ServerGroupUnknown, "", "Change propagated flag").AddParam(propagated, cond)}
return append(p, actions...)
}
return actions
}
func isMemberJWTTokenInvalid(ctx context.Context, c client.Client, data map[string][]byte, refresh bool) (bool, error) {
cmd := c.GetJWT
if refresh {
cmd = c.RefreshJWT
}
e, err := cmd(ctx)
if err != nil {
return false, errors.Wrapf(err, "Unable to fetch JWT tokens")
}
if e.Result.Active == nil {
return false, errors.Wrapf(err, "There is no active JWT Token")
}
if jwtActive, ok := data[pod.ActiveJWTKey]; !ok {
return false, errors.Errorf("Missing Active JWT Token in folder")
} else if util.SHA256(jwtActive) != e.Result.Active.GetSHA().Checksum() {
log.Info().Str("active", e.Result.Active.GetSHA().Checksum()).Str("expected", util.SHA256(jwtActive)).Msgf("Active key is invalid")
return true, nil
}
if !compareJWTKeys(e.Result.Passive, data) {
return true, nil
}
return false, nil
}
func compareJWTKeys(e client.Entries, keys map[string][]byte) bool {
for k := range keys {
if k == pod.ActiveJWTKey {
continue
}
if !e.Contains(k) {
log.Info().Msgf("Missing JWT Key")
return false
}
}
for _, entry := range e {
if entry.GetSHA() == "" {
continue
}
if _, ok := keys[entry.GetSHA().Checksum()]; !ok {
return false
}
}
return true
}

2
pkg/deployment/reconcile/plan_builder_restore.go
View file

@ -83,7 +83,7 @@ func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec a
secret := *spec.RestoreEncryptionSecret
// Additional logic to do restore with encryption key
keyfolder, err := builderCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(builderCtx.GetName()), meta.GetOptions{})
keyfolder, err := builderCtx.SecretsInterface().Get(pod.GetEncryptionFolderSecretName(builderCtx.GetName()), meta.GetOptions{})
if err != nil {
log.Err(err).Msgf("Unable to fetch encryption folder")
return nil

108
pkg/deployment/reconcile/plan_builder_tls.go
View file

@ -255,7 +255,7 @@ func createCAAppendPlan(ctx context.Context,
if _, exists := trusted.Data[certSha]; !exists {
return api.Plan{api.NewAction(api.ActionTypeAppendTLSCACertificate, api.ServerGroupUnknown, "", "Append CA to truststore").
AddParam(actionTypeAppendTLSCACertificateChecksum, certSha)}
AddParam(checksum, certSha)}
}
return nil
@ -340,15 +340,14 @@ func createCACleanPlan(ctx context.Context,
for sha := range trusted.Data {
if certSha != sha {
return api.Plan{api.NewAction(api.ActionTypeCleanTLSCACertificate, api.ServerGroupUnknown, "", "Clean CA from truststore").
AddParam(actionTypeAppendTLSCACertificateChecksum, sha)}
AddParam(checksum, sha)}
}
}
return nil
}
// createKeyfileRenewalPlan creates plan to renew server keyfile
func createKeyfileRenewalPlan(ctx context.Context,
func createKeyfileRenewalPlanDefault(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
@ -367,10 +366,12 @@ func createKeyfileRenewalPlan(ctx context.Context,
if !plan.IsEmpty() {
return nil
}
if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member); renew {
if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member, api.TLSRotateModeRecreate); renew {
log.Info().Msg("Renewal of keyfile required")
plan = append(plan, createKeyfileRotationPlan(log, spec, status, group, member, recreate)...)
if recreate {
plan = append(plan, api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal"))
}
plan = append(plan, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...)
}
}
@ -380,43 +381,84 @@ func createKeyfileRenewalPlan(ctx context.Context,
return plan
}
func createKeyfileRenewalPlanMode(
func createKeyfileRenewalPlanInPlace(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
member api.MemberStatus) api.TLSRotateMode {
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
if !spec.TLS.IsSecure() {
return nil
}
var plan api.Plan
status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
if !group.IsArangod() {
return nil
}
for _, member := range members {
if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member, api.TLSRotateModeInPlace); renew {
log.Info().Msg("Renewal of keyfile required")
if recreate {
plan = append(plan, api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal"))
}
plan = append(plan, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile"))
}
}
return nil
})
return plan
}
func createKeyfileRenewalPlan(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
if !spec.TLS.IsSecure() {
return nil
}
switch createKeyfileRenewalPlanMode(spec, status) {
case api.TLSRotateModeInPlace:
return createKeyfileRenewalPlanInPlace(ctx, log, apiObject, spec, status, cachedStatus, context)
default:
return createKeyfileRenewalPlanDefault(ctx, log, apiObject, spec, status, cachedStatus, context)
}
}
func createKeyfileRenewalPlanMode(
spec api.DeploymentSpec, status api.DeploymentStatus) api.TLSRotateMode {
if !spec.TLS.IsSecure() {
return api.TLSRotateModeRecreate
}
if spec.TLS.Mode.Get() != api.TLSRotateModeInPlace {
return api.TLSRotateModeRecreate
mode := spec.TLS.Mode.Get()
status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
if mode != api.TLSRotateModeInPlace {
return nil
}
for _, member := range list {
if mode != api.TLSRotateModeInPlace {
return nil
}
if i := status.CurrentImage; i == nil {
return api.TLSRotateModeRecreate
mode = api.TLSRotateModeRecreate
} else {
if !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || i.ImageID != member.ImageID {
return api.TLSRotateModeRecreate
mode = api.TLSRotateModeRecreate
}
}
}
return api.TLSRotateModeInPlace
}
return nil
})
func createKeyfileRotationPlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, group api.ServerGroup, member api.MemberStatus, recreate bool) api.Plan {
p := api.Plan{}
if recreate {
p = append(p,
api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal"))
}
switch createKeyfileRenewalPlanMode(spec, status, member) {
case api.TLSRotateModeInPlace:
p = append(p, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile"))
default:
p = append(p, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...)
}
return p
return mode
}
func checkServerValidCertRequest(ctx context.Context, apiObject k8sutil.APIObject, group api.ServerGroup, member api.MemberStatus, ca Certificates) (*tls.ConnectionState, error) {
@ -440,7 +482,7 @@ func keyfileRenewalRequired(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext,
group api.ServerGroup, member api.MemberStatus) (bool, bool) {
group api.ServerGroup, member api.MemberStatus, mode api.TLSRotateMode) (bool, bool) {
if !spec.TLS.IsSecure() {
return false, false
}
@ -485,7 +527,7 @@ func keyfileRenewalRequired(ctx context.Context,
}
// Ensure secret is propagated only on 3.7.0+ enterprise and inplace mode
if createKeyfileRenewalPlanMode(spec, status, member) == api.TLSRotateModeInPlace {
if mode == api.TLSRotateModeInPlace {
conn, err := context.GetServerClient(ctx, group, member.ID)
if err != nil {
log.Warn().Err(err).Msg("Unable to get client")
@ -513,7 +555,7 @@ func keyfileRenewalRequired(ctx context.Context,
keyfileSha := util.SHA256(keyfile)
if tls.Result.KeyFile.Checksum != keyfileSha {
if tls.Result.KeyFile.GetSHA().Checksum() != keyfileSha {
return true, false
}
}

8
pkg/deployment/resources/exporter.go
View file

@ -26,6 +26,8 @@ import (
"sort"
"strconv"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
@ -34,7 +36,7 @@ import (
)
// ArangodbExporterContainer creates metrics container
func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sutil.HTTPProbeConfig,
func ArangodbExporterContainer(image string, args []string, livenessProbe *probes.HTTPProbeConfig,
resources v1.ResourceRequirements, securityContext *v1.SecurityContext,
spec api.DeploymentSpec) v1.Container {
@ -96,8 +98,8 @@ func createExporterArgs(spec api.DeploymentSpec) []string {
return args
}
func createExporterLivenessProbe(isSecure bool) *k8sutil.HTTPProbeConfig {
probeCfg := &k8sutil.HTTPProbeConfig{
func createExporterLivenessProbe(isSecure bool) *probes.HTTPProbeConfig {
probeCfg := &probes.HTTPProbeConfig{
LocalPath: "/",
Port: k8sutil.ArangoExporterPort,
Secure: isSecure,

146
pkg/deployment/resources/pod_creator_probes.go
View file

@ -23,24 +23,37 @@
package resources
import (
"fmt"
"os"
"path/filepath"
"github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/jwt"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
core "k8s.io/api/core/v1"
)
type Probe interface {
Create() *core.Probe
SetSpec(spec *api.ServerGroupProbeSpec)
}
type probeCheckBuilder struct {
liveness, readiness probeBuilder
}
type probeBuilder func(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error)
type probeBuilder func(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error)
func nilProbeBuilder(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func nilProbeBuilder(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
return nil, nil
}
func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
if !r.isReadinessProbeEnabled(spec, group, version) {
return nil, nil
}
@ -65,16 +78,12 @@ func (r *Resources) getReadinessProbe(spec api.DeploymentSpec, group api.ServerG
probeSpec := groupSpec.GetProbesSpec()
config.InitialDelaySeconds = probeSpec.ReadinessProbeSpec.GetInitialDelaySeconds(config.InitialDelaySeconds)
config.PeriodSeconds = probeSpec.ReadinessProbeSpec.GetPeriodSeconds(config.PeriodSeconds)
config.TimeoutSeconds = probeSpec.ReadinessProbeSpec.GetTimeoutSeconds(config.TimeoutSeconds)
config.SuccessThreshold = probeSpec.ReadinessProbeSpec.GetSuccessThreshold(config.SuccessThreshold)
config.FailureThreshold = probeSpec.ReadinessProbeSpec.GetFailureThreshold(config.FailureThreshold)
config.SetSpec(probeSpec.ReadinessProbeSpec)
return config, nil
}
func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
if !r.isLivenessProbeEnabled(spec, group, version) {
return nil, nil
}
@ -99,11 +108,7 @@ func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGr
probeSpec := groupSpec.GetProbesSpec()
config.InitialDelaySeconds = probeSpec.LivenessProbeSpec.GetInitialDelaySeconds(config.InitialDelaySeconds)
config.PeriodSeconds = probeSpec.LivenessProbeSpec.GetPeriodSeconds(config.PeriodSeconds)
config.TimeoutSeconds = probeSpec.LivenessProbeSpec.GetTimeoutSeconds(config.TimeoutSeconds)
config.SuccessThreshold = probeSpec.LivenessProbeSpec.GetSuccessThreshold(config.SuccessThreshold)
config.FailureThreshold = probeSpec.LivenessProbeSpec.GetFailureThreshold(config.FailureThreshold)
config.SetSpec(probeSpec.LivenessProbeSpec)
return config, nil
}
@ -139,20 +144,20 @@ func (r *Resources) isLivenessProbeEnabled(spec api.DeploymentSpec, group api.Se
func (r *Resources) probeBuilders() map[api.ServerGroup]probeCheckBuilder {
return map[api.ServerGroup]probeCheckBuilder{
api.ServerGroupSingle: {
liveness: r.probeBuilderLivenessCore,
readiness: r.probeBuilderReadinessCore,
liveness: r.probeBuilderLivenessCoreOperator,
readiness: r.probeBuilderReadinessCoreOperator,
},
api.ServerGroupAgents: {
liveness: r.probeBuilderLivenessCore,
readiness: r.probeBuilderReadinessSimpleCore,
liveness: r.probeBuilderLivenessCoreOperator,
readiness: r.probeBuilderReadinessSimpleCoreOperator,
},
api.ServerGroupDBServers: {
liveness: r.probeBuilderLivenessCore,
readiness: r.probeBuilderReadinessSimpleCore,
liveness: r.probeBuilderLivenessCoreOperator,
readiness: r.probeBuilderReadinessSimpleCoreOperator,
},
api.ServerGroupCoordinators: {
liveness: r.probeBuilderLivenessCore,
readiness: r.probeBuilderReadinessCore,
liveness: r.probeBuilderLivenessCoreOperator,
readiness: r.probeBuilderReadinessCoreOperator,
},
api.ServerGroupSyncMasters: {
liveness: r.probeBuilderLivenessSync,
@ -165,7 +170,42 @@ func (r *Resources) probeBuilders() map[api.ServerGroup]probeCheckBuilder {
}
}
func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) probeCommand(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version, endpoint string) ([]string, error) {
binaryPath, err := os.Executable()
if err != nil {
return nil, err
}
exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
args := []string{
exePath,
"lifecycle",
"probe",
fmt.Sprintf("--endpoint=%s", endpoint),
}
if spec.IsSecure() {
args = append(args, "--ssl")
}
if spec.IsAuthenticated() {
args = append(args, "--auth")
}
return args, nil
}
func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
args, err := r.probeCommand(spec, group, version, "/_api/version")
if err != nil {
return nil, err
}
return &probes.CMDProbeConfig{
Command: args,
}, nil
}
func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
authorization := ""
if spec.IsAuthenticated() {
secretData, err := r.getJWTSecret(spec)
@ -177,14 +217,32 @@ func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.
return nil, maskAny(err)
}
}
return &k8sutil.HTTPProbeConfig{
return &probes.HTTPProbeConfig{
LocalPath: "/_api/version",
Secure: spec.IsSecure(),
Authorization: authorization,
}, nil
}
func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) probeBuilderReadinessSimpleCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
p, err := r.probeBuilderReadinessCoreOperator(spec, group, version)
if err != nil {
return nil, err
}
if p == nil {
return nil, nil
}
p.SetSpec(&api.ServerGroupProbeSpec{
InitialDelaySeconds: util.NewInt32(15),
PeriodSeconds: util.NewInt32(10),
})
return p, nil
}
func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
p, err := r.probeBuilderLivenessCore(spec, group, version)
if err != nil {
return nil, err
@ -194,13 +252,39 @@ func (r *Resources) probeBuilderReadinessSimpleCore(spec api.DeploymentSpec, gro
return nil, nil
}
p.InitialDelaySeconds = 15
p.PeriodSeconds = 10
p.SetSpec(&api.ServerGroupProbeSpec{
InitialDelaySeconds: util.NewInt32(15),
PeriodSeconds: util.NewInt32(10),
})
return p, nil
}
func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
localPath := "/_api/version"
switch spec.GetMode() {
case api.DeploymentModeActiveFailover:
localPath = "/_admin/echo"
}
// /_admin/server/availability is the way to go, it is available since 3.3.9
if version.CompareTo("3.3.9") >= 0 {
localPath = "/_admin/server/availability"
}
args, err := r.probeCommand(spec, group, version, localPath)
if err != nil {
return nil, err
}
return &probes.CMDProbeConfig{
Command: args,
InitialDelaySeconds: 2,
PeriodSeconds: 2,
}, nil
}
func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
localPath := "/_api/version"
switch spec.GetMode() {
case api.DeploymentModeActiveFailover:
@ -223,7 +307,7 @@ func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api
return nil, maskAny(err)
}
}
probeCfg := &k8sutil.HTTPProbeConfig{
probeCfg := &probes.HTTPProbeConfig{
LocalPath: localPath,
Secure: spec.IsSecure(),
Authorization: authorization,
@ -234,7 +318,7 @@ func (r *Resources) probeBuilderReadinessCore(spec api.DeploymentSpec, group api
return probeCfg, nil
}
func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (*k8sutil.HTTPProbeConfig, error) {
func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
authorization := ""
port := k8sutil.ArangoSyncMasterPort
if group == api.ServerGroupSyncWorkers {
@ -261,7 +345,7 @@ func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.
// Don't have a probe
return nil, nil
}
return &k8sutil.HTTPProbeConfig{
return &probes.HTTPProbeConfig{
LocalPath: "/_api/version",
Secure: spec.IsSecure(),
Authorization: authorization,

17
pkg/deployment/resources/secret_hashes.go
View file

@ -138,6 +138,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error
}
if spec.IsAuthenticated() {
if image == nil || image.ArangoDBVersion.CompareTo("3.7.0") < 0 {
secretName := spec.Authentication.GetJWTSecretName()
getExpectedHash := func() string { return getHashes().AuthJWT }
setExpectedHash := func(h string) error {
@ -148,6 +149,20 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error
} else if !hashOK {
badSecretNames = append(badSecretNames, secretName)
}
} else {
if _, exists := cachedStatus.Secret(pod.JWTSecretFolder(deploymentName)); !exists {
secretName := spec.Authentication.GetJWTSecretName()
getExpectedHash := func() string { return getHashes().AuthJWT }
setExpectedHash := func(h string) error {
return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h }))
}
if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
return maskAny(err)
} else if !hashOK {
badSecretNames = append(badSecretNames, secretName)
}
}
}
}
if spec.RocksDB.IsEncrypted() {
if image == nil || image.ArangoDBVersion.CompareTo("3.7.0") < 0 {
@ -162,7 +177,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error
badSecretNames = append(badSecretNames, secretName)
}
} else {
if _, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(deploymentName)); !exists {
if _, exists := cachedStatus.Secret(pod.GetEncryptionFolderSecretName(deploymentName)); !exists {
secretName := spec.RocksDB.Encryption.GetKeySecretName()
getExpectedHash := func() string { return getHashes().RocksDBEncryptionKey }
setExpectedHash := func(h string) error {

64
pkg/deployment/resources/secrets.go
View file

@ -29,6 +29,8 @@ import (
"fmt"
"time"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/rs/zerolog"
operatorErrors "github.com/arangodb/kube-arangodb/pkg/util/errors"
@ -74,6 +76,8 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins
status, _ := r.context.GetStatus()
apiObject := r.context.GetAPIObject()
deploymentName := apiObject.GetName()
image := status.CurrentImage
imageFound := status.CurrentImage != nil
defer metrics.SetDuration(inspectSecretsDurationGauges.WithLabelValues(deploymentName), start)
counterMetric := inspectedSecretsCounters.WithLabelValues(deploymentName)
@ -83,6 +87,14 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins
return maskAny(err)
}
if imageFound {
if pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) {
if err := r.ensureTokenSecretFolder(cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil {
return maskAny(err)
}
}
}
if spec.Metrics.IsEnabled() {
if err := r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName()); err != nil {
return maskAny(err)
@ -131,7 +143,7 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins
}
if spec.RocksDB.IsEncrypted() {
if i := status.CurrentImage; i != nil && i.Enterprise && i.ArangoDBVersion.CompareTo("3.7.0") >= 0 {
if err := r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetKeyfolderSecretName(deploymentName)); err != nil {
if err := r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName)); err != nil {
return maskAny(err)
}
}
@ -157,9 +169,28 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Ins
return nil
}
// ensureTokenSecret checks if a secret with given name exists in the namespace
// of the deployment. If not, it will add such a secret with a random
// token.
func (r *Resources) ensureTokenSecretFolder(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error {
if _, exists := cachedStatus.Secret(folderSecretName); exists {
return nil
}
s, exists := cachedStatus.Secret(secretName)
if !exists {
return errors.Errorf("Token secret does not exist")
}
token, ok := s.Data[constants.SecretKeyToken]
if !ok {
return errors.Errorf("Token secret is invalid")
}
if err := r.createSecretWithKey(secrets, folderSecretName, util.SHA256(token), token); err != nil {
return err
}
return nil
}
func (r *Resources) ensureTokenSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName string) error {
if _, exists := cachedStatus.Secret(secretName); !exists {
return r.createTokenSecret(secrets, secretName)
@ -196,20 +227,28 @@ func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName str
func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error {
if _, exists := cachedStatus.Secret(secretName); !exists {
return r.createSecretWithEmptyKey(secrets, secretName, keyName)
return r.createSecretWithKey(secrets, secretName, keyName, nil)
}
return nil
}
func (r *Resources) createSecretWithEmptyKey(secrets k8sutil.SecretInterface, secretName, keyName string) error {
func (r *Resources) ensureSecretWithKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error {
if _, exists := cachedStatus.Secret(secretName); !exists {
return r.createSecretWithKey(secrets, secretName, keyName, value)
}
return nil
}
func (r *Resources) createSecretWithKey(secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error {
// Create secret
secret := &core.Secret{
ObjectMeta: meta.ObjectMeta{
Name: secretName,
},
Data: map[string][]byte{
keyName: {},
keyName: value,
},
}
// Attach secret to owner
@ -242,17 +281,28 @@ func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretNam
}
func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error {
_, folderExists := cachedStatus.Secret(secretName)
keyfile, exists := cachedStatus.Secret(keyfileSecretName)
if !exists {
if folderExists {
return nil
}
return errors.Errorf("Unable to find original secret %s", keyfileSecretName)
}
if len(keyfile.Data) == 0 {
if folderExists {
return nil
}
return errors.Errorf("Missing key in secret")
}
d, ok := keyfile.Data[constants.SecretEncryptionKey]
if !ok {
if folderExists {
return nil
}
return errors.Errorf("Missing key in secret")
}

24
pkg/util/arangod/client.go
View file

@ -135,6 +135,24 @@ func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface
return c, nil
}
func CreateArangodAgencyConnection(ctx context.Context, apiObject *api.ArangoDeployment) (driver.Connection, error) {
var dnsNames []string
for _, m := range apiObject.Status.Members.Agents {
dnsName := k8sutil.CreatePodDNSName(apiObject, api.ServerGroupAgents.AsRole(), m.ID)
dnsNames = append(dnsNames, dnsName)
}
shortTimeout := false
connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, dnsNames, shortTimeout)
if err != nil {
return nil, maskAny(err)
}
agencyConn, err := agency.NewAgencyConnection(connConfig)
if err != nil {
return nil, maskAny(err)
}
return agencyConn, nil
}
// CreateArangodAgencyClient creates a go-driver client for accessing the agents of the given deployment.
func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (agency.Agency, error) {
var dnsNames []string
@ -143,7 +161,7 @@ func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface,
dnsNames = append(dnsNames, dnsName)
}
shortTimeout := false
connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, dnsNames, shortTimeout)
connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, dnsNames, shortTimeout)
if err != nil {
return nil, maskAny(err)
}
@ -182,7 +200,7 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec
// CreateArangodClientForDNSName creates a go-driver client for a given DNS name.
func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, shortTimeout bool) (driver.Client, error) {
connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}, shortTimeout)
connConfig, err := createArangodHTTPConfigForDNSNames(ctx, apiObject, []string{dnsName}, shortTimeout)
if err != nil {
return nil, maskAny(err)
}
@ -209,7 +227,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa
}
// createArangodHTTPConfigForDNSNames creates a go-driver HTTP connection config for a given DNS names.
func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (http.ConnectionConfig, error) {
func createArangodHTTPConfigForDNSNames(ctx context.Context, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (http.ConnectionConfig, error) {
scheme := "http"
transport := sharedHTTPTransport
if shortTimeout {

129
pkg/util/arangod/conn/factory.go Normal file
View file

@ -0,0 +1,129 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Adam Janikowski
//
package conn
import (
"github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/agency"
"github.com/arangodb/go-driver/http"
)
type Auth func() (driver.Authentication, error)
type Config func() (http.ConnectionConfig, error)
type Factory interface {
Connection(hosts ...string) (driver.Connection, error)
AgencyConnection(hosts ...string) (driver.Connection, error)
Client(hosts ...string) (driver.Client, error)
Agency(hosts ...string) (agency.Agency, error)
}
func NewFactory(auth Auth, config Config) Factory {
return &factory{
auth: auth,
config: config,
}
}
type factory struct {
auth Auth
config Config
}
func (f factory) AgencyConnection(hosts ...string) (driver.Connection, error) {
cfg, err := f.config()
if err != nil {
return nil, err
}
cfg.Endpoints = hosts
conn, err := agency.NewAgencyConnection(cfg)
if err != nil {
return nil, err
}
if f.auth == nil {
return conn, nil
}
auth, err := f.auth()
if err != nil {
return nil, err
}
return conn.SetAuthentication(auth)
}
func (f factory) Client(hosts ...string) (driver.Client, error) {
conn, err := f.Connection(hosts...)
if err != nil {
return nil, err
}
config := driver.ClientConfig{
Connection: conn,
}
if f.auth != nil {
auth, err := f.auth()
if err != nil {
return nil, err
}
config.Authentication = auth
}
return driver.NewClient(config)
}
func (f factory) Agency(hosts ...string) (agency.Agency, error) {
conn, err := f.AgencyConnection(hosts...)
if err != nil {
return nil, err
}
return agency.NewAgency(conn)
}
func (f factory) Connection(hosts ...string) (driver.Connection, error) {
cfg, err := f.config()
if err != nil {
return nil, err
}
cfg.Endpoints = hosts
conn, err := http.NewConnection(cfg)
if err != nil {
return nil, err
}
if f.auth == nil {
return conn, nil
}
auth, err := f.auth()
if err != nil {
return nil, err
}
return conn.SetAuthentication(auth)
}

4
pkg/util/checksum.go
View file

@ -27,6 +27,10 @@ import (
"fmt"
)
func SHA256FromString(data string) string {
return SHA256([]byte(data))
}
func SHA256(data []byte) string {
return fmt.Sprintf("%0x", sha256.Sum256(data))
}

8
pkg/util/k8sutil/lifecycle.go
View file

@ -33,7 +33,7 @@ import (
const (
initLifecycleContainerName = "init-lifecycle"
lifecycleVolumeMountDir = "/lifecycle/tools"
LifecycleVolumeMountDir = "/lifecycle/tools"
lifecycleVolumeName = "lifecycle"
)
@ -46,7 +46,7 @@ func InitLifecycleContainer(image string, resources *v1.ResourceRequirements, se
c := v1.Container{
Name: initLifecycleContainerName,
Image: image,
Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", lifecycleVolumeMountDir),
Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", LifecycleVolumeMountDir),
VolumeMounts: []v1.VolumeMount{
LifecycleVolumeMount(),
},
@ -66,7 +66,7 @@ func NewLifecycle() (*v1.Lifecycle, error) {
if err != nil {
return nil, maskAny(err)
}
exePath := filepath.Join(lifecycleVolumeMountDir, filepath.Base(binaryPath))
exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath))
lifecycle := &v1.Lifecycle{
PreStop: &v1.Handler{
Exec: &v1.ExecAction{
@ -91,7 +91,7 @@ func GetLifecycleEnv() []v1.EnvVar {
func LifecycleVolumeMount() v1.VolumeMount {
return v1.VolumeMount{
Name: lifecycleVolumeName,
MountPath: lifecycleVolumeMountDir,
MountPath: LifecycleVolumeMountDir,
}
}

86
pkg/util/k8sutil/probes.go
View file

@ -1,86 +0,0 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//
package k8sutil
import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
// HTTPProbeConfig contains settings for creating a liveness/readiness probe.
type HTTPProbeConfig struct {
// Local path to GET
LocalPath string // `e.g. /_api/version`
// Secure connection?
Secure bool
// Value for an Authorization header (can be empty)
Authorization string
// Port to inspect (defaults to ArangoPort)
Port int
// Number of seconds after the container has started before liveness probes are initiated (defaults to 30)
InitialDelaySeconds int32
// Number of seconds after which the probe times out (defaults to 2).
TimeoutSeconds int32
// How often (in seconds) to perform the probe (defaults to 10).
PeriodSeconds int32
// Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1).
SuccessThreshold int32
// Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3).
FailureThreshold int32
}
// Create creates a probe from given config
func (config HTTPProbeConfig) Create() *v1.Probe {
scheme := v1.URISchemeHTTP
if config.Secure {
scheme = v1.URISchemeHTTPS
}
var headers []v1.HTTPHeader
if config.Authorization != "" {
headers = append(headers, v1.HTTPHeader{
Name: "Authorization",
Value: config.Authorization,
})
}
def := func(value, defaultValue int32) int32 {
if value != 0 {
return value
}
return defaultValue
}
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: config.LocalPath,
Port: intstr.FromInt(int(def(int32(config.Port), ArangoPort))),
Scheme: scheme,
HTTPHeaders: headers,
},
},
InitialDelaySeconds: def(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe
TimeoutSeconds: def(config.TimeoutSeconds, 2), // Timeout of each probe is 2s
PeriodSeconds: def(config.PeriodSeconds, 60), // Interval between probes is 10s
SuccessThreshold: def(config.SuccessThreshold, 1), // Single probe is enough to indicate success
FailureThreshold: def(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state
}
}

142
pkg/util/k8sutil/probes/probes.go Normal file
View file

@ -0,0 +1,142 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//
package probes
import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
// HTTPProbeConfig contains settings for creating a liveness/readiness probe.
type HTTPProbeConfig struct {
// Local path to GET
LocalPath string // `e.g. /_api/version`
// Secure connection?
Secure bool
// Value for an Authorization header (can be empty)
Authorization string
// Port to inspect (defaults to ArangoPort)
Port int
// Number of seconds after the container has started before liveness probes are initiated (defaults to 30)
InitialDelaySeconds int32
// Number of seconds after which the probe times out (defaults to 2).
TimeoutSeconds int32
// How often (in seconds) to perform the probe (defaults to 10).
PeriodSeconds int32
// Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1).
SuccessThreshold int32
// Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3).
FailureThreshold int32
}
func (config *HTTPProbeConfig) SetSpec(spec *api.ServerGroupProbeSpec) {
config.InitialDelaySeconds = spec.GetInitialDelaySeconds(config.InitialDelaySeconds)
config.TimeoutSeconds = spec.GetTimeoutSeconds(config.TimeoutSeconds)
config.PeriodSeconds = spec.GetPeriodSeconds(config.PeriodSeconds)
config.SuccessThreshold = spec.GetSuccessThreshold(config.SuccessThreshold)
config.FailureThreshold = spec.GetFailureThreshold(config.FailureThreshold)
}
// Create creates a probe from given config
func (config HTTPProbeConfig) Create() *core.Probe {
scheme := core.URISchemeHTTP
if config.Secure {
scheme = core.URISchemeHTTPS
}
var headers []core.HTTPHeader
if config.Authorization != "" {
headers = append(headers, core.HTTPHeader{
Name: "Authorization",
Value: config.Authorization,
})
}
def := func(value, defaultValue int32) int32 {
if value != 0 {
return value
}
return defaultValue
}
return &core.Probe{
Handler: core.Handler{
HTTPGet: &core.HTTPGetAction{
Path: config.LocalPath,
Port: intstr.FromInt(int(def(int32(config.Port), k8sutil.ArangoPort))),
Scheme: scheme,
HTTPHeaders: headers,
},
},
InitialDelaySeconds: defaultInt32(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe
TimeoutSeconds: defaultInt32(config.TimeoutSeconds, 2), // Timeout of each probe is 2s
PeriodSeconds: defaultInt32(config.PeriodSeconds, 60), // Interval between probes is 10s
SuccessThreshold: defaultInt32(config.SuccessThreshold, 1), // Single probe is enough to indicate success
FailureThreshold: defaultInt32(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state
}
}
type CMDProbeConfig struct {
// Command to be executed
Command []string
// Number of seconds after the container has started before liveness probes are initiated (defaults to 30)
InitialDelaySeconds int32
// Number of seconds after which the probe times out (defaults to 2).
TimeoutSeconds int32
// How often (in seconds) to perform the probe (defaults to 10).
PeriodSeconds int32
// Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1).
SuccessThreshold int32
// Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3).
FailureThreshold int32
}
func (config *CMDProbeConfig) SetSpec(spec *api.ServerGroupProbeSpec) {
config.InitialDelaySeconds = spec.GetInitialDelaySeconds(config.InitialDelaySeconds)
config.TimeoutSeconds = spec.GetTimeoutSeconds(config.TimeoutSeconds)
config.PeriodSeconds = spec.GetPeriodSeconds(config.PeriodSeconds)
config.SuccessThreshold = spec.GetSuccessThreshold(config.SuccessThreshold)
config.FailureThreshold = spec.GetFailureThreshold(config.FailureThreshold)
}
// Create creates a probe from given config
func (config CMDProbeConfig) Create() *core.Probe {
return &core.Probe{
Handler: core.Handler{
Exec: &core.ExecAction{
Command: config.Command,
},
},
InitialDelaySeconds: defaultInt32(config.InitialDelaySeconds, 15*60), // Wait 15min before first probe
TimeoutSeconds: defaultInt32(config.TimeoutSeconds, 2), // Timeout of each probe is 2s
PeriodSeconds: defaultInt32(config.PeriodSeconds, 60), // Interval between probes is 10s
SuccessThreshold: defaultInt32(config.SuccessThreshold, 1), // Single probe is enough to indicate success
FailureThreshold: defaultInt32(config.FailureThreshold, 10), // Need 10 failed probes to consider a failed state
}
}
func defaultInt32(value, defaultValue int32) int32 {
if value != 0 {
return value
}
return defaultValue
}

2
pkg/util/k8sutil/probes_test.go → pkg/util/k8sutil/probes/probes_test.go
View file

@ -20,7 +20,7 @@
// Author Jan Christoph Uhde
//
package k8sutil
package probes
import (
"testing"