[Feature] Change log level on the fly (#788)

Makefile

@@ -123,9 +123,9 @@ ifdef VERBOSE
 endif
 
 EXCLUDE_DIRS := tests vendor .gobuild deps tools
-SOURCES_QUERY := find $(SRCDIR) -name '*.go' -type f -not -path '$(SRCDIR)/tests/*' -not -path '$(SRCDIR)/vendor/*' -not -path '$(SRCDIR)/.gobuild/*' -not -path '$(SRCDIR)/deps/*' -not -path '$(SRCDIR)/tools/*'
+SOURCES_QUERY := find ./ -type f -name '*.go' $(foreach EXCLUDE_DIR,$(EXCLUDE_DIRS), -not -path "./$(EXCLUDE_DIR)/*")
 SOURCES := $(shell $(SOURCES_QUERY))
-DASHBOARDSOURCES := $(shell find $(DASHBOARDDIR)/src -name '*.js' -not -path './test/*') $(DASHBOARDDIR)/package.json
+DASHBOARDSOURCES := $(shell find $(DASHBOARDDIR)/src -name '*.js') $(DASHBOARDDIR)/package.json
 
 .DEFAULT_GOAL := all
 .PHONY: all

pkg/apis/deployment/v1/plan.go

@@ -169,6 +169,8 @@ const (
 	// Runtime Updates
 	// ActionTypeRuntimeContainerImageUpdate updates container image in runtime
 	ActionTypeRuntimeContainerImageUpdate ActionType = "RuntimeContainerImageUpdate"
+	// ActionTypeRuntimeContainerArgsLogLevelUpdate updates the container's executor arguments.
+	ActionTypeRuntimeContainerArgsLogLevelUpdate ActionType = "RuntimeContainerArgsLogLevelUpdate"
 )
 
 const (

pkg/apis/deployment/v2alpha1/plan.go

@@ -171,6 +171,8 @@ const (
 	// Runtime Updates
 	// ActionTypeRuntimeContainerImageUpdate updates container image in runtime
 	ActionTypeRuntimeContainerImageUpdate ActionType = "RuntimeContainerImageUpdate"
+	// ActionTypeRuntimeContainerArgsLogLevelUpdate updates the container's executor arguments.
+	ActionTypeRuntimeContainerArgsLogLevelUpdate ActionType = "RuntimeContainerArgsLogLevelUpdate"
 )
 
 const (

pkg/deployment/reconcile/action_helper.go

@@ -102,7 +102,7 @@ func (a actionImpl) Timeout(deploymentSpec api.DeploymentSpec) time.Duration {
 	return a.timeout(deploymentSpec)
 }
 
-// Return the MemberID used / created in this action
+// MemberID returns the member ID used / created in the current action.
 func (a actionImpl) MemberID() string {
 	return *a.memberIDRef
 }

pkg/deployment/reconcile/action_runtime_container_args_udpate.go

@@ -0,0 +1,291 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech
+//
+
+package reconcile
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	core "k8s.io/api/core/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
+	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+)
+
+func init() {
+	registerAction(api.ActionTypeRuntimeContainerArgsLogLevelUpdate, runtimeContainerArgsUpdate)
+}
+
+func runtimeContainerArgsUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionRuntimeContainerArgsUpdate{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+var _ ActionReloadCachedStatus = &actionRuntimeContainerArgsUpdate{}
+var _ ActionPost = &actionRuntimeContainerArgsUpdate{}
+
+type actionRuntimeContainerArgsUpdate struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+}
+
+// Post updates arguments for the specific Arango member.
+func (a actionRuntimeContainerArgsUpdate) Post(ctx context.Context) error {
+
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("member is gone already")
+		return nil
+	}
+
+	memberName := m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group)
+	member, ok := a.actionCtx.GetCachedStatus().ArangoMember(memberName)
+	if !ok {
+		return errors.Errorf("ArangoMember %s not found", memberName)
+	}
+
+	containerName, ok := a.action.GetParam(rotation.ContainerName)
+	if !ok {
+		a.log.Warn().Msgf("Unable to find action's param %s", rotation.ContainerName)
+		return nil
+	}
+
+	log := a.log.With().Str("containerName", containerName).Logger()
+	updateMemberStatusArgs := func(obj *api.ArangoMember, s *api.ArangoMemberStatus) bool {
+		if obj.Spec.Template == nil || s.Template == nil ||
+			obj.Spec.Template.PodSpec == nil || s.Template.PodSpec == nil {
+			log.Info().Msgf("Nil Member definition")
+			return false
+		}
+
+		if len(obj.Spec.Template.PodSpec.Spec.Containers) != len(s.Template.PodSpec.Spec.Containers) {
+			log.Info().Msgf("Invalid size of containers")
+			return false
+		}
+
+		for id := range obj.Spec.Template.PodSpec.Spec.Containers {
+			if obj.Spec.Template.PodSpec.Spec.Containers[id].Name == containerName {
+				if s.Template.PodSpec.Spec.Containers[id].Name != containerName {
+					log.Info().Msgf("Invalid order of containers")
+					return false
+				}
+
+				s.Template.PodSpec.Spec.Containers[id].Command = obj.Spec.Template.PodSpec.Spec.Containers[id].Command
+				log.Info().Msgf("Updating container args")
+				return true
+			}
+		}
+
+		log.Info().Msgf("can not find the container")
+
+		return false
+	}
+
+	err := a.actionCtx.WithArangoMemberStatusUpdate(ctx, member.GetNamespace(), member.GetName(), updateMemberStatusArgs)
+	if err != nil {
+		return errors.WithMessage(err, "Error while updating member status")
+	}
+
+	return nil
+}
+
+// ReloadCachedStatus reloads the inspector cache when the action is done.
+func (a actionRuntimeContainerArgsUpdate) ReloadCachedStatus() bool {
+	return true
+}
+
+// Start starts the action for changing conditions on the provided member.
+func (a actionRuntimeContainerArgsUpdate) Start(ctx context.Context) (bool, error) {
+
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("member is gone already")
+		return true, nil
+	}
+
+	if !m.Phase.IsReady() {
+		a.log.Info().Msg("Member is not ready, unable to run update operation")
+		return true, nil
+	}
+
+	containerName, ok := a.action.GetParam(rotation.ContainerName)
+	if !ok {
+		return true, nil
+	}
+
+	memberName := m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group)
+	member, ok := a.actionCtx.GetCachedStatus().ArangoMember(memberName)
+	if !ok {
+		return false, errors.Errorf("ArangoMember %s not found", memberName)
+	}
+
+	pod, ok := a.actionCtx.GetCachedStatus().Pod(m.PodName)
+	if !ok {
+		a.log.Info().Str("podName", m.PodName).Msg("pod is not present")
+		return true, nil
+	}
+
+	var op cmpContainer = func(containerSpec core.Container, containerStatus core.Container) error {
+		topicsLogLevel := map[string]string{}
+
+		// Set log levels from the provided spec.
+		for _, arg := range containerSpec.Command {
+			if ok, topic, value := getTopicAndLevel(arg); ok {
+				topicsLogLevel[topic] = value
+			}
+		}
+
+		if err := a.setLogLevel(ctx, topicsLogLevel); err != nil {
+			return errors.WithMessage(err, "can not set log level")
+		}
+
+		a.log.Info().Interface("topics", topicsLogLevel).Msg("send log level to the ArangoDB")
+		return nil
+	}
+
+	if err := checkContainer(member, pod, containerName, op); err != nil && err != api.NotFoundError {
+		return false, errors.WithMessagef(err, "can not check the container %s", containerName)
+	}
+
+	return true, nil
+}
+
+type cmpContainer func(spec core.Container, status core.Container) error
+
+func checkContainer(member *api.ArangoMember, pod *core.Pod, containerName string, action cmpContainer) error {
+	spec, status, err := validateMemberAndPod(member, pod)
+	if err != nil {
+		return err
+	}
+
+	id := getIndexContainer(pod, spec, status, containerName)
+	if id < 0 {
+		return api.NotFoundError
+	}
+
+	return action(spec.Spec.Containers[id], status.Spec.Containers[id])
+}
+
+// getIndexContainer returns the index of the container from the list of containers.
+func getIndexContainer(pod *core.Pod, spec *core.PodTemplateSpec, status *core.PodTemplateSpec,
+	containerName string) int {
+
+	for id := range pod.Spec.Containers {
+		if pod.Spec.Containers[id].Name == spec.Spec.Containers[id].Name ||
+			pod.Spec.Containers[id].Name == status.Spec.Containers[id].Name ||
+			pod.Spec.Containers[id].Name == containerName {
+
+			return id
+		}
+	}
+
+	return -1
+}
+
+func validateMemberAndPod(member *api.ArangoMember, pod *core.Pod) (*core.PodTemplateSpec, *core.PodTemplateSpec, error) {
+
+	if member.Spec.Template == nil || member.Spec.Template.PodSpec == nil {
+		return nil, nil, fmt.Errorf("member spec is not present")
+	}
+
+	if member.Status.Template == nil || member.Status.Template.PodSpec == nil {
+		return nil, nil, fmt.Errorf("member status is not present")
+	}
+
+	if len(pod.Spec.Containers) != len(member.Spec.Template.PodSpec.Spec.Containers) {
+		return nil, nil, fmt.Errorf("spec container count is not equal")
+	}
+
+	if len(pod.Spec.Containers) != len(member.Status.Template.PodSpec.Spec.Containers) {
+		return nil, nil, fmt.Errorf("status container count is not equal")
+	}
+
+	return member.Spec.Template.PodSpec, member.Status.Template.PodSpec, nil
+}
+
+// CheckProgress returns always true because it does not have to wait for any result.
+func (a actionRuntimeContainerArgsUpdate) CheckProgress(_ context.Context) (bool, bool, error) {
+	return true, false, nil
+}
+
+// setLogLevel sets the log's levels for the specific server.
+func (a actionRuntimeContainerArgsUpdate) setLogLevel(ctx context.Context, logLevels map[string]string) error {
+	if len(logLevels) == 0 {
+		return nil
+	}
+
+	ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout())
+	defer cancel()
+	cli, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID)
+	if err != nil {
+		return err
+	}
+	conn := cli.Connection()
+
+	req, err := conn.NewRequest("PUT", "_admin/log/level")
+	if err != nil {
+		return err
+	}
+
+	if _, err := req.SetBody(logLevels); err != nil {
+		return err
+	}
+
+	ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
+	defer cancel()
+	resp, err := conn.Do(ctxChild, req)
+	if err != nil {
+		return err
+	}
+
+	return resp.CheckStatus(200)
+}
+
+// getTopicAndLevel returns topics and log level from the argument.
+func getTopicAndLevel(arg string) (bool, string, string) {
+	if !strings.HasPrefix(strings.TrimLeft(arg, " "), "--log.level") {
+		return false, "", ""
+	}
+
+	logLevelOption := k8sutil.ExtractStringToOptionPair(arg)
+	if len(logLevelOption.Value) > 0 {
+		logValueOption := k8sutil.ExtractStringToOptionPair(logLevelOption.Value)
+		if len(logValueOption.Value) > 0 {
+			// It is the topic log, e.g.: --log.level=request=INFO.
+			return true, logValueOption.Key, logValueOption.Value
+		} else {
+			// It is the general log, e.g.: --log.level=INFO.
+			return true, "general", logLevelOption.Value
+		}
+	}
+
+	return false, "", ""
+}

pkg/deployment/reconcile/plan_executor.go

@@ -188,7 +188,7 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
 
 			if err := getActionPost(action, ctx); err != nil {
 				log.Err(err).Msgf("Post action failed")
-				return nil, true, errors.WithStack(err)
+				return nil, false, errors.WithStack(err)
 			}
 		} else {
 			if plan[0].StartTime.IsZero() {
@@ -206,8 +206,7 @@ func (d *Reconciler) executeAction(ctx context.Context, log zerolog.Logger, plan
 		// Not started yet
 		ready, err := action.Start(ctx)
 		if err != nil {
-			log.Debug().Err(err).
-				Msg("Failed to start action")
+			log.Error().Err(err).Msg("Failed to start action")
 			return false, false, false, errors.WithStack(err)
 		}
 

pkg/deployment/rotation/arangod_containers.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,10 +17,15 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+// Author Tomasz Mielech
+//
 
 package rotation
 
 import (
+	"strings"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
@@ -32,7 +37,7 @@ const (
 	ContainerImage = "image"
 )
 
-func containersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) compareFunc {
+func containersCompare(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodSpec) compareFunc {
 	return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
 		a, b := spec.Containers, status.Containers
 
@@ -42,7 +47,15 @@ func containersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup,
 
 		for id := range a {
 			if ac, bc := &a[id], &b[id]; ac.Name == k8sutil.ServerContainerName && ac.Name == bc.Name {
-				// Nothing to do
+				if !IsOnlyLogLevelChanged(ac.Command, bc.Command) {
+					continue
+				}
+
+				plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerArgsLogLevelUpdate).
+					AddParam(ContainerName, ac.Name))
+
+				bc.Command = ac.Command
+				mode = mode.And(InPlaceRotation)
 			} else if ac.Name == bc.Name {
 				if ac.Image != bc.Image {
 					// Image changed
@@ -100,3 +113,20 @@ func initContainersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGr
 		return
 	}
 }
+
+// IsOnlyLogLevelChanged returns true when status and spec log level arguments are different.
+// If any other argument than --log.level is different false is returned.
+func IsOnlyLogLevelChanged(specArgs, statusArgs []string) bool {
+	diff := util.DiffStrings(specArgs, statusArgs)
+	if len(diff) == 0 {
+		return false
+	}
+
+	for _, arg := range diff {
+		if !strings.HasPrefix(strings.TrimLeft(arg, " "), "--log.level") {
+			return false
+		}
+	}
+
+	return true
+}

pkg/deployment/rotation/arangod_containers_test.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,17 +17,20 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+// Author Tomasz Mielech
+//
 
 package rotation
 
 import (
 	"testing"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
-
+	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 func Test_ArangoDContainers_SidecarImages(t *testing.T) {
@@ -144,3 +147,78 @@ func Test_InitContainers(t *testing.T) {
 		runTestCases(t)(testCases...)
 	})
 }
+
+func Test_Container_Args(t *testing.T) {
+	testCases := []TestCase{
+		{
+			name: "Only log level arguments of the ArangoDB server have been changed",
+			spec: buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName,
+				[]string{"--log.level=INFO", "--log.level=requests=error"})),
+			status:       buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName, []string{"--log.level=INFO"})),
+			expectedMode: InPlaceRotation,
+			expectedPlan: api.Plan{
+				api.NewAction(api.ActionTypeRuntimeContainerArgsLogLevelUpdate, 0, ""),
+			},
+		},
+		{
+			name: "Only log level arguments of the Sidecar have been changed",
+			spec: buildPodSpec(addContainerWithCommand("sidecar",
+				[]string{"--log.level=INFO", "--log.level=requests=error"})),
+			status:       buildPodSpec(addContainerWithCommand("sidecar", []string{"--log.level=INFO"})),
+			expectedMode: GracefulRotation,
+		},
+		{
+			name:   "ArangoDB server arguments have not been changed",
+			spec:   buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName, []string{"--log.level=INFO"})),
+			status: buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName, []string{"--log.level=INFO"})),
+		},
+		{
+			name: "Not only log level arguments of the ArangoDB server have been changed",
+			spec: buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName, []string{"--log.level=INFO",
+				"--server.endpoint=localhost"})),
+			status:       buildPodSpec(addContainerWithCommand(k8sutil.ServerContainerName, []string{"--log.level=INFO"})),
+			expectedMode: GracefulRotation,
+		},
+	}
+
+	runTestCases(t)(testCases...)
+}
+
+func TestIsOnlyLogLevelChanged(t *testing.T) {
+	type args struct {
+		specArgs   []string
+		statusArgs []string
+	}
+	tests := map[string]struct {
+		args args
+		want bool
+	}{
+		"log level not changed": {
+			args: args{
+				specArgs:   []string{"--log.level=INFO"},
+				statusArgs: []string{"--log.level=INFO"},
+			},
+		},
+		"log level changed": {
+			args: args{
+				specArgs:   []string{"--log.level=INFO", "--log.level=requests=DEBUG"},
+				statusArgs: []string{"--log.level=INFO"},
+			},
+			want: true,
+		},
+		"log level and server endpoint changed": {
+			args: args{
+				specArgs:   []string{"--log.level=INFO", "--log.level=requests=DEBUG", "--server.endpoint=localhost"},
+				statusArgs: []string{"--log.level=INFO"},
+			},
+		},
+	}
+
+	for testName, testCase := range tests {
+		t.Run(testName, func(t *testing.T) {
+			got := IsOnlyLogLevelChanged(testCase.args.specArgs, testCase.args.statusArgs)
+
+			assert.Equal(t, testCase.want, got)
+		})
+	}
+}

pkg/deployment/rotation/check.go

@@ -41,6 +41,7 @@ const (
 	EnforcedRotation
 )
 
+// And returns the higher value of the rotation mode.
 func (m Mode) And(b Mode) Mode {
 	if m > b {
 		return m

pkg/deployment/rotation/compare.go

@@ -21,8 +21,6 @@
 package rotation
 
 import (
-	"encoding/json"
-
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/rs/zerolog"
@@ -51,12 +49,16 @@ func compareFuncs(builder api.ActionBuilder, f ...compareFunc) (mode Mode, plan
 	return
 }
 
-func compare(log zerolog.Logger, deploymentSpec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup, spec, status *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, err error) {
+func compare(log zerolog.Logger, deploymentSpec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup,
+	spec, status *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, err error) {
+
 	if spec.Checksum == status.Checksum {
 		return SkippedRotation, nil, nil
 	}
 
-	mode = SkippedRotation
+	// If checksums are different and rotation is not needed and there are no changes between containers
+	// then silent rotation must be applied to adjust status checksum.
+	mode = SilentRotation
 
 	podStatus := status.PodSpec.DeepCopy()
 
@@ -79,21 +81,19 @@ func compare(log zerolog.Logger, deploymentSpec api.DeploymentSpec, member api.M
 		return SkippedRotation, nil, err
 	}
 
-	newSpec, err := api.GetArangoMemberPodTemplate(podStatus, checksum)
+	newStatus, err := api.GetArangoMemberPodTemplate(podStatus, checksum)
 	if err != nil {
 		log.Err(err).Msg("Error while getting template")
 		return SkippedRotation, nil, err
 	}
 
-	if spec.RotationNeeded(newSpec) {
-		l := log.Info().Str("id", member.ID).Str("Before", spec.PodSpecChecksum)
-		if d, err := json.Marshal(status); err == nil {
-			l = l.Str("status", string(d))
-		}
-		if d, err := json.Marshal(newSpec); err == nil {
-			l = l.Str("spec", string(d))
-		}
-		l.Msgf("Pod needs rotation - templates does not match")
+	if spec.RotationNeeded(newStatus) {
+		log.Info().Str("before", spec.PodSpecChecksum).
+			Str("id", member.ID).
+			Interface("spec", spec).
+			Interface("status", newStatus).
+			Msg("Pod needs rotation - templates does not match")
+
 		return GracefulRotation, nil, nil
 	}
 

pkg/deployment/rotation/utils_test.go

@@ -127,6 +127,12 @@ func addSidecarWithImage(name, image string) podSpecBuilder {
 	})
 }
 
+func addContainerWithCommand(name string, command []string) podSpecBuilder {
+	return addContainer(name, func(c *core.Container) {
+		c.Command = command
+	})
+}
+
 type deploymentBuilder func(depl *api.DeploymentSpec)
 
 func buildDeployment(b ...deploymentBuilder) api.DeploymentSpec {

pkg/util/k8sutil/pair.go

@@ -162,3 +162,17 @@ func (o OptionPair) CompareTo(other OptionPair) int {
 func NewOptionPair(pairs ...OptionPair) OptionPairs {
 	return pairs
 }
+
+// ExtractStringToOptionPair extracts command line argument into the OptionPair.
+func ExtractStringToOptionPair(arg string) OptionPair {
+	trimmed := strings.TrimLeft(arg, " ")
+	index := strings.Index(trimmed, "=")
+	if index < 0 {
+		return OptionPair{Key: strings.TrimRight(trimmed, " ")}
+	}
+
+	return OptionPair{
+		Key:   strings.Trim(trimmed[0:index], " "),
+		Value: trimmed[index+1:],
+	}
+}

pkg/util/k8sutil/pair_test.go

@@ -0,0 +1,78 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech
+//
+
+package k8sutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtractStringToOptionPair(t *testing.T) {
+	tests := map[string]struct {
+		arg  string
+		want OptionPair
+	}{
+		"empty argument": {},
+		"not trimmed key argument on the left side": {
+			arg: "           --log.level=requests=debug",
+			want: OptionPair{
+				Key:   "--log.level",
+				Value: "requests=debug",
+			},
+		},
+		"key argument not trimmed on the both sides": {
+			arg: "           --log.level   =requests=debug",
+			want: OptionPair{
+				Key:   "--log.level",
+				Value: "requests=debug",
+			},
+		},
+		"key argument not trimmed on the both sides without value": {
+			arg: "  --log.level  ",
+			want: OptionPair{
+				Key: "--log.level",
+			},
+		},
+		"key argument not trimmed on the both sides without value with equal": {
+			arg: "  --log.level =",
+			want: OptionPair{
+				Key: "--log.level",
+			},
+		},
+		"key argument not trimmed on the right side with some value": {
+			arg: "--log.level =  value   ",
+			want: OptionPair{
+				Key:   "--log.level",
+				Value: "  value   ",
+			},
+		},
+	}
+
+	for testName, testCase := range tests {
+		t.Run(testName, func(t *testing.T) {
+			got := ExtractStringToOptionPair(testCase.arg)
+			assert.Equal(t, testCase.want, got)
+		})
+	}
+}

pkg/util/strings.go

@@ -63,3 +63,28 @@ func PrefixStringArray(a []string, prefix string) []string {
 
 	return b
 }
+
+// DiffStringsOneWay returns the elements in `compareWhat` that are not in `compareTo`.
+func DiffStringsOneWay(compareWhat, compareTo []string) []string {
+	compareToMap := make(map[string]struct{}, len(compareTo))
+	for _, x := range compareTo {
+		compareToMap[x] = struct{}{}
+	}
+
+	var diff []string
+	for _, x := range compareWhat {
+		if _, found := compareToMap[x]; !found {
+			diff = append(diff, x)
+		}
+	}
+
+	return diff
+}
+
+// DiffStrings returns the elements in `compareWhat` that are not in `compareTo` and
+// elements in `compareTo` that are not in `compareFrom`.
+func DiffStrings(compareWhat, compareTo []string) []string {
+	diff := DiffStringsOneWay(compareWhat, compareTo)
+
+	return append(diff, DiffStringsOneWay(compareTo, compareWhat)...)
+}

pkg/util/strings_test.go

@@ -0,0 +1,148 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech
+//
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiff(t *testing.T) {
+	type args struct {
+		compareWhat []string
+		compareTo   []string
+	}
+	tests := map[string]struct {
+		args args
+		want []string
+	}{
+		"two nil slices": {},
+		"source slice is nil": {
+			args: args{
+				compareWhat: nil,
+				compareTo:   []string{"1"},
+			},
+			want: nil,
+		},
+		"destination slice is nil": {
+			args: args{
+				compareWhat: []string{"1"},
+			},
+			want: []string{"1"},
+		},
+		"source slice has more elements": {
+			args: args{
+				compareWhat: []string{"1", "2"},
+				compareTo:   []string{"2"},
+			},
+			want: []string{"1"},
+		},
+		"destination slice has more elements": {
+			args: args{
+				compareWhat: []string{"1", "2"},
+				compareTo:   []string{"1", "2", "3"},
+			},
+		},
+		"destination and source slices have overlapping elements": {
+			args: args{
+				compareWhat: []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"},
+				compareTo:   []string{"1", "3", "5", "7", "9"},
+			},
+			want: []string{"2", "4", "6", "8", "10"},
+		},
+		"destination slice contains source slice": {
+			args: args{
+				compareWhat: []string{"1", "3", "5", "7", "9"},
+				compareTo:   []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"},
+			},
+		},
+	}
+
+	for testName, testCase := range tests {
+		t.Run(testName, func(t *testing.T) {
+			diff := DiffStringsOneWay(testCase.args.compareWhat, testCase.args.compareTo)
+			assert.Equal(t, testCase.want, diff)
+		})
+	}
+}
+
+func TestDiffStrings(t *testing.T) {
+	type args struct {
+		compareWhat []string
+		compareTo   []string
+	}
+	tests := map[string]struct {
+		args args
+		want []string
+	}{
+		"two nil slices": {},
+		"source slice is nil": {
+			args: args{
+				compareTo: []string{"1"},
+			},
+			want: []string{"1"},
+		},
+		"destination slice is nil": {
+			args: args{
+				compareWhat: []string{"1"},
+			},
+			want: []string{"1"},
+		},
+		"source slice has more elements": {
+			args: args{
+				compareWhat: []string{"1", "2"},
+				compareTo:   []string{"2"},
+			},
+			want: []string{"1"},
+		},
+		"destination slice has more elements": {
+			args: args{
+				compareWhat: []string{"1", "2"},
+				compareTo:   []string{"1", "2", "3"},
+			},
+			want: []string{"3"},
+		},
+		"destination and source slices have all different elements": {
+			args: args{
+				compareWhat: []string{"1", "2", "3"},
+				compareTo:   []string{"4", "5", "6"},
+			},
+			want: []string{"1", "2", "3", "4", "5", "6"},
+		},
+		"destination and source slices have some overlapping elements": {
+			args: args{
+				compareWhat: []string{"1", "2", "3", "4"},
+				compareTo:   []string{"3", "4", "5", "6"},
+			},
+			want: []string{"1", "2", "5", "6"},
+		},
+	}
+	for testName, testCase := range tests {
+		t.Run(testName, func(t *testing.T) {
+			got := DiffStrings(testCase.args.compareWhat, testCase.args.compareTo)
+
+			assert.Equal(t, testCase.want, got)
+		})
+	}
+}