mirrors/kube-arangodb
1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00
Code Activity

[Feature] Add support for managed services (#1016)

Browse source
This commit is contained in:
Tomasz Mielech 2022-07-16 07:11:28 +02:00 committed by GitHub
parent fcb9b35130
commit 07d6e01545
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 141 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -5,6 +5,7 @@
- (Refactor) Use cached member's clients
- (Feature) Move PVC resize action to high-priority plan
- (Feature) Remove forgotten ArangoDB jobs during restart
- (Feature) Add support for managed services
## [1.2.14](https://github.com/arangodb/kube-arangodb/tree/1.2.14) (2022-07-14)
- (Feature) Add ArangoSync TLS based rotation

13
pkg/apis/deployment/v1/external_access_spec.go
View file

@ -41,8 +41,11 @@ type ExternalAccessSpec struct {
// cloud-provider does not support the feature.
// More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/
LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
// Advertised Endpoint is passed to the coordinators/single servers for advertising a specific endpoint
// AdvertisedEndpoint is passed to the coordinators/single servers for advertising a specific endpoint
AdvertisedEndpoint *string `json:"advertisedEndpoint,omitempty"`
// ManagedServiceNames keeps names of services which are not managed by KubeArangoDB.
// It is only relevant when type of service is `managed`.
ManagedServiceNames []string `json:"managedServiceNames,omitempty"`
}
// GetType returns the value of type.
@ -65,6 +68,11 @@ func (s ExternalAccessSpec) GetAdvertisedEndpoint() string {
return util.StringOrDefault(s.AdvertisedEndpoint)
}
// GetManagedServiceNames returns a list of managed service names.
func (s ExternalAccessSpec) GetManagedServiceNames() []string {
return s.ManagedServiceNames
}
// HasAdvertisedEndpoint return whether an advertised endpoint was specified or not
func (s ExternalAccessSpec) HasAdvertisedEndpoint() bool {
return s.AdvertisedEndpoint != nil
@ -110,6 +118,9 @@ func (s *ExternalAccessSpec) SetDefaultsFrom(source ExternalAccessSpec) {
if s.AdvertisedEndpoint == nil {
s.AdvertisedEndpoint = source.AdvertisedEndpoint
}
if s.ManagedServiceNames == nil && len(source.ManagedServiceNames) > 0 {
s.ManagedServiceNames = append([]string{}, source.ManagedServiceNames...)
}
}
// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec.

6
pkg/apis/deployment/v1/external_access_type.go
View file

@ -38,12 +38,15 @@ const (
ExternalAccessTypeLoadBalancer ExternalAccessType = "LoadBalancer"
// ExternalAccessTypeNodePort yields a cluster with a service of type `NodePort` to provide external access
ExternalAccessTypeNodePort ExternalAccessType = "NodePort"
// ExternalAccessTypeManaged yields a cluster with a service which controls only selector.
ExternalAccessTypeManaged ExternalAccessType = "Managed"
)
func (t ExternalAccessType) IsNone() bool { return t == ExternalAccessTypeNone }
func (t ExternalAccessType) IsAuto() bool { return t == ExternalAccessTypeAuto }
func (t ExternalAccessType) IsLoadBalancer() bool { return t == ExternalAccessTypeLoadBalancer }
func (t ExternalAccessType) IsNodePort() bool { return t == ExternalAccessTypeNodePort }
func (t ExternalAccessType) IsManaged() bool { return t == ExternalAccessTypeManaged }
// AsServiceType returns the k8s ServiceType for this ExternalAccessType.
// If type is "Auto", ServiceTypeLoadBalancer is returned.
@ -62,7 +65,8 @@ func (t ExternalAccessType) AsServiceType() core.ServiceType {
// Return errors when validation fails, nil on success.
func (t ExternalAccessType) Validate() error {
switch t {
case ExternalAccessTypeNone, ExternalAccessTypeAuto, ExternalAccessTypeLoadBalancer, ExternalAccessTypeNodePort:
case ExternalAccessTypeNone, ExternalAccessTypeAuto, ExternalAccessTypeLoadBalancer, ExternalAccessTypeNodePort,
ExternalAccessTypeManaged:
return nil
default:
return errors.WithStack(errors.Wrapf(ValidationError, "Unknown external access type: '%s'", string(t)))

5
pkg/apis/deployment/v1/zz_generated.deepcopy.go generated
View file

@ -1517,6 +1517,11 @@ func (in *ExternalAccessSpec) DeepCopyInto(out *ExternalAccessSpec) {
*out = new(string)
**out = **in
}
if in.ManagedServiceNames != nil {
in, out := &in.ManagedServiceNames, &out.ManagedServiceNames
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}

13
pkg/apis/deployment/v2alpha1/external_access_spec.go
View file

@ -41,8 +41,11 @@ type ExternalAccessSpec struct {
// cloud-provider does not support the feature.
// More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/
LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
// Advertised Endpoint is passed to the coordinators/single servers for advertising a specific endpoint
// AdvertisedEndpoint is passed to the coordinators/single servers for advertising a specific endpoint
AdvertisedEndpoint *string `json:"advertisedEndpoint,omitempty"`
// ManagedServiceNames keeps names of services which are not managed by KubeArangoDB.
// It is only relevant when type of service is `managed`.
ManagedServiceNames []string `json:"managedServiceNames,omitempty"`
}
// GetType returns the value of type.
@ -65,6 +68,11 @@ func (s ExternalAccessSpec) GetAdvertisedEndpoint() string {
return util.StringOrDefault(s.AdvertisedEndpoint)
}
// GetManagedServiceNames returns a list of managed service names.
func (s ExternalAccessSpec) GetManagedServiceNames() []string {
return s.ManagedServiceNames
}
// HasAdvertisedEndpoint return whether an advertised endpoint was specified or not
func (s ExternalAccessSpec) HasAdvertisedEndpoint() bool {
return s.AdvertisedEndpoint != nil
@ -110,6 +118,9 @@ func (s *ExternalAccessSpec) SetDefaultsFrom(source ExternalAccessSpec) {
if s.AdvertisedEndpoint == nil {
s.AdvertisedEndpoint = source.AdvertisedEndpoint
}
if s.ManagedServiceNames == nil && len(source.ManagedServiceNames) > 0 {
s.ManagedServiceNames = append([]string{}, source.ManagedServiceNames...)
}
}
// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec.

6
pkg/apis/deployment/v2alpha1/external_access_type.go
View file

@ -38,12 +38,15 @@ const (
ExternalAccessTypeLoadBalancer ExternalAccessType = "LoadBalancer"
// ExternalAccessTypeNodePort yields a cluster with a service of type `NodePort` to provide external access
ExternalAccessTypeNodePort ExternalAccessType = "NodePort"
// ExternalAccessTypeManaged yields a cluster with a service which controls only selector.
ExternalAccessTypeManaged ExternalAccessType = "Managed"
)
func (t ExternalAccessType) IsNone() bool { return t == ExternalAccessTypeNone }
func (t ExternalAccessType) IsAuto() bool { return t == ExternalAccessTypeAuto }
func (t ExternalAccessType) IsLoadBalancer() bool { return t == ExternalAccessTypeLoadBalancer }
func (t ExternalAccessType) IsNodePort() bool { return t == ExternalAccessTypeNodePort }
func (t ExternalAccessType) IsManaged() bool { return t == ExternalAccessTypeManaged }
// AsServiceType returns the k8s ServiceType for this ExternalAccessType.
// If type is "Auto", ServiceTypeLoadBalancer is returned.
@ -62,7 +65,8 @@ func (t ExternalAccessType) AsServiceType() core.ServiceType {
// Return errors when validation fails, nil on success.
func (t ExternalAccessType) Validate() error {
switch t {
case ExternalAccessTypeNone, ExternalAccessTypeAuto, ExternalAccessTypeLoadBalancer, ExternalAccessTypeNodePort:
case ExternalAccessTypeNone, ExternalAccessTypeAuto, ExternalAccessTypeLoadBalancer, ExternalAccessTypeNodePort,
ExternalAccessTypeManaged:
return nil
default:
return errors.WithStack(errors.Wrapf(ValidationError, "Unknown external access type: '%s'", string(t)))

5
pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go generated
View file

@ -1517,6 +1517,11 @@ func (in *ExternalAccessSpec) DeepCopyInto(out *ExternalAccessSpec) {
*out = new(string)
**out = **in
}
if in.ManagedServiceNames != nil {
in, out := &in.ManagedServiceNames, &out.ManagedServiceNames
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}

110
pkg/deployment/resources/services.go
View file

@ -28,11 +28,13 @@ import (
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/deployment/features"
"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
"github.com/arangodb/kube-arangodb/pkg/metrics"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/globals"
@ -231,8 +233,8 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn
if single {
role = "single"
}
if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, role, "database",
shared.ArangoPort, false, withLeader, spec.ExternalAccess, apiObject); err != nil {
if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, role, shared.ArangoPort,
false, withLeader, spec.ExternalAccess, apiObject); err != nil {
return errors.WithStack(err)
}
@ -240,8 +242,7 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn
// External (and internal) Sync master service
counterMetric.Inc()
eaServiceName := k8sutil.CreateSyncMasterClientServiceName(deploymentName)
role := "syncmaster"
if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, role, "sync",
if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, api.ServerGroupSyncMastersString,
shared.ArangoSyncMasterPort, true, false, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject); err != nil {
return errors.WithStack(err)
}
@ -274,13 +275,18 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn
return reconcileRequired.Reconcile(ctx)
}
// EnsureServices creates all services needed to service the deployment
// ensureExternalAccessServices ensures all services needed for a deployment.
func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStatus inspectorInterface.Inspector,
svcs servicev1.ModInterface, eaServiceName, svcRole, title string, port int, noneIsClusterIP bool, withLeader bool,
svcs servicev1.ModInterface, eaServiceName, svcRole string, port int, noneIsClusterIP bool, withLeader bool,
spec api.ExternalAccessSpec, apiObject k8sutil.APIObject) error {
log := r.log.Str("section", "service-ea")
// Database external access service
if spec.GetType().IsManaged() {
// Managed services should not be created or removed by the operator.
return r.ensureExternalAccessManagedServices(ctx, cachedStatus, svcs, eaServiceName, svcRole, spec, apiObject,
withLeader)
}
log := r.log.Str("section", "service-ea").Str("role", svcRole).Str("service", eaServiceName)
createExternalAccessService := false
deleteExternalAccessService := false
eaServiceType := spec.GetType().AsServiceType() // Note: Type auto defaults to ServiceTypeLoadBalancer
@ -307,7 +313,7 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat
// See if LoadBalancer has been configured & the service is "old enough"
oldEnoughTimestamp := time.Now().Add(-1 * time.Minute) // How long does the load-balancer provisioner have to act.
if len(existing.Status.LoadBalancer.Ingress) == 0 && existing.GetObjectMeta().GetCreationTimestamp().Time.Before(oldEnoughTimestamp) {
log.Str("service", eaServiceName).Info("LoadBalancerIP of %s external access service is not set, switching to NodePort", title)
log.Info("LoadBalancerIP of external access service is not set, switching to NodePort")
createExternalAccessService = true
eaServiceType = core.ServiceTypeNodePort
deleteExternalAccessService = true // Remove the LoadBalancer ex service, then add the NodePort one
@ -340,7 +346,7 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat
return err
})
if err != nil {
log.Err(err).Debug("Failed to update %s external access service", title)
log.Err(err).Debug("Failed to update external access service")
return errors.WithStack(err)
}
}
@ -352,12 +358,12 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat
}
if deleteExternalAccessService {
log.Str("service", eaServiceName).Info("Removing obsolete %s external access service", title)
log.Info("Removing obsolete external access service")
err := globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
return svcs.Delete(ctxChild, eaServiceName, meta.DeleteOptions{})
})
if err != nil {
log.Err(err).Debug("Failed to remove %s external access service", title)
log.Err(err).Debug("Failed to remove external access service")
return errors.WithStack(err)
}
}
@ -371,12 +377,88 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat
_, newlyCreated, err := k8sutil.CreateExternalAccessService(ctxChild, svcs, eaServiceName, svcRole, apiObject,
eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner(), withLeader)
if err != nil {
log.Err(err).Debug("Failed to create %s external access service", title)
log.Err(err).Debug("Failed to create external access service")
return errors.WithStack(err)
}
if newlyCreated {
log.Str("service", eaServiceName).Debug("Created %s external access service", title)
log.Debug("Created %s external access service")
}
}
return nil
}
// ensureExternalAccessServices ensures if there are correct selectors on a managed services.
// If hardcoded external service names are not on the list of managed services then it will be checked additionally.
func (r *Resources) ensureExternalAccessManagedServices(ctx context.Context, cachedStatus inspectorInterface.Inspector,
services servicev1.ModInterface, eaServiceName, svcRole string, spec api.ExternalAccessSpec,
apiObject k8sutil.APIObject, withLeader bool) error {
log := r.log.Str("section", "service-ea").Str("role", svcRole).Str("service", eaServiceName)
managedServiceNames := spec.GetManagedServiceNames()
deploymentName := apiObject.GetName()
var selector map[string]string
if withLeader {
selector = k8sutil.LabelsForLeaderMember(deploymentName, svcRole, "")
} else {
selector = k8sutil.LabelsForDeployment(deploymentName, svcRole)
}
// Check if hardcoded service has correct selector.
if svc, ok := cachedStatus.Service().V1().GetSimple(eaServiceName); !ok {
// Hardcoded service (e.g. <deplname>-ea or <deplname>-sync) is not mandatory in `managed` type.
if len(managedServiceNames) == 0 {
log.Warn("the field \"spec.externalAccess.managedServiceNames\" should be provided for \"managed\" service type")
return nil
}
} else if changed, err := ensureManagedServiceSelector(ctx, selector, svc, services); err != nil {
return errors.WithMessage(err, "failed to ensure service selector")
} else if changed {
log.Info("selector applied to the managed service \"%s\"", svc.GetName())
}
for _, svcName := range managedServiceNames {
if svcName == eaServiceName {
// Hardcoded service has been applied before this loop.
continue
}
svc, ok := cachedStatus.Service().V1().GetSimple(svcName)
if !ok {
log.Warn("managed service \"%s\" should have existed", svcName)
continue
}
if changed, err := ensureManagedServiceSelector(ctx, selector, svc, services); err != nil {
return errors.WithMessage(err, "failed to ensure service selector")
} else if changed {
log.Info("selector applied to the managed service \"%s\"", svcName)
}
}
return nil
}
// ensureManagedServiceSelector ensures if there is correct selector on a service.
func ensureManagedServiceSelector(ctx context.Context, selector map[string]string, svc *core.Service,
services servicev1.ModInterface) (bool, error) {
for key, value := range selector {
if currentValue, ok := svc.Spec.Selector[key]; ok && value == currentValue {
continue
}
p := patch.NewPatch()
p.ItemReplace(patch.NewPath("spec", "selector"), selector)
data, err := p.Marshal()
if err != nil {
return false, errors.WithMessage(err, "failed to marshal service selector")
}
if _, err = services.Patch(ctx, svc.GetName(), types.JSONPatchType, data, meta.PatchOptions{}); err != nil {
return false, errors.WithMessage(err, "failed to patch service selector")
}
return true, nil
}
return false, nil
}