tests: Add e2e alertmanager & prometheus exposure tests

Ensuring functionality of the "Exposing prometheus and alertmanaer"
tutorial. Testing exposure of alertmanager and prometheus via Nodeport,
Kubernetes API and Ingress controller.

Makefile

@@ -33,14 +33,14 @@ e2e-test:
 	go test -timeout 20m -v ./test/e2e/ $(TEST_RUN_ARGS) --kubeconfig "$(HOME)/.kube/config" --operator-image=quay.io/coreos/prometheus-operator:$(TAG) --namespace=$(NAMESPACE) --cluster-ip=$(CLUSTER_IP)
 
 e2e-status:
-	kubectl get prometheus,alertmanager,servicemonitor,statefulsets,deploy,svc,endpoints,pods,cm --all-namespaces
+	kubectl get prometheus,alertmanager,servicemonitor,statefulsets,deploy,svc,endpoints,pods,cm,replicationcontrollers --all-namespaces
 
 e2e:
 	$(MAKE) container
 	$(MAKE) e2e-test
 
 clean-e2e:
-	kubectl -n $(NAMESPACE) delete prometheus,alertmanager,servicemonitor,statefulsets,deploy,svc,endpoints,pods,cm --all
+	kubectl -n $(NAMESPACE) delete prometheus,alertmanager,servicemonitor,statefulsets,deploy,svc,endpoints,pods,cm,replicationcontrollers --all
 	kubectl delete namespace $(NAMESPACE)
 
 promu:

test/e2e/alertmanager_e2e_test.go

@@ -15,7 +15,11 @@
 package e2e
 
 import (
+	"fmt"
+	"k8s.io/client-go/pkg/api/v1"
+	"net/http"
 	"testing"
+	"time"
 )
 
 func TestAlertmanagerCreateDeleteCluster(t *testing.T) {
@@ -79,3 +83,103 @@ func TestAlertmanagerVersionMigration(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestExposingAlertmanagerWithNodePort(t *testing.T) {
+	alertmanager := framework.MakeBasicAlertmanager("test-alertmanager", 1)
+	alertmanagerService := framework.MakeAlertmanagerNodePortService(alertmanager.Name, "nodeport-service", 30903)
+
+	defer func() {
+		if err := framework.DeleteAlertmanagerAndWaitUntilGone(alertmanager.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(alertmanagerService.Name); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err := framework.CreateAlertmanagerAndWaitUntilReady(alertmanager); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(alertmanagerService); err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err := http.Get(fmt.Sprintf("http://%s:30903/", framework.ClusterIP))
+	if err != nil {
+		t.Fatal("Retrieving alertmanager landing page failed with error: ", err)
+	} else if resp.StatusCode != 200 {
+		t.Fatal("Retrieving alertmanager landing page failed with http status code: ", resp.StatusCode)
+	}
+}
+
+func TestExposingAlertmanagerWithKubernetesAPI(t *testing.T) {
+	alertmanager := framework.MakeBasicAlertmanager("test-alertmanager", 1)
+	alertmanagerService := framework.MakeAlertmanagerService(alertmanager.Name, "alertmanager-service", v1.ServiceTypeClusterIP)
+
+	defer func() {
+		if err := framework.DeleteAlertmanagerAndWaitUntilGone(alertmanager.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(alertmanagerService.Name); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err := framework.CreateAlertmanagerAndWaitUntilReady(alertmanager); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(alertmanagerService); err != nil {
+		t.Fatal(err)
+	}
+
+	proxyGet := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).ProxyGet
+	request := proxyGet("", alertmanagerService.Name, "web", "/", make(map[string]string))
+	_, err := request.DoRaw()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestExposingAlertmanagerWithIngress(t *testing.T) {
+	alertmanager := framework.MakeBasicAlertmanager("main", 1)
+	alertmanagerService := framework.MakeAlertmanagerService(alertmanager.Name, "test-group", v1.ServiceTypeClusterIP)
+	ingress := framework.MakeBasicIngress(alertmanagerService.Name, 9093)
+
+	defer func() {
+		if err := framework.DeleteAlertmanagerAndWaitUntilGone(alertmanager.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(alertmanagerService.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.KubeClient.Extensions().Ingresses(framework.Namespace.Name).Delete(ingress.Name, nil); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteNginxIngressControllerIncDefaultBackend(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err := framework.SetupNginxIngressControllerIncDefaultBackend(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateAlertmanagerAndWaitUntilReady(alertmanager); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(alertmanagerService); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateIngress(ingress); err != nil {
+		t.Fatal(err)
+	}
+
+	err := framework.WaitForHTTPSuccessStatusCode(time.Second*30, fmt.Sprintf("http://%s/metrics", framework.ClusterIP))
+	if err != nil {
+		t.Fatal(err)
+	}
+}

test/e2e/framework/alertmanager.go

@@ -53,8 +53,14 @@ func (f *Framework) MakeBasicAlertmanager(name string, replicas int32) *v1alpha1
 	}
 }
 
-func (f *Framework) MakeAlertmanagerService(name, group string) *v1.Service {
-	return &v1.Service{
+func (f *Framework) MakeAlertmanagerNodePortService(name, group string, nodePort int32) *v1.Service {
+	aMService := f.MakeAlertmanagerService(name, group, v1.ServiceTypeNodePort)
+	aMService.Spec.Ports[0].NodePort = nodePort
+	return aMService
+}
+
+func (f *Framework) MakeAlertmanagerService(name, group string, serviceType v1.ServiceType) *v1.Service {
+	service := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: fmt.Sprintf("alertmanager-%s", name),
 			Labels: map[string]string{
@@ -62,13 +68,12 @@ func (f *Framework) MakeAlertmanagerService(name, group string) *v1.Service {
 			},
 		},
 		Spec: v1.ServiceSpec{
-			Type: "NodePort",
+			Type: serviceType,
 			Ports: []v1.ServicePort{
 				v1.ServicePort{
 					Name:       "web",
 					Port:       9093,
 					TargetPort: intstr.FromString("web"),
-					NodePort:   30903,
 				},
 			},
 			Selector: map[string]string{
@@ -76,6 +81,8 @@ func (f *Framework) MakeAlertmanagerService(name, group string) *v1.Service {
 			},
 		},
 	}
+
+	return service
 }
 
 func (f *Framework) CreateAlertmanagerAndWaitUntilReady(a *v1alpha1.Alertmanager) error {

test/e2e/framework/framework.go

@@ -208,6 +208,19 @@ func waitForPodsReady(client v1client.CoreV1Interface, timeout time.Duration, ex
 	}
 }
 
+func (f *Framework) WaitForHTTPSuccessStatusCode(timeout time.Duration, url string) error {
+	return f.Poll(time.Minute*5, time.Second, func() (bool, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return false, err
+		}
+		if err == nil && resp.StatusCode == 200 {
+			return true, nil
+		}
+		return false, nil
+	})
+}
+
 func podRunsImage(p v1.Pod, image string) bool {
 	for _, c := range p.Spec.Containers {
 		if image == c.Image {
@@ -247,3 +260,24 @@ func (f *Framework) GetLogs(podName, containerName string) (string, error) {
 	}
 	return string(logs), err
 }
+
+func (f *Framework) Poll(timeout, pollInterval time.Duration, pollFunc func() (bool, error)) error {
+	t := time.After(timeout)
+	ticker := time.NewTicker(pollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-t:
+			return fmt.Errorf("timed out")
+		case <-ticker.C:
+			b, err := pollFunc()
+			if err != nil {
+				return err
+			}
+			if b {
+				return nil
+			}
+		}
+	}
+}

test/e2e/framework/ingress.go

@@ -0,0 +1,120 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framework
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
+	"k8s.io/client-go/pkg/util/intstr"
+	"os"
+)
+
+func (f *Framework) MakeBasicIngress(serviceName string, servicePort int) *v1beta1.Ingress {
+	return &v1beta1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "monitoring",
+		},
+		Spec: v1beta1.IngressSpec{
+			Rules: []v1beta1.IngressRule{
+				v1beta1.IngressRule{
+					IngressRuleValue: v1beta1.IngressRuleValue{
+						HTTP: &v1beta1.HTTPIngressRuleValue{
+							Paths: []v1beta1.HTTPIngressPath{
+								v1beta1.HTTPIngressPath{
+									Backend: v1beta1.IngressBackend{
+										ServiceName: serviceName,
+										ServicePort: intstr.FromInt(servicePort),
+									},
+									Path: "/metrics",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func (f *Framework) CreateIngress(i *v1beta1.Ingress) error {
+	_, err := f.KubeClient.Extensions().Ingresses(f.Namespace.Name).Create(i)
+	return err
+}
+
+func (f *Framework) SetupNginxIngressControllerIncDefaultBackend() error {
+	// Create Nginx Ingress Replication Controller
+	if err := createReplicationControllerViaYml("./framework/ressources/nxginx-ingress-controller.yml", f); err != nil {
+		return err
+	}
+
+	// Create Default HTTP Backend Replication Controller
+	if err := createReplicationControllerViaYml("./framework/ressources/default-http-backend.yml", f); err != nil {
+		return err
+	}
+
+	// Create Default HTTP Backend Service
+	manifest, err := os.Open("./framework/ressources/default-http-backend-service.yml")
+	if err != nil {
+		return err
+	}
+
+	service := v1.Service{}
+	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&service)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.KubeClient.CoreV1().Services(f.Namespace.Name).Create(&service)
+	if err != nil {
+		return err
+	}
+	if err := f.WaitForServiceReady(service.Name); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (f *Framework) DeleteNginxIngressControllerIncDefaultBackend() error {
+	// Delete Nginx Ingress Replication Controller
+	if err := deleteReplicationControllerViaYml("./framework/ressources/nxginx-ingress-controller.yml", f); err != nil {
+		return err
+	}
+
+	// Delete Default HTTP Backend Replication Controller
+	if err := deleteReplicationControllerViaYml("./framework/ressources/default-http-backend.yml", f); err != nil {
+		return err
+	}
+
+	// Delete Default HTTP Backend Service
+	manifest, err := os.Open("./framework/ressources/default-http-backend-service.yml")
+	if err != nil {
+		return err
+	}
+
+	service := v1.Service{}
+	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&service)
+	if err != nil {
+		return err
+	}
+
+	if err := f.KubeClient.CoreV1().Services(f.Namespace.Name).Delete(service.Name, nil); err != nil {
+		return err
+	}
+
+	return nil
+}

test/e2e/framework/prometheus.go

@@ -86,8 +86,14 @@ func (f *Framework) MakeBasicServiceMonitor(name string) *v1alpha1.ServiceMonito
 	}
 }
 
-func (f *Framework) MakePrometheusService(name, group string) *v1.Service {
-	return &v1.Service{
+func (f *Framework) MakeBasicPrometheusNodePortService(name, group string, nodePort int32) *v1.Service {
+	pService := f.MakePrometheusService(name, group, v1.ServiceTypeNodePort)
+	pService.Spec.Ports[0].NodePort = nodePort
+	return pService
+}
+
+func (f *Framework) MakePrometheusService(name, group string, serviceType v1.ServiceType) *v1.Service {
+	service := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: fmt.Sprintf("prometheus-%s", name),
 			Labels: map[string]string{
@@ -95,13 +101,12 @@ func (f *Framework) MakePrometheusService(name, group string) *v1.Service {
 			},
 		},
 		Spec: v1.ServiceSpec{
-			Type: "NodePort",
+			Type: serviceType,
 			Ports: []v1.ServicePort{
 				v1.ServicePort{
 					Name:       "web",
 					Port:       9090,
 					TargetPort: intstr.FromString("web"),
-					NodePort:   30900,
 				},
 			},
 			Selector: map[string]string{
@@ -109,6 +114,7 @@ func (f *Framework) MakePrometheusService(name, group string) *v1.Service {
 			},
 		},
 	}
+	return service
 }
 
 func (f *Framework) CreatePrometheusAndWaitUntilReady(p *v1alpha1.Prometheus) error {

test/e2e/framework/replication-controller.go

@@ -0,0 +1,88 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framework
+
+import (
+	apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/pkg/api/v1"
+	"os"
+	"time"
+)
+
+func createReplicationControllerViaYml(filepath string, f *Framework) error {
+	manifest, err := os.Open(filepath)
+	if err != nil {
+		return err
+	}
+
+	var rC v1.ReplicationController
+	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&rC)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.KubeClient.CoreV1().ReplicationControllers(f.Namespace.Name).Create(&rC)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deleteReplicationControllerViaYml(filepath string, f *Framework) error {
+	manifest, err := os.Open(filepath)
+	if err != nil {
+		return err
+	}
+
+	var rC v1.ReplicationController
+	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&rC)
+	if err != nil {
+		return err
+	}
+
+	if err := scaleDownReplicationController(f, rC); err != nil {
+		return err
+	}
+
+	if err := f.KubeClient.CoreV1().ReplicationControllers(f.Namespace.Name).Delete(rC.Name, nil); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func scaleDownReplicationController(f *Framework, rC v1.ReplicationController) error {
+	*rC.Spec.Replicas = 0
+	rCAPI := f.KubeClient.CoreV1().ReplicationControllers(f.Namespace.Name)
+
+	_, err := f.KubeClient.CoreV1().ReplicationControllers(f.Namespace.Name).Update(&rC)
+	if err != nil {
+		return err
+	}
+
+	return f.Poll(time.Minute*5, time.Second, func() (bool, error) {
+		currentRC, err := rCAPI.Get(rC.Name, apimetav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if currentRC.Status.Replicas == 0 {
+			return true, nil
+		}
+		return false, nil
+	})
+}

test/e2e/framework/ressources/default-http-backend-service.yml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  labels:
+    k8s-app: default-http-backend
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    k8s-app: default-http-backend

test/e2e/framework/ressources/default-http-backend.yml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: default-http-backend
+spec:
+  replicas: 1
+  selector:
+    k8s-app: default-http-backend
+  template:
+    metadata:
+      labels:
+        k8s-app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.2
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi

test/e2e/framework/ressources/nxginx-ingress-controller.yml

@@ -0,0 +1,55 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-lb
+spec:
+  replicas: 1
+  selector:
+    k8s-app: nginx-ingress-lb
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-lb
+        name: nginx-ingress-lb
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
+        name: nginx-ingress-lb
+        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        # use downward API
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        # we expose 18080 to access nginx stats in url /nginx-status
+        # this is optional
+        - containerPort: 18080
+          hostPort: 18080
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend

test/e2e/framework/service.go

@@ -0,0 +1,53 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framework
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/pkg/api/v1"
+	"time"
+)
+
+func (f *Framework) CreateServiceAndWaitUntilReady(service *v1.Service) error {
+	if _, err := f.KubeClient.CoreV1().Services(f.Namespace.Name).Create(service); err != nil {
+		return err
+	}
+
+	if err := f.WaitForServiceReady(service.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Framework) WaitForServiceReady(serviceName string) error {
+	err := f.Poll(time.Minute*5, time.Second, func() (bool, error) {
+		endpoints, err := f.KubeClient.CoreV1().Endpoints(f.Namespace.Name).Get(serviceName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if len(endpoints.Subsets) != 0 && len(endpoints.Subsets[0].Addresses) > 0 {
+			return true, nil
+		}
+		return false, nil
+	})
+	return err
+}
+
+func (f *Framework) DeleteService(serviceName string) error {
+	if err := f.KubeClient.CoreV1().Services(f.Namespace.Name).Delete(serviceName, nil); err != nil {
+		return err
+	}
+	return nil
+}

test/e2e/prometheus_e2e_test.go

@@ -125,7 +125,7 @@ func TestPrometheusReloadConfig(t *testing.T) {
 	}
 
 	// remounting a ConfigMap can take some time
-	err := poll(time.Minute*5, time.Second*20, func() (bool, error) {
+	err := framework.Poll(time.Minute*5, time.Second, func() (bool, error) {
 		logs, err := framework.GetLogs(fmt.Sprintf("prometheus-%s-0", name), "config-reloader")
 		if err != nil {
 			return false, err
@@ -168,7 +168,7 @@ func TestPrometheusReloadRules(t *testing.T) {
 	}
 
 	// remounting a ConfigMap can take some time
-	err = poll(time.Minute*5, time.Second*20, func() (bool, error) {
+	err = framework.Poll(time.Minute*5, time.Second, func() (bool, error) {
 		logs, err := framework.GetLogs(fmt.Sprintf("prometheus-%s-0", name), "config-reloader")
 		if err != nil {
 			return false, err
@@ -188,6 +188,7 @@ func TestPrometheusReloadRules(t *testing.T) {
 func TestPrometheusDiscovery(t *testing.T) {
 	prometheusName := "test"
 	group := "servicediscovery-test"
+	svc := framework.MakeBasicPrometheusNodePortService(prometheusName, group, 30900)
 
 	defer func() {
 		if err := framework.DeletePrometheusAndWaitUntilGone(prometheusName); err != nil {
@@ -196,21 +197,15 @@ func TestPrometheusDiscovery(t *testing.T) {
 		if err := framework.MonClient.ServiceMonitors(framework.Namespace.Name).Delete(group, nil); err != nil {
 			t.Fatal(err)
 		}
-		if err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Delete(fmt.Sprintf("prometheus-%s", prometheusName), nil); err != nil {
+		if err := framework.DeleteService(svc.Name); err != nil {
 			t.Fatal(err)
 		}
 	}()
 
-	log.Print("Creating Prometheus Service")
-	svc := framework.MakePrometheusService(prometheusName, group)
-	if _, err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Create(svc); err != nil {
-		t.Fatalf("Creating ServiceMonitor failed: %s", err)
-	}
-
 	log.Print("Creating Prometheus ServiceMonitor")
 	s := framework.MakeBasicServiceMonitor(group)
 	if _, err := framework.MonClient.ServiceMonitors(framework.Namespace.Name).Create(s); err != nil {
-		t.Fatalf("Creating ServiceMonitor failed: %s", err)
+		t.Fatalf("Creating ServiceMonitor failed: ", err)
 	}
 
 	p := framework.MakeBasicPrometheus(prometheusName, group, 1)
@@ -219,14 +214,19 @@ func TestPrometheusDiscovery(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	log.Print("Creating Prometheus Service")
+	if err := framework.CreateServiceAndWaitUntilReady(svc); err != nil {
+		t.Fatal(err)
+	}
+
 	log.Print("Validating Prometheus ConfigMap was created")
 	_, err := framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Get(fmt.Sprintf("prometheus-%s", prometheusName), apimetav1.GetOptions{})
 	if err != nil {
-		t.Fatalf("Generated ConfigMap could not be retrieved: %s", err)
+		t.Fatalf("Generated ConfigMap could not be retrieved: ", err)
 	}
 
 	log.Print("Validating Prometheus Targets were properly discovered")
-	err = poll(18*time.Minute, 30*time.Second, isDiscoveryWorking(prometheusName))
+	err = framework.Poll(18*time.Minute, time.Second, isDiscoveryWorking(prometheusName))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -236,12 +236,14 @@ func TestPrometheusAlertmanagerDiscovery(t *testing.T) {
 	prometheusName := "test"
 	alertmanagerName := "test"
 	group := "servicediscovery-test"
+	svc := framework.MakeBasicPrometheusNodePortService(prometheusName, group, 30900)
+	amsvc := framework.MakeAlertmanagerNodePortService(alertmanagerName, group, 30903)
 
 	defer func() {
 		if err := framework.DeleteAlertmanagerAndWaitUntilGone(alertmanagerName); err != nil {
 			t.Fatal(err)
 		}
-		if err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Delete(fmt.Sprintf("alertmanager-%s", alertmanagerName), nil); err != nil {
+		if err := framework.DeleteService(amsvc.Name); err != nil {
 			t.Fatal(err)
 		}
 		if err := framework.DeletePrometheusAndWaitUntilGone(prometheusName); err != nil {
@@ -250,23 +252,11 @@ func TestPrometheusAlertmanagerDiscovery(t *testing.T) {
 		if err := framework.MonClient.ServiceMonitors(framework.Namespace.Name).Delete(group, nil); err != nil {
 			t.Fatal(err)
 		}
-		if err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Delete(fmt.Sprintf("prometheus-%s", prometheusName), nil); err != nil {
+		if err := framework.DeleteService(svc.Name); err != nil {
 			t.Fatal(err)
 		}
 	}()
 
-	log.Print("Creating Prometheus Service")
-	svc := framework.MakePrometheusService(prometheusName, group)
-	if _, err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Create(svc); err != nil {
-		t.Fatalf("Creating ServiceMonitor failed: %s", err)
-	}
-
-	log.Print("Creating Prometheus ServiceMonitor")
-	s := framework.MakeBasicServiceMonitor(group)
-	if _, err := framework.MonClient.ServiceMonitors(framework.Namespace.Name).Create(s); err != nil {
-		t.Fatalf("Creating ServiceMonitor failed: %s", err)
-	}
-
 	p := framework.MakeBasicPrometheus(prometheusName, group, 1)
 	framework.AddAlertingToPrometheus(p, alertmanagerName)
 	p.Spec.Version = "v1.5.0"
@@ -274,26 +264,137 @@ func TestPrometheusAlertmanagerDiscovery(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	log.Print("Creating Prometheus Service")
+	if err := framework.CreateServiceAndWaitUntilReady(svc); err != nil {
+		t.Fatal(err)
+	}
+
+	log.Print("Creating Prometheus ServiceMonitor")
+	s := framework.MakeBasicServiceMonitor(group)
+	if _, err := framework.MonClient.ServiceMonitors(framework.Namespace.Name).Create(s); err != nil {
+		t.Fatalf("Creating ServiceMonitor failed: ", err)
+	}
+
 	log.Print("Validating Prometheus ConfigMap was created")
 	_, err := framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Get(fmt.Sprintf("prometheus-%s", prometheusName), apimetav1.GetOptions{})
 	if err != nil {
-		t.Fatalf("Generated ConfigMap could not be retrieved: %s", err)
+		t.Fatalf("Generated ConfigMap could not be retrieved: ", err)
 	}
 
-	log.Print("Creating Alertmanager Service")
-	amsvc := framework.MakeAlertmanagerService(alertmanagerName, group)
-	if _, err := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).Create(amsvc); err != nil {
-		t.Fatalf("Creating ServiceMonitor failed: %s", err)
-	}
-
-	time.Sleep(time.Minute)
-
 	if err := framework.CreateAlertmanagerAndWaitUntilReady(framework.MakeBasicAlertmanager(alertmanagerName, 3)); err != nil {
 		t.Fatal(err)
 	}
 
+	log.Print("Creating Alertmanager Service")
+	if err := framework.CreateServiceAndWaitUntilReady(amsvc); err != nil {
+		t.Fatal(err)
+	}
+
 	log.Print("Validating Prometheus properly discovered alertmanagers")
-	err = poll(18*time.Minute, 30*time.Second, isAlertmanagerDiscoveryWorking(alertmanagerName))
+	err = framework.Poll(18*time.Minute, time.Second, isAlertmanagerDiscoveryWorking(alertmanagerName))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestExposingPrometheusWithNodePort(t *testing.T) {
+	basicPrometheus := framework.MakeBasicPrometheus("test", "test", 1)
+	service := framework.MakeBasicPrometheusNodePortService(basicPrometheus.Name, "nodeport-service", 30900)
+
+	defer func() {
+		if err := framework.DeletePrometheusAndWaitUntilGone(basicPrometheus.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(service.Name); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err := framework.CreatePrometheusAndWaitUntilReady(basicPrometheus); err != nil {
+		t.Fatal("Creating prometheus failed: ", err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(service); err != nil {
+		t.Fatal("Creating prometheus service failed: ", err)
+	}
+
+	resp, err := http.Get(fmt.Sprintf("http://%s:30900/metrics", framework.ClusterIP))
+	if err != nil {
+		t.Fatal("Retrieving prometheus metrics failed with error: ", err)
+	} else if resp.StatusCode != 200 {
+		t.Fatal("Retrieving prometheus metrics failed with http status code: ", resp.StatusCode)
+	}
+}
+
+func TestExposingPrometheusWithKubernetesAPI(t *testing.T) {
+	basicPrometheus := framework.MakeBasicPrometheus("basic-prometheus", "test-group", 1)
+	service := framework.MakePrometheusService(basicPrometheus.Name, "test-group", v1.ServiceTypeClusterIP)
+
+	defer func() {
+		if err := framework.DeletePrometheusAndWaitUntilGone(basicPrometheus.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(service.Name); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err := framework.CreatePrometheusAndWaitUntilReady(basicPrometheus); err != nil {
+		t.Fatal("Creating prometheus failed: ", err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(service); err != nil {
+		t.Fatal("Creating prometheus service failed: ", err)
+	}
+
+	ProxyGet := framework.KubeClient.CoreV1().Services(framework.Namespace.Name).ProxyGet
+	request := ProxyGet("", service.Name, "web", "/metrics", make(map[string]string))
+	_, err := request.DoRaw()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestExposingPrometheusWithIngress(t *testing.T) {
+	prometheus := framework.MakeBasicPrometheus("main", "test-group", 1)
+	prometheusService := framework.MakePrometheusService(prometheus.Name, "test-group", v1.ServiceTypeClusterIP)
+	ingress := framework.MakeBasicIngress(prometheusService.Name, 9090)
+
+	defer func() {
+		if err := framework.DeletePrometheusAndWaitUntilGone(prometheus.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteService(prometheusService.Name); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.KubeClient.Extensions().Ingresses(framework.Namespace.Name).Delete(ingress.Name, nil); err != nil {
+			t.Fatal(err)
+		}
+		if err := framework.DeleteNginxIngressControllerIncDefaultBackend(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := framework.SetupNginxIngressControllerIncDefaultBackend()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = framework.CreatePrometheusAndWaitUntilReady(prometheus)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := framework.CreateServiceAndWaitUntilReady(prometheusService); err != nil {
+		t.Fatal(err)
+	}
+
+	err = framework.CreateIngress(ingress)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = framework.WaitForHTTPSuccessStatusCode(time.Second*30, fmt.Sprintf("http://%s:/metrics", framework.ClusterIP))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -471,24 +572,3 @@ type prometheusAlertmanagerAPIResponse struct {
 	Status string                 `json:"status"`
 	Data   *alertmanagerDiscovery `json:"data"`
 }
-
-func poll(timeout, pollInterval time.Duration, pollFunc func() (bool, error)) error {
-	t := time.After(timeout)
-	ticker := time.NewTicker(pollInterval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-t:
-			return fmt.Errorf("timed out")
-		case <-ticker.C:
-			b, err := pollFunc()
-			if err != nil {
-				return err
-			}
-			if b {
-				return nil
-			}
-		}
-	}
-}