Add prometheus (#202)

* Add prometheus

microk8s-resources/actions/prometheus/prometheus-serviceMonitorKubeScheduler.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-scheduler
+  name: kube-scheduler
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: http-metrics
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kube-scheduler

microk8s-resources/actions/prometheus/prometheus-serviceMonitorKubelet.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kubelet
+  name: kubelet
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    port: https-metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    path: /metrics/cadvisor
+    port: https-metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kubelet

microk8s-resources/wrappers/microk8s-status.wrapper

@@ -13,13 +13,13 @@ KUBECTL="$SNAP/kubectl --kubeconfig=$SNAP/client.config"
 declare -A addon
 addon[dns]="pod/kube-dns"
 addon[dashboard]="pod/kubernetes-dashboard"
-addon[dashboard]="pod/kubernetes-dashboard"
 addon[ingress]="pod/nginx-ingress-microk8s-controller"
 addon[storage]="pod/hostpath-provisioner"
 addon[registry]="pod/registry"
 addon[metrics-server]="pod/metrics-server"
 addon[istio]="pod/istio"
 addon[gpu]="daemonset.apps/nvidia-device-plugin-daemonset"
+addon[prometheus]="pod/prometheus-k8s-0"
 
 
 function show_help() {

snapcraft.yaml

@@ -10,7 +10,7 @@ description: |-
  testing, or use it on a VM as a small, cheap, reliable k8s for CI/CD. It's 
  also a great k8s for appliances - develop your IoT apps for k8s and deploy 
  them to MicroK8s on your boxes.
- 
+
 grade: stable
 confinement: classic
 
@@ -190,6 +190,10 @@ parts:
         rm "actions/disable.istio.sh"
         # added because the completer is still defined
         touch istioctl.bash
+        # Prometheus support
+        rm "actions/enable.prometheus.sh"
+        rm "actions/disable.prometheus.sh"
+        rm -rf "actions/prometheus"
       else
         # Istio addon
         echo "Preparing istio"
@@ -277,3 +281,4 @@ parts:
       else
         echo "Skipped"
       fi
+

tests/test-addons.py

@@ -10,6 +10,7 @@ from validators import (
     validate_registry,
     validate_forward,
     validate_metrics_server,
+    validate_prometheus,
 )
 from utils import microk8s_enable, wait_for_pod_state, microk8s_disable, microk8s_reset
 from subprocess import Popen, PIPE, STDOUT, CalledProcessError
@@ -25,9 +26,9 @@ class TestAddons(object):
         yield
         microk8s_reset()
 
-    def test_dns_dashboard(self):
+    def test_basic(self):
         """
-        Sets up dashboard and validates it works.
+        Sets up and tests dashboard, dns, storage, registry, ingress.
 
         """
         print("Enabling DNS")
@@ -37,16 +38,6 @@ class TestAddons(object):
         microk8s_enable("dashboard")
         print("Validating dashboard")
         validate_dns_dashboard()
-        print("Disabling DNS")
-        microk8s_disable("dns")
-        print("Disabling dashboard")
-        microk8s_disable("dashboard")
-
-    def test_storage_registry(self):
-        """
-        Sets up and tests the storage addon and the private registry.
-
-        """
         print("Enabling storage")
         microk8s_enable("storage")
         print("Validating storage")
@@ -54,23 +45,23 @@ class TestAddons(object):
         microk8s_enable("registry")
         print("Validating registry")
         validate_registry()
-        print("Disabling registry")
-        microk8s_disable("registry")
-        print("Disabling storage")
-        p = Popen("/snap/bin/microk8s.disable storage".split(), stdout=PIPE, stdin=PIPE, stderr=STDOUT)
-        p.communicate(input=b'Y\n')[0]
-
-    def test_ingress(self):
-        """
-        Sets up ingress addon and validates it works.
-
-        """
         print("Enabling ingress")
         microk8s_enable("ingress")
         print("Validating ingress")
         validate_ingress()
         print("Disabling ingress")
         microk8s_disable("ingress")
+        print("Validating Port Forward")
+        validate_forward()
+        print("Disabling registry")
+        microk8s_disable("registry")
+        print("Disabling storage")
+        p = Popen("/snap/bin/microk8s.disable storage".split(), stdout=PIPE, stdin=PIPE, stderr=STDOUT)
+        p.communicate(input=b'Y\n')[0]
+        print("Disabling dashboard")
+        microk8s_disable("dashboard")
+        print("Disabling DNS")
+        microk8s_disable("dns")
 
     def test_gpu(self):
         """
@@ -116,14 +107,6 @@ class TestAddons(object):
         print("Disabling DNS")
         microk8s_disable("dns")
 
-    def test_forward(self):
-        """
-        Test port forward.
-
-        """
-        print("Validating Port Forward")
-        validate_forward()
-
     def test_metrics_server(self):
         """
         Test the metrics server.
@@ -136,3 +119,15 @@ class TestAddons(object):
         print("Disabling metrics-server")
         microk8s_disable("metrics-server")
 
+    def test_prometheus(self):
+        """
+        Test the Prometheus operator.
+
+        """
+        print("Enabling Prometheus")
+        microk8s_enable("prometheus")
+        print("Validating the Prometheus")
+        validate_prometheus()
+        print("Disabling Prometheus")
+        microk8s_disable("prometheus")
+

tests/test-live-addons.py

@@ -6,6 +6,7 @@ from validators import (
     validate_registry,
     validate_forward,
     validate_metrics_server,
+    validate_prometheus,
 )
 from utils import wait_for_pod_state
 
@@ -64,3 +65,10 @@ class TestLiveAddons(object):
 
         """
         validate_metrics_server()
+
+    def test_prometheus(self):
+        """
+        Validates prometheus addon.
+
+        """
+        validate_prometheus()

tests/test-upgrade.py

@@ -7,7 +7,8 @@ from validators import (
     validate_gpu,
     validate_registry,
     validate_forward,
-    validate_metrics_server
+    validate_metrics_server,
+    validate_prometheus
 )
 from subprocess import check_call, CalledProcessError, check_output
 from utils import (
@@ -105,6 +106,14 @@ class TestUpgrade(object):
         except:
             print('Will not test the metrics server')
 
+        try:
+            enable = microk8s_enable("prometheus", timeout_insec=30)
+            assert "Nothing to do for" not in enable
+            validate_prometheus()
+            test_matrix['prometheus'] = validate_prometheus
+        except:
+            print('Will not test the metrics server')
+
         # Refresh the snap to the target
         if upgrade_to.endswith('.snap'):
             cmd = "sudo snap install {} --classic --dangerous".format(upgrade_to)

tests/utils.py

@@ -75,12 +75,17 @@ def kubectl_get(target, timeout_insec=300):
     return yaml.load(output)
 
 
-def wait_for_pod_state(pod, namespace, desired_state, desired_reason=None, label=None):
+def wait_for_pod_state(pod, namespace, desired_state, desired_reason=None, label=None, timeout_insec=600):
     """
     Wait for a a pod state. If you do not specify a pod name and you set instead a label
     only the first pod will be checked.
     """
+    deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)
     while True:
+        if datetime.datetime.now() > deadline:
+            raise TimeoutError("Pod {} not in {} after {] seconds.".format(pod,
+                                                                           desired_state,
+                                                                           timeout_insec))
         cmd = 'po {} -n {}'.format(pod, namespace)
         if label:
             cmd += ' -l {}'.format(label)
@@ -129,12 +134,13 @@ def wait_for_installation():
     time.sleep(30)
 
 
-def microk8s_enable(addon):
+def microk8s_enable(addon, timeout_insec=300):
     """
     Disable an addon
 
     Args:
         addon: name of the addon
+        timeout_insec: seconds to keep retrying
 
     """
     # NVidia pre-check so as to not wait for a timeout.
@@ -145,7 +151,7 @@ def microk8s_enable(addon):
             raise CalledProcessError(1, "Nothing to do for gpu")
 
     cmd = '/snap/bin/microk8s.enable {}'.format(addon)
-    return run_until_success(cmd, timeout_insec=300)
+    return run_until_success(cmd, timeout_insec)
 
 
 def microk8s_disable(addon):