Improve documentation and testing for PodMonitors (#317)

* Imrpove documentation and testing for PodMonitors

* Fix missed references and a typo

### Motivation

Before upgrading to 3.0.0, we want to make sure the kube-prometheus-stack is well documented. 

### Modifications

* Update tests and examples to fully disable `PodMonitors` and the installation of the kube-prometheus-stack CRDs.

### Verifying this change

The current tests will cover these changes.

.ci/clusters/values-bk-tls.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-broker-tls.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:

.ci/clusters/values-function.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -54,8 +71,16 @@ broker:
     PF_functionInstanceMinResources_ram: "268435456"
     PF_functionInstanceMinResources_disk: "268435456"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-jwt-asymmetric.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 2
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-jwt-symmetric.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-local-pv.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-pulsar-image.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 volumes:
   persistence: false
@@ -34,9 +42,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -45,6 +59,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -54,8 +71,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-tls.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-zk-tls.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

.ci/clusters/values-zkbk-tls.yaml

@@ -19,6 +19,14 @@
 
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 # disabled AntiAffinity
 affinity:
@@ -31,9 +39,15 @@ components:
 
 zookeeper:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     diskUsageThreshold: "0.999"
     diskUsageWarnThreshold: "0.999"
@@ -42,6 +56,9 @@ bookkeeper:
 
 broker:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -51,8 +68,16 @@ broker:
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
 
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
 proxy:
   replicaCount: 1
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
 
 toolset:
   useProxy: false

README.md

@@ -36,8 +36,8 @@ This Helm Chart includes all the components of Apache Pulsar for a complete expe
     - [x] Proxies
 - [x] Management & monitoring components:
     - [x] Pulsar Manager
-    - [x] Prometheus
+    - [x] Optional PodMonitors for each component (enabled by default)
-    - [x] Grafana
+    - [x] [Kube-Prometheus-Stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) (as of 3.0.0)
 
 It includes support for:
 
@@ -164,6 +164,27 @@ You can also checkout out the example values file for different deployments.
 - [Deploy a Pulsar cluster with JWT authentication using symmetric key](examples/values-jwt-symmetric.yaml)
 - [Deploy a Pulsar cluster with JWT authentication using asymmetric key](examples/values-jwt-asymmetric.yaml)
 
+## Disabling Kube-Prometheus-Stack CRDs
+
+In order to disable the kube-prometheus-stack fully, it is necessary to add the following to your `values.yaml`:
+
+```yaml
+kube-prometheus-stack:
+  enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
+```
+
+Otherwise, the helm chart installation will attempt to install the CRDs for the kube-prometheus-stack. Additionally,
+you'll need to disable each of the component's `PodMonitors`. This is shown in some [examples](./examples) and is
+verified in some [tests](./.ci/clusters).
+
 ## Upgrading
 
 Once your Pulsar Chart is installed, configuration changes and chart

examples/values-bookkeeper-aws.yaml

@@ -39,8 +39,19 @@ components:
 ## disable monitoring stack
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 zookeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   volumes:
     # use a persistent volume or emptyDir
     persistence: true
@@ -61,6 +72,9 @@ zookeeper:
         # provisioner: kubernetes.io/gce-pd
 bookkeeper:
   replicaCount: 3
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   volumes:
     # use a persistent volume or emptyDir
     persistence: true
@@ -78,3 +92,18 @@ bookkeeper:
       ## If you already have an existent storage class and want to reuse it, you can specify its name with the option below
       ##
       storageClassName: gp2
+
+broker:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+proxy:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false

examples/values-cs.yaml

@@ -38,4 +38,37 @@ components:
 
 ## disable monitoring stack
 kube-prometheus-stack:
-  enabled: false
+  enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
+
+zookeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+bookkeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+broker:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+proxy:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false

examples/values-local-cluster.yaml

@@ -29,4 +29,37 @@ components:
 
 ## disable monitoring stack
 kube-prometheus-stack:
-  enabled: false
+  enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
+
+zookeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+bookkeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+broker:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+proxy:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false

examples/values-zookeeper-aws.yaml

@@ -39,11 +39,22 @@ components:
 ## disable monitoring stack
 kube-prometheus-stack:
   enabled: false
+  prometheusOperator:
+    enabled: false
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    enabled: false
 
 zookeeper:
   # External zookeeper server list in case of global-zk list to create zk cluster across zk deployed on different clusters/namespaces
   # Example value: "us-east1-pulsar-zookeeper-0.us-east1-pulsar-zookeeper.us-east1.svc.cluster.local:2888:3888,us-east1-pulsar-zookeeper-1.us-east1-pulsar-zookeeper.us-east1.svc.cluster.local:2888:3888,us-east1-pulsar-zookeeper-2.us-east1-pulsar-zookeeper.us-east1.svc.cluster.local:2888:3888,us-west1-pulsar-zookeeper-0.us-west1-pulsar-zookeeper.us-west1.svc.cluster.local:2888:3888,us-west1-pulsar-zookeeper-1.us-west1-pulsar-zookeeper.us-west1.svc.cluster.local:2888:3888,us-west1-pulsar-zookeeper-2.us-west1-pulsar-zookeeper.us-west1.svc.cluster.local:2888:3888"
   externalZookeeperServerList: ""
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
   volumes:
     # use a persistent volume or emptyDir
     persistence: true
@@ -62,3 +73,23 @@ zookeeper:
         # type: pd-ssd
         # fsType: xfs
         # provisioner: kubernetes.io/gce-pd
+
+bookkeeper:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+autorecovery:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+broker:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false
+
+proxy:
+  # Disable pod monitor since we're disabling CRD installation
+  podMonitor:
+    enabled: false