Tiered Storage config (#205)

* Add tiered storage config * Check Tiered Storage on README * GitHub PR #205 changes (1st round) Remove <= 2.6.0 configs. Add missing GCS secret volumeMount. Update GCS example name. * Cleanup comments * Bump chart version * GitHub PR #205 changes (2nd round) Moved storageOffload under broker section. Fixed some typos. Added AWS S3 IRSA annotation comment. * GitHub PR #205 changes (3rd round) Moved AWS and Azure credentials into K8S secrets using same StreamNative Helm Chart approach. * Trim trailing spaces --------- Co-authored-by: Lari Hotari <lhotari@apache.org> Co-authored-by: Marcelo Pellicano <mpellicanodeoliveira@bluecatnetworks.com>
2024-01-17 06:06:16 -05:00 · 2024-01-17 06:06:16 -05:00 · cfa156f738
commit cfa156f738
parent 18e67f2bf8
4 changed files with 180 additions and 1 deletions
--- a/README.md
+++ b/README.md
@ -64,7 +64,7 @@ It includes support for:
    - [x] Non-persistence storage
    - [x] Persistence Volume
    - [x] Local Persistent Volumes
-    - [ ] Tiered Storage
+    - [x] Tiered Storage
 - [x] Functions
    - [x] Kubernetes Runtime
    - [x] Process Runtime
--- a/charts/pulsar/templates/broker-configmap.yaml
+++ b/charts/pulsar/templates/broker-configmap.yaml
@ -43,6 +43,58 @@ data:
  zooKeeperSessionTimeoutMillis: "30000"
  statusFilePath: "{{ template "pulsar.home" . }}/status"

+  # Tiered storage settings
+  {{- if .Values.broker.storageOffload.driver }}
+  {{- if eq .Values.broker.storageOffload.driver "aws-s3" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  s3ManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}" 
+  s3ManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}" 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  s3ManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
+  s3ManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  gcsManagedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
+  gcsManagedLedgerOffloadRegion: "{{ .Values.broker.storageOffload.region }}"
+  gcsManagedLedgerOffloadServiceAccountKeyFile: "/pulsar/gcp-service-account/{{ .Values.broker.storageOffload.gcsServiceAccountJsonFile }}"
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  gcsManagedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.readBufferSizeInBytes }}
+  gcsManagedLedgerOffloadReadBufferSizeInBytes: "{{ .Values.broker.storageOffload.readBufferSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- if eq .Values.broker.storageOffload.driver "azureblob" }}
+  managedLedgerOffloadDriver: "{{ .Values.broker.storageOffload.driver }}" 
+  managedLedgerOffloadBucket: "{{ .Values.broker.storageOffload.bucket }}"
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}
+  PULSAR_PREFIX_managedLedgerOffloadThresholdInBytes: "{{ .Values.broker.storageOffload.managedLedgerOffloadAutoTriggerSizeThresholdBytes }}" 
+  {{- end }} 
+  {{- if .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}
+  PULSAR_PREFIX_managedLedgerOffloadDeletionLagInMillis: "{{ .Values.broker.storageOffload.managedLedgerOffloadDeletionLagMs }}" 
+  {{- end }}
+  {{- if .Values.broker.storageOffload.maxBlockSizeInBytes }}
+  managedLedgerOffloadMaxBlockSizeInBytes: "{{ .Values.broker.storageOffload.maxBlockSizeInBytes }}" 
+  {{- end }}
+  {{- end }}
+  {{- end }}
+
  # Function Worker Settings
  # function worker configuration
  {{- if not (or .Values.components.functions .Values.extra.functionsAsPods) }}
--- a/charts/pulsar/templates/broker-statefulset.yaml
+++ b/charts/pulsar/templates/broker-statefulset.yaml
@ -251,6 +251,13 @@ spec:
            readOnly: true
          {{- end }}
          {{- end }}
+          {{- if .Values.broker.storageOffload.driver }}
+          {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+          - name: gcp-service-account
+            readOnly: true
+            mountPath: /pulsar/gcp-service-account
+          {{- end }}
+          {{- end }}
          {{- if .Values.broker.extraVolumeMounts }}
 {{ toYaml .Values.broker.extraVolumeMounts | indent 10 }}
          {{- end }}
@ -259,6 +266,31 @@ spec:
        securityContext:
          readOnlyRootFilesystem: false
      {{- end }}
+      env:
+        {{- if and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "aws-s3") }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.broker.storageOffload.secret }}
+              key: AWS_ACCESS_KEY_ID
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.broker.storageOffload.secret }}
+              key: AWS_SECRET_ACCESS_KEY
+        {{- end }}
+        {{- if and .Values.broker.storageOffload (eq .Values.broker.storageOffload.driver "azureblob") }}
+        - name: AZURE_STORAGE_ACCOUNT
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.broker.storageOffload.secret }}
+              key: AZURE_STORAGE_ACCOUNT
+        - name: AZURE_STORAGE_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.broker.storageOffload.secret }}
+              key: AZURE_STORAGE_ACCESS_KEY
+        {{- end }}
      volumes:
      {{- if .Values.broker.extraVolumes }}
 {{ toYaml .Values.broker.extraVolumes | indent 6 }}
@ -289,6 +321,13 @@ spec:
              path: broker/token
      {{- end}}
      {{- end}}
+      {{- if .Values.broker.storageOffload.driver }}
+      {{- if eq .Values.broker.storageOffload.driver "google-cloud-storage" }}
+      - name: gcp-service-account
+        secret:
+          secretName: {{ .Values.broker.storageOffload.gcsServiceAccountSecret }}
+      {{- end }}
+      {{- end }}
      {{- include "pulsar.broker.certs.volumes" . | nindent 6 }}
      {{- include "pulsar.imagePullSecrets" . | nindent 6}}
 {{- end }}
--- a/charts/pulsar/values.yaml
+++ b/charts/pulsar/values.yaml
@ -420,6 +420,7 @@ zookeeper:
    usePolicy: true
    maxUnavailable: 1

+
 ## Pulsar: Bookkeeper cluster
 ## templates/bookkeeper-statefulset.yaml
 ##
@ -830,6 +831,93 @@ broker:
  ## templates/broker-service-account.yaml
  service_account:
    annotations: {}
+      ## You may use the following annotation in order to use EKS IAM Roles for Service Accounts (IRSA)
+      # eks.amazonaws.com/role-arn: arn:aws:iam::66666:role/my-iam-role-with-s3-access
+  ## Tiered Storage
+  ##
+  storageOffload: {}
+    ## General
+    ## =======
+    # maxBlockSizeInBytes: "64000000"
+    # readBufferSizeInBytes: "1000000"
+    ## The following are default values for the cluster. They can be changed
+    ## on each namespace.
+    # managedLedgerOffloadDeletionLagMs: "14400000"
+    # managedLedgerOffloadAutoTriggerSizeThresholdBytes: "-1" # disabled
+
+    ## For AWS S3
+    ## ======
+    ## Either you must create an IAM account with access to the bucket and
+    ## generate keys for that account, or use IAM Roles for Service Accounts (IRSA)
+    ## (example on `.Value.broker.service_account.annotations` section above)
+    ##
+    # driver: aws-s3
+    # bucket: <bucket>
+    # region: <region>
+    ## Secret that stores AWS credentials, using the following command:
+    ## ```
+    ## kubectl -n pulsar create secret generic \
+    ##   --from-literal=AWS_ACCESS_KEY_ID=<AWS ACCESS KEY> \
+    ##   --from-literal=AWS_SECRET_ACCESS_KEY=<AWS SECRET KEY> \
+    ##   <secret name>
+    ## ```
+    # secret: <secret name> # [k8s secret name that stores AWS credentials]
+
+    ## For S3 Compatible
+    ## =================
+    ## Need to create access and secret key for S3 compatible service
+    #
+    # driver: aws-s3
+    # bucket: <bucket>
+    # region: <region>
+    # serviceEndpoint: host:port
+    ## Secret that stores AWS credentials, using the following command:
+    ## ```
+    ## kubectl -n pulsar create secret generic \
+    ##   --from-literal=AWS_ACCESS_KEY_ID=<AWS ACCESS KEY> \
+    ##   --from-literal=AWS_SECRET_ACCESS_KEY=<AWS SECRET KEY> \
+    ##   <aws secret name>
+    ## ```
+    # secret: <aws secret name> # [k8s secret name that stores AWS credentials]
+
+    ## For Azure Blob
+    ## =================
+    ## Need to create an Azure storage account and a blob containter (bucket)
+    ## To retrieve key, see https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal#code-try-1
+    #
+    # driver: azureblob
+    # bucket: <bucket>
+    # region: <region>
+    ## Secret that stores AZURE credentials, using the following command:
+    ## ```
+    ## kubectl -n pulsar create secret generic \
+    ##   --from-literal=AZURE_STORAGE_ACCOUNT=<AZURE STORAGE ACCOUNT> \
+    ##   --from-literal=AZURE_STORAGE_ACCESS_KEY=<AZURE STORAGE ACCESS KEY> \
+    ##   <azure secret name>
+    ## ```
+    # secret: <azure secret name> # [k8s secret name that stores AZURE credentials]
+
+    ## For Google Cloud Storage
+    ## ====================
+    ## You must create a service account that has access to the objects in GCP buckets
+    ## and upload its key as a JSON file to a secret.
+    ##
+    ## 1. Go to https://console.cloud.google.com/iam-admin/serviceaccounts
+    ## 2. Select your project.
+    ## 3. Create a new service account.
+    ## 4. Give the service account permission to access the bucket. For example,
+    ##    the "Storage Object Admin" role.
+    ## 5. Create a key for the service account and save it as a JSON file.
+    ## 6. Save the JSON file in a secret:
+    ##       kubectl create secret generic pulsar-gcp-sa-secret \
+    ##          --from-file=google-service-account-key.json \
+    ##          --namespace pulsar
+    ##
+    # driver: google-cloud-storage
+    # bucket: <bucket>
+    # region: <region>
+    # gcsServiceAccountSecret: pulsar-gcp-sa-secret # pragma: allowlist secret
+    # gcsServiceAccountJsonFile: google-service-account-key.json

 ## Pulsar: Functions Worker
 ## templates/function-worker-configmap.yaml