From c5247b92dfa6e4f37d45b3fb3a2bc6a9f199ef5c Mon Sep 17 00:00:00 2001
From: andyzhangx <xiazhang@microsoft.com>
Date: Tue, 21 Sep 2021 03:28:25 +0000
Subject: [PATCH 1/3] feat: support arm64 image build

chore: fix container build failure

revert

fix build failure
---
 Dockerfile |  2 +-
 Makefile   | 38 +++++++++++++++++++++++++++++++++++---
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index af10df3b..614e158e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,7 +17,7 @@ ARG ARCH=amd64
 FROM k8s.gcr.io/build-image/debian-base:buster-v1.6.0
 
 # Copy nfsplugin from build _output directory
-COPY bin/nfsplugin /nfsplugin
+COPY bin/${ARCH}/nfsplugin /nfsplugin
 
 # this is a workaround to install nfs-common & nfs-kernel-server and don't quit with error
 # https://github.com/kubernetes-sigs/blob-csi-driver/issues/214#issuecomment-781602430
diff --git a/Makefile b/Makefile
index c294a4d7..02efde86 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,12 @@ IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGENAME):latest
 E2E_HELM_OPTIONS ?= --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) --set image.nfs.tag=$(IMAGE_VERSION) --set image.nfs.pullPolicy=Always
 E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS}
 
+# Output type of docker buildx build
+OUTPUT_TYPE ?= docker
+
+ALL_ARCH.linux = arm64 amd64
+ALL_OS_ARCH = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
+
 all: nfs
 
 .PHONY: verify
@@ -88,20 +94,46 @@ local-k8s-uninstall:
 
 .PHONY: nfs
 nfs:
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags "${LDFLAGS} ${EXT_LDFLAGS}" -mod vendor -o bin/nfsplugin ./cmd/nfsplugin
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -a -ldflags "${LDFLAGS} ${EXT_LDFLAGS}" -mod vendor -o bin/${ARCH}/nfsplugin ./cmd/nfsplugin
+
+.PHONY: container-build
+container-build:
+	docker buildx build --pull --output=type=$(OUTPUT_TYPE) --platform="linux/$(ARCH)" \
+		-t $(IMAGE_TAG)-linux-$(ARCH) --build-arg ARCH=$(ARCH) .
 
 .PHONY: container
-container: nfs
-	docker build --no-cache -t $(IMAGE_TAG) .
+container:
+	docker buildx rm container-builder || true
+	docker buildx create --use --name=container-builder
+	# enable qemu for arm64 build
+	# https://github.com/docker/buildx/issues/464#issuecomment-741507760
+	docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64
+	docker run --rm --privileged tonistiigi/binfmt --install all
+	for arch in $(ALL_ARCH.linux); do \
+		ARCH=$${arch} $(MAKE) nfs; \
+		ARCH=$${arch} $(MAKE) container-build; \
+	done
 
 .PHONY: push
 push:
+ifdef CI
+	docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
+	docker manifest push --purge $(IMAGE_TAG)
+	docker manifest inspect $(IMAGE_TAG)
+else
 	docker push $(IMAGE_TAG)
+endif
 
 .PHONY: push-latest
 push-latest:
+ifdef CI
+	docker manifest create --amend $(IMAGE_TAG_LATEST) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
+	docker manifest push --purge $(IMAGE_TAG_LATEST)
+	docker manifest inspect $(IMAGE_TAG_LATEST)
+else
 	docker tag $(IMAGE_TAG) $(IMAGE_TAG_LATEST)
 	docker push $(IMAGE_TAG_LATEST)
+endif
 
 .PHONY: install-nfs-server
 install-nfs-server:

From 1bcaffd3a37c3738ad61884636e2cdf50e14fa60 Mon Sep 17 00:00:00 2001
From: andyzhangx <xiazhang@microsoft.com>
Date: Tue, 21 Sep 2021 04:43:44 +0000
Subject: [PATCH 2/3] chore: set registry type in e2e test

fix bash

registry

fix registry typ

fix typo

set registry
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 02efde86..5cba210c 100644
--- a/Makefile
+++ b/Makefile
@@ -111,7 +111,7 @@ container:
 	docker run --rm --privileged tonistiigi/binfmt --install all
 	for arch in $(ALL_ARCH.linux); do \
 		ARCH=$${arch} $(MAKE) nfs; \
-		ARCH=$${arch} $(MAKE) container-build; \
+		ARCH=$${arch} OUTPUT_TYPE=registry $(MAKE) container-build; \
 	done
 
 .PHONY: push

From e12fe411ae4b0c79420ceb4176b6fb221ac2433d Mon Sep 17 00:00:00 2001
From: andyzhangx <xiazhang@microsoft.com>
Date: Tue, 21 Sep 2021 13:27:23 +0000
Subject: [PATCH 3/3] chore: export vars in Makefile

---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 5cba210c..11aaef76 100644
--- a/Makefile
+++ b/Makefile
@@ -51,6 +51,8 @@ OUTPUT_TYPE ?= docker
 ALL_ARCH.linux = arm64 amd64
 ALL_OS_ARCH = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
 
+.EXPORT_ALL_VARIABLES:
+
 all: nfs
 
 .PHONY: verify
@@ -111,7 +113,7 @@ container:
 	docker run --rm --privileged tonistiigi/binfmt --install all
 	for arch in $(ALL_ARCH.linux); do \
 		ARCH=$${arch} $(MAKE) nfs; \
-		ARCH=$${arch} OUTPUT_TYPE=registry $(MAKE) container-build; \
+		ARCH=$${arch} $(MAKE) container-build; \
 	done
 
 .PHONY: push
@@ -145,7 +147,7 @@ install-helm:
 
 .PHONY: e2e-bootstrap
 e2e-bootstrap: install-helm
-	docker pull $(IMAGE_TAG) || make container push
+	OUTPUT_TYPE=registry $(MAKE) container push
 	helm install csi-driver-nfs ./charts/latest/csi-driver-nfs --namespace kube-system --wait --timeout=15m -v=5 --debug \
 		${E2E_HELM_OPTIONS} \
 		--set controller.logLevel=8 \