diff --git a/Makefile b/Makefile
index 12514dd3a..e1f5d9259 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ IMAGE_ALL_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7
 ensure-buildx:
 	./hack/init-buildx.sh
 
-IMAGE_BUILDX_CMD ?= DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull
+IMAGE_BUILDX_CMD ?= DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --builder=nfd-builder --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull
 
 IMAGE_BUILD_ARGS = --build-arg VERSION=$(VERSION) \
                 --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \
diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh
index 12312c352..a37a25cac 100755
--- a/hack/init-buildx.sh
+++ b/hack/init-buildx.sh
@@ -19,7 +19,7 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
 
 # We can skip setup if the current builder already has multi-arch
 # AND if it isn't the docker driver, which doesn't work
-current_builder="$(docker buildx inspect)"
+current_builder="$(docker buildx inspect nfd-builder || true)"
 # linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6
 if ! grep -Eq "^Driver:\s*docker$"  <<<"${current_builder}" && \
      grep -q "linux/amd64" <<<"${current_builder}" && \