ci(buildkite): stop the coverage build racing suites over its image tag (#13439)

Every agent shares one Docker daemon, and the coverage build used the
same `authelia:dist` tag that every suite's `pre-command` loads, retags
and removes. When another build's suites loaded their image between this
build's `docker build` and the `docker save` in `pre-artifact`, the tag
was gone (`reference does not exist`) or pointed at the other build's
image. Since `pre-artifact` had no `pipefail`, the failed save uploaded
an empty archive, the job passed and every suite in the build failed on
load.

The coverage image is now built under a per-job tag through a new
`--tag` flag on `authelia-scripts docker build`. `pre-artifact` takes
`authelia:dist` only to save the archive, inside the same lock the
suites hold around their load, and `pre-exit` removes the per-job tag.
With every write to `authelia:dist` under that lock, suites can no
longer pick up another build's image between their load and retag
either.

`pre-artifact` now fails the job when the save fails, and skips the save
when the build itself failed or was cancelled.

Signed-off-by: Amir Zarrinkafsh <3339418+nightah@users.noreply.github.com>

.buildkite/hooks/pre-artifact

@@ -14,6 +14,13 @@ if [[ "${BUILDKITE_LABEL}" == ":hammer_and_wrench: Unit Test" ]]; then
   rm -rf goreleaser
 fi
 
-if [[ "${BUILDKITE_LABEL}" == ":docker: Build Image [coverage]" ]]; then
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build Image [coverage]" ]] && [[ "${BUILDKITE_COMMAND_EXIT_STATUS}" == 0 ]]; then
-  docker save "${DOCKER_IMAGE}" | zstdmt -T0 -12 > "authelia-image-coverage.tar.zst"
+  set -o pipefail
+  (
+    flock 9
+    docker tag "authelia:dist-${BUILDKITE_JOB_ID}" "${DOCKER_IMAGE}" && docker save "${DOCKER_IMAGE}" | zstdmt -T0 -12 > "authelia-image-coverage.tar.zst"
+    rc=$?
+    docker rmi "${DOCKER_IMAGE}" > /dev/null 2>&1
+    exit "${rc}"
+  ) 9> /buildkite/.cache/authelia-image-load.lock || exit 1
 fi

.buildkite/hooks/pre-exit

@@ -22,6 +22,7 @@ if [[ ! "${BUILDKITE_COMMAND}" =~ buildkite-agent\ pipeline\ upload ]]; then
     docker network ls -q --filter "label=com.docker.compose.project=${PROJECT}" | xargs -r docker network rm
     docker images --filter "reference=${PROJECT}-*" --format '{{.Repository}}:{{.Tag}}' | xargs -r docker rmi -f > /dev/null 2>&1
     [[ -n "${SUITE_IMAGE}" ]] && docker rmi -f "${SUITE_IMAGE}" > /dev/null 2>&1
+    [[ "${BUILDKITE_LABEL}" == ":docker: Build Image [coverage]" ]] && docker rmi -f "authelia:dist-${BUILDKITE_JOB_ID}" > /dev/null 2>&1
 
     if [[ "${BUILDKITE_LABEL}" == ":grype: Vulnerability Scanning" ]]; then
       docker images --filter "reference=authelia/authelia:*" --filter "reference=authelia/authelia-cve:*" --format '{{.Repository}}:{{.Tag}}' | xargs -r docker rmi > /dev/null 2>&1

.buildkite/pipeline.sh

@@ -87,7 +87,7 @@ steps:
         if: build.env("CI_BYPASS") != "true"
 
       - label: ":docker: Build Image [coverage]"
-        command: "authelia-scripts docker build --container=coverage"
+        command: "authelia-scripts docker build --container=coverage --tag=authelia:dist-\$\${BUILDKITE_JOB_ID}"
         retry:
           manual:
             permit_on_passed: true

cmd/authelia-scripts/cmd/docker.go

@@ -15,7 +15,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var container string
+var container, intermediateTag string
 
 var (
 	containers       = []string{"dev", "coverage"}
@@ -61,6 +61,7 @@ func newDockerBuildCmd() (cmd *cobra.Command) {
 	}
 
 	cmd.PersistentFlags().StringVar(&container, "container", defaultContainer, "target container among: "+strings.Join(containers, ", "))
+	cmd.PersistentFlags().StringVar(&intermediateTag, "tag", IntermediateDockerImageName, "local tag the image is built as")
 
 	return cmd
 }
@@ -91,7 +92,7 @@ func cmdDockerBuildRun(_ *cobra.Command, _ []string) {
 
 	docker := &Docker{}
 
-	err = docker.Tag(IntermediateDockerImageName, DockerImageName)
+	err = docker.Tag(intermediateTag, DockerImageName)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -163,7 +164,7 @@ func dockerBuildOfficialImage(arch string) error {
 		log.Fatal(err)
 	}
 
-	return docker.Build(IntermediateDockerImageName, dockerfile, ".", buildMetaData)
+	return docker.Build(intermediateTag, dockerfile, ".", buildMetaData)
 }
 
 func login(docker *Docker, registry string) {