test(suites): oidc conformance (#13097)

This adds the official OpenID Connect 1.0 Conformance Suites used by the
OpenID Foundation to assert conformance for certification to our test
suites. This allows us to offer the quality guarantee that every single
commit of Authelia conforms to the OpenID Connect 1.0 and related
specifications.

The tests allow the following results to indicate success automatically:
`PASSED`, `REVIEW` (result that manual review must occur), `SKIPPED`
(result that indicates the conformance test is not needed due to
reasonable criteria). In addition only the `FINISHED` status is
accepted.

If the result is `WARNING` which is just an informative result which
does not indicate lack of conformance it can be manually allowed on a
per module basis with an explanation. This allows both detecting new
`WARNING` results and necessitates documenting them for future
improvements.

All of the result and status checks align entirely with the standard
conformance certification process, ensuring that each commit would be
certifiable. The storage for the suites is able to be configured for
persistence, and every plan created carries the commit sha or tag;
allowing for historical evidence of conformance.

.buildkite/steps/e2etests.sh

@@ -15,15 +15,33 @@ declare -A SUITE_AGENTS=(
 
 declare -A SUITE_TIMEOUTS=(
   [Kubernetes]="30"
+  [OIDCConformance]="120"
 )
 
+declare -A SUITE_NO_FAILFAST=(
+  [OIDCConformance]="true"
+)
+
+DEBUG_REGEX='\[(debug test|test debug)\]'
+SUITE_DEBUG="false"
+
+if [[ "${BUILDKITE_MESSAGE:-}" =~ ${DEBUG_REGEX} ]]; then
+  SUITE_DEBUG="true"
+fi
+
 for SUITE_NAME in $(authelia-scripts suites list); do
   AGENT="${SUITE_AGENTS[${SUITE_NAME}]:-all}"
   TIMEOUT="${SUITE_TIMEOUTS[${SUITE_NAME}]:-20}"
+  FAILFAST="--failfast"
+
+  if [[ "${SUITE_NO_FAILFAST[${SUITE_NAME}]:-false}" == "true" ]]; then
+    FAILFAST=""
+  fi
 cat << EOF
   - label: ":selenium: ${SUITE_NAME} Suite"
-    command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --failfast --headless"
+    command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} ${FAILFAST} --headless"
     artifact_paths:
+      - "oidc-conformance-plans/*.zip"
       - "screenshots/**/*.access.log"
       - "screenshots/**/*.console.json"
       - "screenshots/**/*.containers.log"
@@ -43,5 +61,6 @@ cat << EOF
       suite: "${AGENT}"
     env:
       SUITE: "${SUITE_NAME}"
+      SUITE_DEBUG: "${SUITE_DEBUG}"
 EOF
 done

.gitignore

@@ -26,6 +26,7 @@ coverage.txt
 test-results-*.json
 test-results-*.xml
 screenshots/
+oidc-conformance-plans/
 
 # Generated by TypeScript compiler
 dist/

cmd/authelia-gen/cmd_misc.go

@@ -18,6 +18,7 @@ import (
 	"go.yaml.in/yaml/v4"
 
 	"github.com/authelia/authelia/v4/internal/configuration/schema"
+	"github.com/authelia/authelia/v4/internal/oidc/conformance"
 	"github.com/authelia/authelia/v4/internal/utils"
 )
 
@@ -188,19 +189,11 @@ func miscOIDCConformance(version, token, consent, policy, brand string, authelia
 	return nil
 }
 
-func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL, suiteNames ...string) (suites []OpenIDConnectConformanceSuite) {
-	builders := []*OpenIDConnectConformanceSuiteBuilder{
-		{brand, "config", "Config", true, version, consent, policy, nil, autheliaURL},
-		{brand, "basic", "Basic", true, version, consent, policy, suiteURL, autheliaURL},
-		{brand, suiteNameBasicFormPost, "Basic (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
-		{brand, "hybrid", "Hybrid", true, version, consent, policy, suiteURL, autheliaURL},
-		{brand, suiteNameHybridFormPost, "Hybrid (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
-		{brand, "implicit", "Implicit", true, version, consent, policy, suiteURL, autheliaURL},
-		{brand, suiteNameImplicitFormPost, "Implicit (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
-	}
+func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL, suiteNames ...string) (suites []conformance.Suite) {
+	builders := conformance.Builders(version, consent, policy, brand, suiteURL, autheliaURL)
 
 	for _, builder := range builders {
-		if len(suiteNames) != 0 && !utils.IsStringInSlice(builder.name, suiteNames) {
+		if len(suiteNames) != 0 && !utils.IsStringInSlice(builder.Name, suiteNames) {
 			continue
 		}
 
@@ -210,7 +203,7 @@ func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suit
 	return suites
 }
 
-func doOIDCConformanceSuitePostPlan(client *http.Client, base *url.URL, plan string, variant *OpenIDConnectConformanceSuitePlanVariant, body *bytes.Buffer) (err error) {
+func doOIDCConformanceSuitePostPlan(client *http.Client, base *url.URL, plan string, variant *conformance.PlanVariant, body *bytes.Buffer) (err error) {
 	if client == nil {
 		return nil
 	}

cmd/authelia-gen/const.go

@@ -51,18 +51,6 @@ const (
 	fileGitHubIssueTemplateBR = ".github/ISSUE_TEMPLATE/bug-report.yml"
 )
 
-const (
-	suiteNameBasicFormPost           = "basic-form-post"
-	suiteNameHybridFormPost          = "hybrid-form-post"
-	suiteNameImplicitFormPost        = "implicit-form-post"
-	suiteConformanceBasic            = "conformance-basic"
-	suiteConformanceBasicFormPost    = "conformance-basic-form-post"
-	suiteConformanceImplicit         = "conformance-implicit"
-	suiteConformanceImplicitFormPost = "conformance-implicit-form-post"
-	suiteConformanceHybrid           = "conformance-hybrid"
-	suiteConformanceHybridFormPost   = "conformance-hybrid-form-post"
-)
-
 const (
 	pathJSONSchema = "json-schema"
 	extJSON        = ".json"

cmd/authelia-gen/types.go

@@ -8,8 +8,6 @@ import (
 	"fmt"
 	"strings"
 	"time"
-
-	"github.com/authelia/authelia/v4/internal/configuration/schema"
 )
 
 type tmplIssueTemplateData struct {
@@ -230,90 +228,6 @@ type PackageJSON struct {
 	Version string `json:"version"`
 }
 
-// OpenIDConnectConformanceSuite represents a single OpenID Connect conformance suite.
-type OpenIDConnectConformanceSuite struct {
-	Name    string
-	Plan    OpenIDConnectConformanceSuitePlan
-	Clients []schema.IdentityProvidersOpenIDConnectClient
-}
-
-// OpenIDConnectConformanceSuitePlan represents the test plan for an OpenIDConnectConformanceSuite.
-type OpenIDConnectConformanceSuitePlan struct {
-	Name        string `json:"-"`
-	Alias       string `json:"alias"`
-	Description string `json:"description"`
-	Publish     string `json:"publish"`
-
-	Variant          *OpenIDConnectConformanceSuitePlanVariant   `json:"-"`
-	Server           OpenIDConnectConformanceSuitePlanServer     `json:"server"`
-	Client           *OpenIDConnectConformanceSuitePlanClient    `json:"client,omitempty"`
-	ClientAlternate  *OpenIDConnectConformanceSuitePlanClient    `json:"client2,omitempty"`
-	ClientSecretPost *OpenIDConnectConformanceSuitePlanClient    `json:"client_secret_post,omitempty"`
-	MutualTLS        *OpenIDConnectConformanceSuitePlanMutualTLS `json:"mtls,omitempty"`
-	Resource         *OpenIDConnectConformanceSuitePlanResource  `json:"resource,omitempty"`
-
-	EKYCVerifiedClaimsRequest string `json:"ekyc_verified_claims_request,omitempty"`
-	EKYCUserinfo              string `json:"ekyc_userinfo,omitempty"`
-}
-
-// OpenIDConnectConformanceSuitePlanVariant represents the variant of an OpenIDConnectConformanceSuitePlan.
-type OpenIDConnectConformanceSuitePlanVariant struct {
-	ServerMetadata           string `json:"server_metadata,omitempty"`
-	ClientRegistration       string `json:"client_registration,omitempty"`
-	ClientAuthType           string `json:"client_auth_type,omitempty"`
-	ResponseType             string `json:"response_type,omitempty"`
-	ResponseMode             string `json:"response_mode,omitempty"`
-	CIBAMode                 string `json:"ciba_mode,omitempty"`
-	FAPIProfile              string `json:"fapi_profile,omitempty"`
-	FAPIResponseMode         string `json:"fapi_response_mode,omitempty"`
-	FAPIAuthRequestMethod    string `json:"fapi_auth_request_method,omitempty"`
-	SenderConstrain          string `json:"sender_constrain,omitempty"`
-	AuthorizationRequestType string `json:"authorization_request_type,omitempty"`
-	OpenID                   string `json:"openid,omitempty"`
-}
-
-// OpenIDConnectConformanceSuitePlanServer represents the server details of an OpenIDConnectConformanceSuitePlan.
-type OpenIDConnectConformanceSuitePlanServer struct {
-	DiscoveryURL          string `json:"discoveryUrl,omitempty"`
-	Issuer                string `json:"issuer,omitempty"`
-	JWKSURI               string `json:"jwks_uri,omitempty"`
-	AuthorizationEndpoint string `json:"authorization_endpoint,omitempty"`
-	TokenEndpoint         string `json:"token_endpoint,omitempty"`
-	UserinfoEndpoint      string `json:"userinfo_endpoint,omitempty"`
-	ACRValues             string `json:"acr_values,omitempty"`
-	LoginHint             string `json:"login_hint,omitempty"`
-}
-
-// OpenIDConnectConformanceSuitePlanClient represents a client registered with an OpenIDConnectConformanceSuitePlan.
-type OpenIDConnectConformanceSuitePlanClient struct {
-	ID                   string `json:"client_id,omitempty"`
-	Secret               string `json:"client_secret,omitempty"`
-	Name                 string `json:"client_name,omitempty"`
-	Scope                string `json:"scope,omitempty"`
-	SecretJWTAlgorithm   string `json:"client_secret_jwt_alg,omitempty"`
-	DPOPSigningAlgorithm string `json:"dpop_signing_alg,omitempty"`
-	InitialAccessToken   string `json:"initial_access_token,omitempty"`
-	HintType             string `json:"hint_type,omitempty"`
-	HintValue            string `json:"hint_value,omitempty"`
-	JWKS                 string `json:"jwks,omitempty"`
-}
-
-// OpenIDConnectConformanceSuitePlanMutualTLS represents the Mutual TLS details of an OpenIDConnectConformanceSuitePlan.
-type OpenIDConnectConformanceSuitePlanMutualTLS struct {
-	Certificate          string `json:"certificate,omitempty"`
-	Key                  string `json:"key,omitempty"`
-	CertificateAuthority string `json:"ca,omitempty"`
-}
-
-// OpenIDConnectConformanceSuitePlanResource represents the protected resource details of an OpenIDConnectConformanceSuitePlan.
-type OpenIDConnectConformanceSuitePlanResource struct {
-	ResourceURL                 string `json:"resourceUrl"`
-	ResourceURLAccountRequests  string `json:"resourceUrlAccountRequests,omitempty"`
-	ResourceURLAccountsResource string `json:"resourceUrlAccountsResource,omitempty"`
-	InstitutionID               string `json:"institution_id,omitempty"`
-	RichAuthorizationRequest    string `json:"richAuthorizationRequest,omitempty"`
-}
-
 // AllContributors represents the .all-contributorsrc configuration file.
 type AllContributors struct {
 	Contributors []Contributor `json:"contributors"`

cmd/authelia-scripts/cmd/helpers_results.go

@@ -8,16 +8,28 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"strings"
 )
 
 type testEvent struct {
 	Action string `json:"Action"`
+	Test   string `json:"Test"`
 	Output string `json:"Output"`
 }
 
 type testOutputWriter struct {
-	out io.Writer
-	buf bytes.Buffer
+	out       io.Writer
+	buf       bytes.Buffer
+	grouped   bool
+	pending   []testFraming
+	buildkite bool
+	deferring bool
+	deferred  bytes.Buffer
+}
+
+type testFraming struct {
+	test string
+	line string
 }
 
 func (w *testOutputWriter) Write(p []byte) (int, error) {
@@ -34,17 +46,136 @@ func (w *testOutputWriter) Write(p []byte) (int, error) {
 		var event testEvent
 
 		if json.Unmarshal(line, &event) != nil {
-			if _, err = w.out.Write(line); err != nil {
+			if err = w.write(string(line)); err != nil {
 				return len(p), err
 			}
 
 			continue
 		}
 
-		if event.Action == "output" {
-			if _, err = io.WriteString(w.out, event.Output); err != nil {
-				return len(p), err
-			}
+		if err = w.handle(event); err != nil {
+			return len(p), err
 		}
 	}
 }
+
+func (w *testOutputWriter) handle(event testEvent) (err error) {
+	if event.Action != "output" {
+		return nil
+	}
+
+	if !w.grouped {
+		return w.write(event.Output)
+	}
+
+	switch output := event.Output; {
+	case isTestFramingLine(output):
+		w.pending = append(w.pending, testFraming{test: event.Test, line: output})
+
+		return nil
+	case isBuildkiteMarkerLine(output) && !isTestSummaryLine(output):
+		if err = w.release(func(framing testFraming) bool { return !isTestOrParent(framing.test, event.Test) }); err != nil {
+			return err
+		}
+
+		if err = w.write(output); err != nil {
+			return err
+		}
+
+		return w.release(func(testFraming) bool { return true })
+	default:
+		if err = w.release(func(testFraming) bool { return true }); err != nil {
+			return err
+		}
+
+		return w.write(output)
+	}
+}
+
+func (w *testOutputWriter) write(output string) (err error) {
+	if w.buildkite && !w.deferring && isTopLevelTestSummaryLine(output) {
+		w.deferring = true
+	}
+
+	if w.deferring {
+		w.deferred.WriteString(output)
+
+		return nil
+	}
+
+	_, err = io.WriteString(w.out, output)
+
+	return err
+}
+
+func (w *testOutputWriter) Flush() (err error) {
+	if err = w.release(func(testFraming) bool { return true }); err != nil {
+		return err
+	}
+
+	_, err = w.out.Write(w.deferred.Bytes())
+
+	w.deferred.Reset()
+
+	return err
+}
+
+func (w *testOutputWriter) release(match func(framing testFraming) bool) (err error) {
+	kept := w.pending[:0]
+
+	for _, framing := range w.pending {
+		if !match(framing) {
+			kept = append(kept, framing)
+
+			continue
+		}
+
+		if err = w.write(framing.line); err != nil {
+			return err
+		}
+	}
+
+	w.pending = kept
+
+	return nil
+}
+
+func isTestOrParent(test, child string) bool {
+	return test == child || strings.HasPrefix(child, test+"/")
+}
+
+func isTestFramingLine(output string) bool {
+	for _, prefix := range []string{"=== RUN", "=== PAUSE", "=== CONT", "=== NAME"} {
+		if strings.HasPrefix(output, prefix) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isTestSummaryLine(output string) bool {
+	trimmed := strings.TrimLeft(output, " ")
+
+	for _, prefix := range []string{"--- PASS: ", "--- FAIL: ", "--- SKIP: "} {
+		if strings.HasPrefix(trimmed, prefix) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isTopLevelTestSummaryLine(output string) bool {
+	return isTestSummaryLine(output) && !strings.HasPrefix(output, " ")
+}
+
+func isBuildkiteMarkerLine(output string) bool {
+	for _, prefix := range []string{"--- ", "+++ ", "~~~ ", "^^^ +++"} {
+		if strings.HasPrefix(output, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

cmd/authelia-scripts/cmd/helpers_results_test.go

@@ -6,9 +6,12 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestTestOutputWriterShouldReconstructConsoleOutput(t *testing.T) {
@@ -63,3 +66,188 @@ func TestTestOutputWriterShouldReconstructConsoleOutput(t *testing.T) {
 		})
 	}
 }
+
+func TestTestOutputWriterShouldPlaceFramingInsideBuildkiteGroups(t *testing.T) {
+	event := func(test, output string) string {
+		return fmt.Sprintf(`{"Action":"output","Test":%q,"Output":%q}`, test, output) + "\n"
+	}
+
+	testCases := []struct {
+		name     string
+		events   []string
+		expected string
+	}{
+		{
+			"ShouldPlaceTheFramingOfTheTestWhichStartedAGroupInsideIt",
+			[]string{
+				event("TestSuite", "=== RUN   TestSuite\n"),
+				event("TestSuite/TestBasic", "=== RUN   TestSuite/TestBasic\n"),
+				event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"),
+			},
+			"--- OIDC Conformance Plan: basic\n=== RUN   TestSuite\n=== RUN   TestSuite/TestBasic\n",
+		},
+		{
+			"ShouldLeaveTheFramingOfTheLastGroupBeforeTheNextOne",
+			[]string{
+				event("TestSuite/TestBasic/Codereuse", "=== RUN   TestSuite/TestBasic/Codereuse\n"),
+				event("TestSuite/TestBasicFormPost", "=== RUN   TestSuite/TestBasicFormPost\n"),
+				event("TestSuite/TestBasicFormPost", "--- OIDC Conformance Plan: basic-form-post\n"),
+			},
+			"=== RUN   TestSuite/TestBasic/Codereuse\n--- OIDC Conformance Plan: basic-form-post\n=== RUN   TestSuite/TestBasicFormPost\n",
+		},
+		{
+			"ShouldPrintTheFramingOfEveryTestBeforeTheNextOutput",
+			[]string{
+				event("TestSuite/TestBasic/Server", "=== RUN   TestSuite/TestBasic/Server\n"),
+				event("TestSuite/TestBasic/MaxAge1", "=== RUN   TestSuite/TestBasic/MaxAge1\n"),
+				event("TestSuite/TestBasic/MaxAge1", "    suite_test.go:1: Module finished with the result 'FAILED'\n"),
+				event("TestSuite/TestBasic/MaxAge1", "^^^ +++\n"),
+			},
+			"=== RUN   TestSuite/TestBasic/Server\n=== RUN   TestSuite/TestBasic/MaxAge1\n    suite_test.go:1: Module finished with the result 'FAILED'\n^^^ +++\n",
+		},
+		{
+			"ShouldPrintHeldFramingBeforeTheSummary",
+			[]string{
+				event("TestSuite/TestBasic/Server", "=== RUN   TestSuite/TestBasic/Server\n"),
+				event("TestSuite/TestBasic/Server", "--- PASS: TestSuite/TestBasic/Server (0.00s)\n"),
+			},
+			"=== RUN   TestSuite/TestBasic/Server\n--- PASS: TestSuite/TestBasic/Server (0.00s)\n",
+		},
+		{
+			"ShouldPrintOutputWhichBelongsToNoTest",
+			[]string{
+				event("TestSuite", "=== RUN   TestSuite\n"),
+				event("", "PASS\n"),
+			},
+			"=== RUN   TestSuite\nPASS\n",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			out := &bytes.Buffer{}
+			writer := &testOutputWriter{out: out, grouped: true}
+
+			for _, write := range tc.events {
+				_, err := writer.Write([]byte(write))
+
+				assert.NoError(t, err)
+			}
+
+			assert.Equal(t, tc.expected, out.String())
+		})
+	}
+
+	t.Run("ShouldPrintEverythingInPlaceWhenNotGrouped", func(t *testing.T) {
+		out := &bytes.Buffer{}
+		writer := &testOutputWriter{out: out}
+
+		for _, write := range []string{
+			event("TestSuite/TestBasic", "=== RUN   TestSuite/TestBasic\n"),
+			event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"),
+		} {
+			_, err := writer.Write([]byte(write))
+
+			assert.NoError(t, err)
+		}
+
+		assert.Equal(t, "=== RUN   TestSuite/TestBasic\n--- OIDC Conformance Plan: basic\n", out.String())
+	})
+}
+
+func TestTestOutputWriterShouldDeferTheSummaryInBuildkite(t *testing.T) {
+	events := `{"Action":"output","Test":"TestSuite/TestBasic","Output":"    suite_test.go:1: Exported the plan log\n"}` + "\n" +
+		`{"Action":"output","Test":"TestSuite","Output":"--- PASS: TestSuite (227.08s)\n"}` + "\n" +
+		`{"Action":"output","Test":"TestSuite/TestBasic","Output":"    --- PASS: TestSuite/TestBasic (116.14s)\n"}` + "\n" +
+		`{"Action":"output","Output":"PASS\n"}` + "\n"
+
+	summary := "--- PASS: TestSuite (227.08s)\n    --- PASS: TestSuite/TestBasic (116.14s)\nPASS\n"
+
+	testCases := []struct {
+		name      string
+		buildkite bool
+		grouped   bool
+		before    string
+	}{
+		{"ShouldDeferTheSummaryInBuildkite", true, false, "    suite_test.go:1: Exported the plan log\n"},
+		{"ShouldDeferTheSummaryInBuildkiteWhenGrouped", true, true, "    suite_test.go:1: Exported the plan log\n"},
+		{"ShouldPrintTheSummaryInPlaceElsewhere", false, false, "    suite_test.go:1: Exported the plan log\n" + summary},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			out := &bytes.Buffer{}
+			writer := &testOutputWriter{out: out, buildkite: tc.buildkite, grouped: tc.grouped}
+
+			_, err := writer.Write([]byte(events))
+			assert.NoError(t, err)
+
+			assert.Equal(t, tc.before, out.String())
+
+			out.WriteString("teardown\n")
+
+			assert.NoError(t, writer.Flush())
+
+			if tc.buildkite {
+				assert.Equal(t, tc.before+"teardown\n"+summary, out.String())
+			} else {
+				assert.Equal(t, tc.before+"teardown\n", out.String())
+			}
+		})
+	}
+}
+
+func TestTestOutputWriterShouldReturnWriteErrors(t *testing.T) {
+	event := func(test, output string) string {
+		return fmt.Sprintf(`{"Action":"output","Test":%q,"Output":%q}`, test, output) + "\n"
+	}
+
+	testCases := []struct {
+		name   string
+		events []string
+	}{
+		{"ShouldReturnTheErrorOfAPlainLine", []string{"not an event\n"}},
+		{"ShouldReturnTheErrorOfHeldFramingBeforeAGroup", []string{
+			event("TestSuite/TestBasic/Codereuse", "=== RUN   TestSuite/TestBasic/Codereuse\n"),
+			event("TestSuite/TestBasicFormPost", "--- OIDC Conformance Plan: basic-form-post\n"),
+		}},
+		{"ShouldReturnTheErrorOfAGroupHeader", []string{
+			event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"),
+		}},
+		{"ShouldReturnTheErrorOfHeldFramingBeforeOutput", []string{
+			event("TestSuite/TestBasic/Server", "=== RUN   TestSuite/TestBasic/Server\n"),
+			event("TestSuite/TestBasic/Server", "    suite_test.go:1: output\n"),
+		}},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			writer := &testOutputWriter{out: testFailingWriter{}, grouped: true}
+
+			var err error
+
+			for _, write := range tc.events {
+				if _, err = writer.Write([]byte(write)); err != nil {
+					break
+				}
+			}
+
+			assert.EqualError(t, err, "write failed")
+		})
+	}
+
+	t.Run("ShouldReturnTheErrorOfHeldFramingWhenFlushing", func(t *testing.T) {
+		writer := &testOutputWriter{out: testFailingWriter{}, grouped: true}
+
+		_, err := writer.Write([]byte(event("TestSuite/TestBasic/Server", "=== RUN   TestSuite/TestBasic/Server\n")))
+		require.NoError(t, err)
+
+		assert.EqualError(t, writer.Flush(), "write failed")
+	})
+}
+
+type testFailingWriter struct{}
+
+func (testFailingWriter) Write(_ []byte) (int, error) {
+	return 0, errors.New("write failed")
+}

cmd/authelia-scripts/cmd/suites.go

@@ -351,8 +351,16 @@ func runSuiteTests(suiteName string, withEnv bool) error {
 
 	defer results.Close()
 
+	output := &testOutputWriter{out: os.Stdout, buildkite: os.Getenv("BUILDKITE") == "true", grouped: os.Getenv("BUILDKITE") == "true" && os.Getenv("SUITE_DEBUG") == "true"}
+
+	defer func() {
+		if err := output.Flush(); err != nil {
+			log.Errorf("Error writing the test summary: %v", err)
+		}
+	}()
+
 	cmd := utils.CommandWithStdout("bash", "-c", testCmdLine)
-	cmd.Stdout = io.MultiWriter(&testOutputWriter{out: os.Stdout}, results)
+	cmd.Stdout = io.MultiWriter(output, results)
 	cmd.Stderr = os.Stderr
 	cmd.Env = os.Environ()
 

docs/content/contributing/development/integration-suites.md

@@ -95,16 +95,17 @@ The suites run several concurrent copies of themselves when the Docker daemon is
 optional and falls back to the value used for a single local run, so leaving them all unset gives the behavior
 described above.
 
-| Variable               | Default         | Purpose                                                                                                                                                                                                                                                                    |
-| :--------------------- | :-------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `SUITE_SLOT`           | _unset_         | Slot number owned by this agent or working tree. When set, `bootstrap.sh` derives `COMPOSE_PROJECT_NAME`, `SUITE_SUBNET`, `LDAP_ADMIN_PORT`, `ENVOY_ADMIN_PORT` and, outside CI, `SUITE_TMP` and `SUITE_TMP_PATH` from it. A slotted shell also leaves `/etc/hosts` alone. |
-| `SUITE_SLOT_AUTO`      | _unset_         | Set to `false` to stop `bootstrap.sh` allocating a slot for this working tree. Has no effect in CI, where the slot is supplied by the agent.                                                                                                                               |
-| `COMPOSE_PROJECT_NAME` | `authelia`      | Compose project name. Also scopes the Traefik Docker provider so it only discovers its own containers.                                                                                                                                                                     |
-| `SUITE_SUBNET`         | `192.168.240`   | First three octets of the suite network.                                                                                                                                                                                                                                   |
-| `SUITE_TMP`            | `/tmp`          | Host directory bound into the suite containers, which always see it at `/tmp`. Give each agent or working tree its own directory, because everything at its top level apart from the agent's own working files is removed when a job finishes.                             |
-| `SUITE_TMP_PATH`       | `/tmp`          | Path the test process itself reads and writes that same content through. In CI this stays `/tmp` because `SUITE_TMP` is bound there inside the agent; locally it is set to `SUITE_TMP`, since the test process runs on the host.                                           |
-| `SUITE_IMAGE`          | `authelia:dist` | Image the backend runs.                                                                                                                                                                                                                                                    |
-| `AGENT_CONTAINER`      | _unset_         | Name of the container the tests run in. When set, that container is attached to the suite network on setup and detached on teardown, so Chrome can reach the portal.                                                                                                       |
+| Variable                              | Default         | Purpose                                                                                                                                                                                                                                                                    |
+| :------------------------------------ | :-------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `SUITE_SLOT`                          | _unset_         | Slot number owned by this agent or working tree. When set, `bootstrap.sh` derives `COMPOSE_PROJECT_NAME`, `SUITE_SUBNET`, `LDAP_ADMIN_PORT`, `ENVOY_ADMIN_PORT` and, outside CI, `SUITE_TMP` and `SUITE_TMP_PATH` from it. A slotted shell also leaves `/etc/hosts` alone. |
+| `SUITE_SLOT_AUTO`                     | _unset_         | Set to `false` to stop `bootstrap.sh` allocating a slot for this working tree. Has no effect in CI, where the slot is supplied by the agent.                                                                                                                               |
+| `COMPOSE_PROJECT_NAME`                | `authelia`      | Compose project name. Also scopes the Traefik Docker provider so it only discovers its own containers.                                                                                                                                                                     |
+| `SUITE_SUBNET`                        | `192.168.240`   | First three octets of the suite network.                                                                                                                                                                                                                                   |
+| `SUITE_TMP`                           | `/tmp`          | Host directory bound into the suite containers, which always see it at `/tmp`. Give each agent or working tree its own directory, because everything at its top level apart from the agent's own working files is removed when a job finishes.                             |
+| `SUITE_TMP_PATH`                      | `/tmp`          | Path the test process itself reads and writes that same content through. In CI this stays `/tmp` because `SUITE_TMP` is bound there inside the agent; locally it is set to `SUITE_TMP`, since the test process runs on the host.                                           |
+| `SUITE_IMAGE`                         | `authelia:dist` | Image the backend runs.                                                                                                                                                                                                                                                    |
+| `AGENT_CONTAINER`                     | _unset_         | Name of the container the tests run in. When set, that container is attached to the suite network on setup and detached on teardown, so Chrome can reach the portal.                                                                                                       |
+| `SUITE_OIDC_CONFORMANCE_MONGODB_HOST` | _unset_         | MongoDB host the OIDCConformance suite's conformance server uses. When set, the suite does not run its own MongoDB; the host has to be reachable from the suite network.                                                                                                   |
 
 ## Running Suites Concurrently
 
@@ -198,4 +199,20 @@ Creating a suite is as easy. Let's take the example of the **Standalone** suite:
 A suite can also be much more complex like setting up a complete Kubernetes ecosystem. You can check the Kubernetes
 suite as example.
 
+## Commit Message Toggles
+
+Our continuous integration reads the following options from the commit message. It reads the message of the commit
+being built, which for a pull request is its most recent commit, so an option has to be in that commit's message. An
+option can appear anywhere in the message; placing it in the body or the footer keeps the header within its format.
+
+- `[skip test]` or `[test skip]`: skips linting, the external tests, the coverage image build, and the integration
+  tests. Every other step, including the unit tests, still runs.
+- `[debug test]` or `[test debug]`: runs the integration suites with their opt-in debug output enabled. The OpenID
+  Connect 1.0 conformance suite groups its build log by plan, with each group holding its tests' `=== RUN` lines and
+  opening by itself when one of them fails, and logs a trace of what it did for each module, stamped with the time in
+  UTC, under that module's subtest. Without this option the build log is left as `go test` writes it. The trace is
+  available locally by setting the `SUITE_DEBUG` environment variable to `true` when running a suite.
+
+A commit whose header begins with `docs` also skips vulnerability scanning.
+
 [delve]: https://github.com/go-delve/delve

docs/content/integration/openid-connect/introduction.md

@@ -46,6 +46,12 @@ actively perform the tests on each version of Authelia to maintain the latest co
 You can view our published conformance tests at [Certified OpenID Providers & Profiles] and
 [Certified OpenID Providers for Logout Profiles].
 
+In addition to the published conformance tests, all commits that change any part of the code run integration tests
+against every module of the conformance plans that Authelia is certified for or is planning to get certified for. These
+plans and modules are all of the tests required for an implementation to become certified. This is both done as a
+quality assurance measure to ensure confidence in Authelia's implementation and also to thoroughly test the
+implementation against normal use cases.
+
 ### OpenID Connect Protocol Suite
 
 <figure>

internal/handlers/handler_oauth2_authorization_consent_core_test.go

@@ -1101,6 +1101,68 @@ func TestHandleOAuth2AuthorizationConsentModeImplicitWithID(t *testing.T) {
 			},
 			expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) {},
 		},
+		{
+			name:        "ShouldUseTheResponseAlreadyGivenOnTheConsentForm",
+			issuer:      mustParseURI(t, "https://auth.example.com"),
+			client:      clientTest,
+			userSession: session.UserSession{Username: testValue, FirstFactorAuthnTimestamp: 1000000, SecondFactorAuthnTimestamp: 1000000},
+			subject:     sub,
+			requester: &oauthelia2.AuthorizeRequest{
+				Request: oauthelia2.Request{
+					Client: clientTest,
+					Form: url.Values{
+						oidc.FormParameterMaximumAge: []string{"10"},
+					},
+					RequestedAt: time.Unix(1000000, 0),
+				},
+			},
+			expected: &model.OAuth2ConsentSession{
+				ID:          40,
+				ClientID:    "test",
+				Subject:     uuid.NullUUID{UUID: sub, Valid: true},
+				Authorized:  true,
+				Form:        "max_age=10",
+				RequestedAt: time.Unix(1000000, 0),
+				RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true},
+			},
+			handled: false,
+			setup: func(t *testing.T, mock *mocks.MockAutheliaCtx) {
+				gomock.InOrder(
+					mock.StorageMock.EXPECT().
+						LoadOAuth2ConsentSessionByChallengeID(gomock.Eq(mock.Ctx), gomock.Eq(challenge)).
+						Return(&model.OAuth2ConsentSession{ID: 40, ChallengeID: challenge, ClientID: "test", Subject: uuid.NullUUID{UUID: sub, Valid: true}, Authorized: true, Form: "max_age=10", RequestedAt: time.Unix(1000000, 0), RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true}, ExpiresAt: mock.Ctx.Providers.Clock.Now().Add(time.Second * 10)}, nil),
+				)
+			},
+			expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) {},
+		},
+		{
+			name:        "ShouldDenyWhenTheConsentFormWasRejected",
+			issuer:      mustParseURI(t, "https://auth.example.com"),
+			client:      clientTest,
+			userSession: session.UserSession{Username: testValue, FirstFactorAuthnTimestamp: 1000000, SecondFactorAuthnTimestamp: 1000000},
+			subject:     sub,
+			requester: &oauthelia2.AuthorizeRequest{
+				Request: oauthelia2.Request{
+					Client: clientTest,
+					Form: url.Values{
+						oidc.FormParameterMaximumAge: []string{"10"},
+					},
+					RequestedAt: time.Unix(1000000, 0),
+				},
+			},
+			expected: nil,
+			handled:  true,
+			setup: func(t *testing.T, mock *mocks.MockAutheliaCtx) {
+				gomock.InOrder(
+					mock.StorageMock.EXPECT().
+						LoadOAuth2ConsentSessionByChallengeID(gomock.Eq(mock.Ctx), gomock.Eq(challenge)).
+						Return(&model.OAuth2ConsentSession{ID: 40, ChallengeID: challenge, ClientID: "test", Subject: uuid.NullUUID{UUID: sub, Valid: true}, Authorized: false, Form: "max_age=10", RequestedAt: time.Unix(1000000, 0), RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true}, ExpiresAt: mock.Ctx.Providers.Clock.Now().Add(time.Second * 10)}, nil),
+				)
+			},
+			expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) {
+				mock.AssertLastLogMessageRegexp(t, regexp.MustCompile(`^Authorization Request with id '[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}' on client with id 'test' using consent mode 'implicit' could not be processed: error occurred performing consent for consent session with id '11303e1f-f8af-436a-9a72-c7361bfc9f37': the user explicitly rejected this consent session$`), nil)
+			},
+		},
 	}
 
 	for _, tc := range testCases {

internal/handlers/handler_oauth2_authorization_consent_implicit.go

@@ -97,6 +97,18 @@ func handleOAuth2AuthorizationConsentModeImplicitWithID(ctx *middlewares.Autheli
 		return nil, true
 	}
 
+	if consent.Responded() {
+		if !consent.IsAuthorized() {
+			ctx.GetLogger().Errorf(logFmtErrConsentCantGrantRejected, requester.GetID(), client.GetID(), client.GetConsentPolicy(), consent.ChallengeID)
+
+			ctx.Providers.OpenIDConnect.WriteDynamicAuthorizeError(ctx, rw, requester, oauthelia2.ErrAccessDenied)
+
+			return nil, true
+		}
+
+		return consent, false
+	}
+
 	var requests *oidc.ClaimsRequests
 
 	if requests, err = oidc.NewClaimRequests(requester.GetRequestForm()); err == nil && requests != nil {

internal/oidc/conformance/builder.go

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-package main
+package conformance
 
 import (
 	"fmt"
@@ -17,80 +17,87 @@ import (
 	"github.com/authelia/authelia/v4/internal/random"
 )
 
-// OpenIDConnectConformanceSuiteBuilder builds an OpenIDConnectConformanceSuite.
-type OpenIDConnectConformanceSuiteBuilder struct {
-	brand         string
-	name          string
-	friendly      string
-	certification bool
-	version       string
-	consent       string
-	policy        string
-	suiteURL      *url.URL
-	autheliaURL   *url.URL
+// SuiteBuilder builds a Suite.
+type SuiteBuilder struct {
+	Brand         string
+	Name          string
+	Friendly      string
+	Certification bool
+	Version       string
+	Consent       string
+	Policy        string
+	SuiteURL      *url.URL
+	AutheliaURL   *url.URL
 }
 
-// Build returns the OpenIDConnectConformanceSuite for this builder.
-func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceSuite {
-	var (
-		apiname, namePrefix, clientIDPrefix, descriptionSuffix string
-		variant                                                *OpenIDConnectConformanceSuitePlanVariant
-	)
-
-	if b.certification {
-		namePrefix = "conformance-"
-		clientIDPrefix = "conformance-certification"
-		descriptionSuffix = "Certification Profile"
-	} else {
-		clientIDPrefix = "conformance-test"
-		descriptionSuffix = "Test Profile"
+// Description returns the plan description naming release as the version of Authelia under test.
+func (b *SuiteBuilder) Description(release string) string {
+	if b.Certification {
+		return fmt.Sprintf("Authelia %s %s Certification Profile", release, b.Friendly)
 	}
 
-	aliasSuffix := fmt.Sprintf("%s-%s", strings.ReplaceAll(strings.ToLower(b.name), ".", "-"), b.brand+strings.ReplaceAll(strings.ToLower(b.version), ".", ""))
+	return fmt.Sprintf("Authelia %s %s Test Profile", release, b.Friendly)
+}
 
-	name := fmt.Sprintf("%s%s", namePrefix, b.name)
-	description := fmt.Sprintf("Authelia %s %s %s", b.version, b.friendly, descriptionSuffix)
+// Build returns the Suite for this builder.
+func (b *SuiteBuilder) Build() Suite {
+	var (
+		apiname, namePrefix, clientIDPrefix string
+		variant                             *PlanVariant
+	)
 
-	switch name {
-	case suiteConformanceBasic, suiteConformanceBasicFormPost, suiteConformanceHybrid, suiteConformanceHybridFormPost, suiteConformanceImplicit, suiteConformanceImplicitFormPost:
-		variant = &OpenIDConnectConformanceSuitePlanVariant{
+	if b.Certification {
+		namePrefix = "conformance-"
+		clientIDPrefix = "conformance-certification"
+	} else {
+		clientIDPrefix = "conformance-test"
+	}
+
+	aliasSuffix := fmt.Sprintf("%s-%s", strings.ReplaceAll(strings.ToLower(b.Name), ".", "-"), b.Brand+strings.ReplaceAll(strings.ToLower(b.Version), ".", ""))
+
+	name := fmt.Sprintf("%s%s", namePrefix, b.Name)
+	description := b.Description(b.Version)
+
+	switch b.Name {
+	case NameBasic, NameBasicFormPost, NameHybrid, NameHybridFormPost, NameImplicit, NameImplicitFormPost:
+		variant = &PlanVariant{
 			ServerMetadata:     "discovery",
 			ClientRegistration: "static_client",
 		}
 	}
 
-	switch name {
-	case "conformance-config":
+	switch b.Name {
+	case NameConfig:
 		apiname = "oidcc-config-certification-test-plan"
-	case suiteConformanceBasic:
+	case NameBasic:
 		apiname = "oidcc-basic-certification-test-plan"
-	case suiteConformanceBasicFormPost:
+	case NameBasicFormPost:
 		apiname = "oidcc-formpost-basic-certification-test-plan"
-	case suiteConformanceHybrid:
+	case NameHybrid:
 		apiname = "oidcc-hybrid-certification-test-plan"
-	case suiteConformanceHybridFormPost:
+	case NameHybridFormPost:
 		apiname = "oidcc-formpost-hybrid-certification-test-plan"
-	case suiteConformanceImplicit:
+	case NameImplicit:
 		apiname = "oidcc-implicit-certification-test-plan"
-	case suiteConformanceImplicitFormPost:
+	case NameImplicitFormPost:
 		apiname = "oidcc-formpost-implicit-certification-test-plan"
 	}
 
-	suite := OpenIDConnectConformanceSuite{
+	suite := Suite{
 		Name: name,
-		Plan: OpenIDConnectConformanceSuitePlan{
+		Plan: Plan{
 			Name:        apiname,
 			Variant:     variant,
 			Alias:       fmt.Sprintf("%s%s", namePrefix, aliasSuffix),
 			Publish:     "summary",
 			Description: description,
-			Server: OpenIDConnectConformanceSuitePlanServer{
-				DiscoveryURL: b.autheliaURL.JoinPath(".well-known/openid-configuration").String(),
+			Server: PlanServer{
+				DiscoveryURL: b.AutheliaURL.JoinPath(".well-known/openid-configuration").String(),
 			},
 		},
 	}
 
-	if b.suiteURL == nil {
+	if b.SuiteURL == nil {
 		return suite
 	}
 
@@ -100,17 +107,17 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 	secretAlternate := r.StringCustom(80, random.CharSetAlphaNumeric)
 	secretPost := r.StringCustom(80, random.CharSetAlphaNumeric)
 
-	suite.Plan.Client = &OpenIDConnectConformanceSuitePlanClient{
+	suite.Plan.Client = &PlanClient{
 		ID:     fmt.Sprintf("%s-%s", clientIDPrefix, aliasSuffix),
 		Secret: secret,
 	}
 
-	suite.Plan.ClientAlternate = &OpenIDConnectConformanceSuitePlanClient{
+	suite.Plan.ClientAlternate = &PlanClient{
 		ID:     fmt.Sprintf("%s-%s-alt", clientIDPrefix, aliasSuffix),
 		Secret: secretAlternate,
 	}
 
-	suite.Plan.ClientSecretPost = &OpenIDConnectConformanceSuitePlanClient{
+	suite.Plan.ClientSecretPost = &PlanClient{
 		ID:     fmt.Sprintf("%s-%s-post", clientIDPrefix, aliasSuffix),
 		Secret: secretPost,
 	}
@@ -121,11 +128,11 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 		responseModes []string
 	)
 
-	switch b.name {
-	case "implicit", suiteNameImplicitFormPost:
+	switch b.Name {
+	case NameImplicit, NameImplicitFormPost:
 		grantTypes = []string{oidc.GrantTypeAuthorizationCode, oidc.GrantTypeImplicit, oidc.GrantTypeRefreshToken}
 		responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow, oidc.ResponseTypeImplicitFlowIDToken, oidc.ResponseTypeImplicitFlowToken, oidc.ResponseTypeImplicitFlowBoth}
-	case "hybrid", suiteNameHybridFormPost:
+	case NameHybrid, NameHybridFormPost:
 		grantTypes = []string{oidc.GrantTypeAuthorizationCode, oidc.GrantTypeImplicit, oidc.GrantTypeRefreshToken}
 		responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow, oidc.ResponseTypeHybridFlowIDToken, oidc.ResponseTypeHybridFlowToken, oidc.ResponseTypeHybridFlowBoth}
 	default:
@@ -133,8 +140,8 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 		responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow}
 	}
 
-	switch b.name {
-	case suiteNameBasicFormPost, suiteNameHybridFormPost, suiteNameImplicitFormPost:
+	switch b.Name {
+	case NameBasicFormPost, NameHybridFormPost, NameImplicitFormPost:
 		responseModes = []string{oidc.ResponseModeFormPost, oidc.ResponseModeFormPostJWT}
 	default:
 		responseModes = []string{oidc.ResponseModeQuery, oidc.ResponseModeQueryJWT}
@@ -145,9 +152,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 			ID:                      suite.Plan.Client.ID,
 			Name:                    description,
 			Secret:                  MustHash(suite.Plan.Client.Secret),
-			RedirectURIs:            []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
-			AuthorizationPolicy:     b.policy,
-			ConsentMode:             b.consent,
+			RedirectURIs:            []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
+			AuthorizationPolicy:     b.Policy,
+			ConsentMode:             b.Consent,
 			Public:                  false,
 			Scopes:                  []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"},
 			ResponseTypes:           responseTypes,
@@ -160,9 +167,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 			ID:                      suite.Plan.ClientAlternate.ID,
 			Name:                    fmt.Sprintf("%s (Alternate)", description),
 			Secret:                  MustHash(suite.Plan.ClientAlternate.Secret),
-			RedirectURIs:            []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
-			AuthorizationPolicy:     b.policy,
-			ConsentMode:             b.consent,
+			RedirectURIs:            []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
+			AuthorizationPolicy:     b.Policy,
+			ConsentMode:             b.Consent,
 			Public:                  false,
 			Scopes:                  []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"},
 			ResponseTypes:           responseTypes,
@@ -175,9 +182,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS
 			ID:                      suite.Plan.ClientSecretPost.ID,
 			Name:                    fmt.Sprintf("%s (Secret Post)", description),
 			Secret:                  MustHash(suite.Plan.ClientSecretPost.Secret),
-			RedirectURIs:            []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
-			AuthorizationPolicy:     b.policy,
-			ConsentMode:             b.consent,
+			RedirectURIs:            []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()},
+			AuthorizationPolicy:     b.Policy,
+			ConsentMode:             b.Consent,
 			Public:                  false,
 			Scopes:                  []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"},
 			ResponseTypes:           responseTypes,
@@ -206,3 +213,16 @@ func MustHash(value string) *schema.PasswordDigest {
 
 	return schema.NewPasswordDigest(digest)
 }
+
+// Builders returns the conformance suite builders for every profile Authelia is certified for, in a fixed order.
+func Builders(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL) []*SuiteBuilder {
+	return []*SuiteBuilder{
+		{brand, NameConfig, "Config", true, version, consent, policy, nil, autheliaURL},
+		{brand, NameBasic, "Basic", true, version, consent, policy, suiteURL, autheliaURL},
+		{brand, NameBasicFormPost, "Basic (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
+		{brand, NameHybrid, "Hybrid", true, version, consent, policy, suiteURL, autheliaURL},
+		{brand, NameHybridFormPost, "Hybrid (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
+		{brand, NameImplicit, "Implicit", true, version, consent, policy, suiteURL, autheliaURL},
+		{brand, NameImplicitFormPost, "Implicit (Form Post)", true, version, consent, policy, suiteURL, autheliaURL},
+	}
+}

internal/oidc/conformance/builder_test.go

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-package main
+package conformance
 
 import (
 	"net/url"
@@ -20,7 +20,16 @@ func TestMustHash(t *testing.T) {
 	})
 }
 
-func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
+func TestSuiteBuilder_Description(t *testing.T) {
+	certification := &SuiteBuilder{Friendly: "Basic", Certification: true, Version: "k3j2x9ab", AutheliaURL: &url.URL{Scheme: "https", Host: "auth.example.com"}}
+	test := &SuiteBuilder{Friendly: "Basic (Form Post)", Version: "k3j2x9ab"}
+
+	assert.Equal(t, "Authelia v4.39.24 (commit 7498f635) Basic Certification Profile", certification.Description("v4.39.24 (commit 7498f635)"))
+	assert.Equal(t, "Authelia commit 7498f635 Basic (Form Post) Test Profile", test.Description("commit 7498f635"))
+	assert.Equal(t, certification.Build().Plan.Description, certification.Description(certification.Version), "the build describes the plan by its version")
+}
+
+func TestSuiteBuilder_Build(t *testing.T) {
 	suiteURL := &url.URL{
 		Scheme: "https",
 		Host:   "conformance.example.com",
@@ -35,20 +44,40 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 
 	testCases := []struct {
 		name     string
-		have     *OpenIDConnectConformanceSuiteBuilder
-		expected OpenIDConnectConformanceSuite
+		have     *SuiteBuilder
+		expected Suite
 	}{
 		{
 			"ShouldHandleConfig",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "config", "Config", true, "4.40", "implicit", "one_factor", nil, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "config", "Config", true, "4.40", "implicit", "one_factor", nil, autheliaURL},
+			Suite{
 				Name: "conformance-config",
-				Plan: OpenIDConnectConformanceSuitePlan{
+				Plan: Plan{
 					Name:        "oidcc-config-certification-test-plan",
 					Alias:       "conformance-config-authelia440",
 					Description: "Authelia 4.40 Config Certification Profile",
 					Publish:     "summary",
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
+						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
+					},
+				},
+			},
+		},
+		{
+			"ShouldHandleBasicTestProfile",
+			&SuiteBuilder{"authelia", "basic", "Basic", false, "4.40", "implicit", "one_factor", nil, autheliaURL},
+			Suite{
+				Name: "basic",
+				Plan: Plan{
+					Name:        "oidcc-basic-certification-test-plan",
+					Alias:       "basic-authelia440",
+					Description: "Authelia 4.40 Basic Test Profile",
+					Publish:     "summary",
+					Variant: &PlanVariant{
+						ServerMetadata:     "discovery",
+						ClientRegistration: "static_client",
+					},
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
 				},
@@ -56,30 +85,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleBasic",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "basic", "Basic", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "basic", "Basic", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-basic",
-				Plan: OpenIDConnectConformanceSuitePlan{
+				Plan: Plan{
 					Name:        "oidcc-basic-certification-test-plan",
 					Alias:       "conformance-basic-authelia440",
 					Description: "Authelia 4.40 Basic Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-basic-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-basic-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-basic-authelia440-post",
 						Secret: "present",
 					},
@@ -126,30 +155,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleBasicFormPost",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "basic-form-post", "Basic (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "basic-form-post", "Basic (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-basic-form-post",
-				Plan: OpenIDConnectConformanceSuitePlan{
-					Name:        "oidcc-basic-form-post-certification-test-plan",
+				Plan: Plan{
+					Name:        "oidcc-formpost-basic-certification-test-plan",
 					Alias:       "conformance-basic-form-post-authelia440",
 					Description: "Authelia 4.40 Basic (Form Post) Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-basic-form-post-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-basic-form-post-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-basic-form-post-authelia440-post",
 						Secret: "present",
 					},
@@ -196,30 +225,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleImplicit",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "implicit", "Implicit", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "implicit", "Implicit", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-implicit",
-				Plan: OpenIDConnectConformanceSuitePlan{
+				Plan: Plan{
 					Name:        "oidcc-implicit-certification-test-plan",
 					Alias:       "conformance-implicit-authelia440",
 					Description: "Authelia 4.40 Implicit Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-implicit-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-implicit-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-implicit-authelia440-post",
 						Secret: "present",
 					},
@@ -266,30 +295,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleImplicitFormPost",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "implicit-form-post", "Implicit (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "implicit-form-post", "Implicit (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-implicit-form-post",
-				Plan: OpenIDConnectConformanceSuitePlan{
-					Name:        "oidcc-implicit-form-post-certification-test-plan",
+				Plan: Plan{
+					Name:        "oidcc-formpost-implicit-certification-test-plan",
 					Alias:       "conformance-implicit-form-post-authelia440",
 					Description: "Authelia 4.40 Implicit (Form Post) Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-implicit-form-post-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-implicit-form-post-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-implicit-form-post-authelia440-post",
 						Secret: "present",
 					},
@@ -336,30 +365,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleHybrid",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "hybrid", "Hybrid", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "hybrid", "Hybrid", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-hybrid",
-				Plan: OpenIDConnectConformanceSuitePlan{
+				Plan: Plan{
 					Name:        "oidcc-hybrid-certification-test-plan",
 					Alias:       "conformance-hybrid-authelia440",
 					Description: "Authelia 4.40 Hybrid Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-hybrid-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-hybrid-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-hybrid-authelia440-post",
 						Secret: "present",
 					},
@@ -406,30 +435,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		},
 		{
 			"ShouldHandleHybridFormPost",
-			&OpenIDConnectConformanceSuiteBuilder{"authelia", "hybrid-form-post", "Hybrid (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
-			OpenIDConnectConformanceSuite{
+			&SuiteBuilder{"authelia", "hybrid-form-post", "Hybrid (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL},
+			Suite{
 				Name: "conformance-hybrid-form-post",
-				Plan: OpenIDConnectConformanceSuitePlan{
-					Name:        "oidcc-hybrid-form-post-certification-test-plan",
+				Plan: Plan{
+					Name:        "oidcc-formpost-hybrid-certification-test-plan",
 					Alias:       "conformance-hybrid-form-post-authelia440",
 					Description: "Authelia 4.40 Hybrid (Form Post) Certification Profile",
 					Publish:     "summary",
-					Variant: &OpenIDConnectConformanceSuitePlanVariant{
+					Variant: &PlanVariant{
 						ServerMetadata:     "discovery",
 						ClientRegistration: "static_client",
 					},
-					Server: OpenIDConnectConformanceSuitePlanServer{
+					Server: PlanServer{
 						DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration",
 					},
-					Client: &OpenIDConnectConformanceSuitePlanClient{
+					Client: &PlanClient{
 						ID:     "conformance-certification-hybrid-form-post-authelia440",
 						Secret: "present",
 					},
-					ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{
+					ClientAlternate: &PlanClient{
 						ID:     "conformance-certification-hybrid-form-post-authelia440-alt",
 						Secret: "present",
 					},
-					ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{
+					ClientSecretPost: &PlanClient{
 						ID:     "conformance-certification-hybrid-form-post-authelia440-post",
 						Secret: "present",
 					},
@@ -481,6 +510,14 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 			actual := tc.have.Build()
 
 			assert.Equal(t, tc.expected.Name, actual.Name)
+			assert.Equal(t, tc.expected.Plan.Name, actual.Plan.Name)
+			assert.Equal(t, tc.expected.Plan.Variant, actual.Plan.Variant)
+
+			assert.Equal(t, tc.expected.Plan.Alias, actual.Plan.Alias)
+			assert.Equal(t, tc.expected.Plan.Description, actual.Plan.Description)
+			assert.Equal(t, tc.expected.Plan.Publish, actual.Plan.Publish)
+			assert.Equal(t, tc.expected.Plan.Server, actual.Plan.Server)
+
 			require.Equal(t, len(tc.expected.Clients), len(actual.Clients))
 
 			for i, expected := range tc.expected.Clients {
@@ -524,3 +561,31 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) {
 		})
 	}
 }
+
+func TestBuilders(t *testing.T) {
+	suiteURL := &url.URL{Scheme: "https", Host: "conformance.example.com"}
+	autheliaURL := &url.URL{Scheme: "https", Host: "auth.example.com"}
+
+	builders := Builders("4.40", "implicit", "one_factor", "authelia", suiteURL, autheliaURL)
+
+	names := make([]string, len(builders))
+
+	for i, builder := range builders {
+		names[i] = builder.Name
+
+		assert.True(t, builder.Certification, builder.Name)
+		assert.Equal(t, "authelia", builder.Brand, builder.Name)
+		assert.Equal(t, "4.40", builder.Version, builder.Name)
+		assert.Equal(t, "implicit", builder.Consent, builder.Name)
+		assert.Equal(t, "one_factor", builder.Policy, builder.Name)
+		assert.Equal(t, autheliaURL, builder.AutheliaURL, builder.Name)
+
+		if builder.Name == NameConfig {
+			assert.Nil(t, builder.SuiteURL, "the config profile registers no clients, so it has no suite to redirect to")
+		} else {
+			assert.Equal(t, suiteURL, builder.SuiteURL, builder.Name)
+		}
+	}
+
+	assert.Equal(t, []string{NameConfig, NameBasic, NameBasicFormPost, NameHybrid, NameHybridFormPost, NameImplicit, NameImplicitFormPost}, names)
+}

internal/oidc/conformance/const.go

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Package conformance provides the OpenID Connect 1.0 conformance suite plan and client definitions Authelia is
+// certified against. It is shared by the authelia-gen conformance generator and the OIDCConformance integration suite
+// so that a plan's client identifiers, secrets and redirect URIs cannot drift from the clients Authelia is configured
+// with.
+package conformance
+
+const (
+	// NameConfig is the builder name of the Config OP profile.
+	NameConfig = "config"
+
+	// NameBasic is the builder name of the Basic OP profile.
+	NameBasic = "basic"
+
+	// NameBasicFormPost is the builder name of the Basic OP profile using the form post response mode.
+	NameBasicFormPost = "basic-form-post"
+
+	// NameHybrid is the builder name of the Hybrid OP profile.
+	NameHybrid = "hybrid"
+
+	// NameHybridFormPost is the builder name of the Hybrid OP profile using the form post response mode.
+	NameHybridFormPost = "hybrid-form-post"
+
+	// NameImplicit is the builder name of the Implicit OP profile.
+	NameImplicit = "implicit"
+
+	// NameImplicitFormPost is the builder name of the Implicit OP profile using the form post response mode.
+	NameImplicitFormPost = "implicit-form-post"
+)

internal/oidc/conformance/types.go

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package conformance
+
+import (
+	"github.com/authelia/authelia/v4/internal/configuration/schema"
+)
+
+// Suite represents a single OpenID Connect conformance suite.
+type Suite struct {
+	Name    string
+	Plan    Plan
+	Clients []schema.IdentityProvidersOpenIDConnectClient
+}
+
+// Plan represents the test plan for a Suite.
+type Plan struct {
+	Name        string `json:"-"`
+	Alias       string `json:"alias"`
+	Description string `json:"description"`
+	Publish     string `json:"publish"`
+
+	Variant          *PlanVariant   `json:"-"`
+	Server           PlanServer     `json:"server"`
+	Client           *PlanClient    `json:"client,omitempty"`
+	ClientAlternate  *PlanClient    `json:"client2,omitempty"`
+	ClientSecretPost *PlanClient    `json:"client_secret_post,omitempty"`
+	MutualTLS        *PlanMutualTLS `json:"mtls,omitempty"`
+	Resource         *PlanResource  `json:"resource,omitempty"`
+
+	EKYCVerifiedClaimsRequest string `json:"ekyc_verified_claims_request,omitempty"`
+	EKYCUserinfo              string `json:"ekyc_userinfo,omitempty"`
+}
+
+// PlanVariant represents the variant of a Plan.
+type PlanVariant struct {
+	ServerMetadata           string `json:"server_metadata,omitempty"`
+	ClientRegistration       string `json:"client_registration,omitempty"`
+	ClientAuthType           string `json:"client_auth_type,omitempty"`
+	ResponseType             string `json:"response_type,omitempty"`
+	ResponseMode             string `json:"response_mode,omitempty"`
+	CIBAMode                 string `json:"ciba_mode,omitempty"`
+	FAPIProfile              string `json:"fapi_profile,omitempty"`
+	FAPIResponseMode         string `json:"fapi_response_mode,omitempty"`
+	FAPIAuthRequestMethod    string `json:"fapi_auth_request_method,omitempty"`
+	SenderConstrain          string `json:"sender_constrain,omitempty"`
+	AuthorizationRequestType string `json:"authorization_request_type,omitempty"`
+	OpenID                   string `json:"openid,omitempty"`
+}
+
+// PlanServer represents the server details of a Plan.
+type PlanServer struct {
+	DiscoveryURL          string `json:"discoveryUrl,omitempty"`
+	Issuer                string `json:"issuer,omitempty"`
+	JWKSURI               string `json:"jwks_uri,omitempty"`
+	AuthorizationEndpoint string `json:"authorization_endpoint,omitempty"`
+	TokenEndpoint         string `json:"token_endpoint,omitempty"`
+	UserinfoEndpoint      string `json:"userinfo_endpoint,omitempty"`
+	ACRValues             string `json:"acr_values,omitempty"`
+	LoginHint             string `json:"login_hint,omitempty"`
+}
+
+// PlanClient represents a client registered with a Plan.
+type PlanClient struct {
+	ID                   string `json:"client_id,omitempty"`
+	Secret               string `json:"client_secret,omitempty"`
+	Name                 string `json:"client_name,omitempty"`
+	Scope                string `json:"scope,omitempty"`
+	SecretJWTAlgorithm   string `json:"client_secret_jwt_alg,omitempty"`
+	DPOPSigningAlgorithm string `json:"dpop_signing_alg,omitempty"`
+	InitialAccessToken   string `json:"initial_access_token,omitempty"`
+	HintType             string `json:"hint_type,omitempty"`
+	HintValue            string `json:"hint_value,omitempty"`
+	JWKS                 string `json:"jwks,omitempty"`
+}
+
+// PlanMutualTLS represents the Mutual TLS details of a Plan.
+type PlanMutualTLS struct {
+	Certificate          string `json:"certificate,omitempty"`
+	Key                  string `json:"key,omitempty"`
+	CertificateAuthority string `json:"ca,omitempty"`
+}
+
+// PlanResource represents the protected resource details of a Plan.
+type PlanResource struct {
+	ResourceURL                 string `json:"resourceUrl"`
+	ResourceURLAccountRequests  string `json:"resourceUrlAccountRequests,omitempty"`
+	ResourceURLAccountsResource string `json:"resourceUrlAccountsResource,omitempty"`
+	InstitutionID               string `json:"institution_id,omitempty"`
+	RichAuthorizationRequest    string `json:"richAuthorizationRequest,omitempty"`
+}

internal/suites/OIDCConformance/Dockerfile.server

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2026 Authelia
+#
+# SPDX-License-Identifier: Apache-2.0
+
+FROM registry.gitlab.com/openid/conformance-suite:release-v5.2.4
+
+# The conformance server fetches Authelia's discovery document, JWKS, token endpoint and userinfo endpoint over TLS
+# signed by the suite development CA, which no public root vouches for. Importing into the JDK's own cacerts adds that
+# CA while leaving the public roots in place, which a -Djavax.net.ssl.trustStore override would not.
+COPY --link common/pki/ca.public.crt /tmp/authelia-ca.crt
+
+RUN keytool -importcert -noprompt -alias authelia-development-ca -file /tmp/authelia-ca.crt -cacerts -storepass changeit

internal/suites/OIDCConformance/compose.mongodb.yml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2026 Authelia
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+services:
+  conformance-mongodb:
+    image: 'mongo:6.0.13'
+    cpus: 2
+    mem_limit: 2g
+    networks:
+      authelianet: {}
+
+  conformance-server:
+    depends_on:
+      - 'conformance-mongodb'
+...

internal/suites/OIDCConformance/compose.yml

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2026 Authelia
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+services:
+  authelia-backend:
+    environment:
+      X_AUTHELIA_CONFIG_FILTERS: 'template'
+      X_AUTHELIA_CONFIG: '/config/configuration.yml,/tmp/conformance-clients.yml'
+    volumes:
+      - './OIDCConformance/configuration.yml:/config/configuration.yml'
+      - './OIDCConformance/users.yml:/config/users.yml'
+      - './common/pki:/pki'
+      - './common/pki/public.crt:/certs/public.crt'
+      - '${SUITE_TMP:-/tmp}:/tmp'
+
+  conformance-server:
+    build:
+      context: '.'
+      dockerfile: 'OIDCConformance/Dockerfile.server'
+    cpus: 4
+    mem_limit: 4g
+    environment:
+      BASE_URL: 'https://conformance.example.com:8443'
+      MONGODB_HOST: '${SUITE_OIDC_CONFORMANCE_MONGODB_HOST:-conformance-mongodb}'
+      JAVA_EXTRA_ARGS: '-Dfintechlabs.devmode=true'
+      OIDC_GOOGLE_CLIENTID: 'google-client'
+      OIDC_GOOGLE_SECRET: 'google-secret'
+      OIDC_GITLAB_CLIENTID: 'gitlab-client'
+      OIDC_GITLAB_SECRET: 'gitlab-secret'
+    depends_on:
+      - 'authelia-backend'
+    expose:
+      - 8080
+    networks:
+      authelianet: {}
+
+...

internal/suites/OIDCConformance/configuration.yml

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: 2026 Authelia
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+certificates_directory: '/certs/'
+
+server:
+  address: 'tcp://:9091'
+  timeouts:
+    read: '30s'
+  tls:
+    certificate: '/pki/public.backend.crt'
+    key: '/pki/private.backend.pem'
+
+log:
+  level: 'debug'
+
+storage:
+  encryption_key: 'a_not_so_secure_encryption_key'
+  postgres:
+    address: 'tcp://postgres:5432'
+    database: 'authelia'
+    username: 'admin'
+    password: 'password'
+
+notifier:
+  smtp:
+    address: 'smtp://mail.example.com:1025'
+    sender: 'admin@example.com'
+
+identity_validation:
+  reset_password:
+    jwt_secret: 'unsecure_secret'
+
+
+# Every plan in this suite authenticates as the same user against this one Authelia, concurrently, and a module which
+# leaves the sign in form on screen has its credentials resubmitted. Three failures within the default two minute
+# window would ban the account for the default five minutes, failing unrelated modules in the other six plans with
+# errors which point nowhere near the cause. Regulation is not what this suite tests, so it is disabled outright.
+regulation:
+  max_retries: 0
+
+session:
+  secret: 'unsecure_session_secret'
+  cookies:
+    - domain: 'example.com'
+      authelia_url: 'https://login.example.com:8080'
+      expiration: '1 hour'
+      inactivity: '5 minutes'
+      remember_me: '1 year'
+  redis:
+    host: 'redis'
+    port: 6379
+
+authentication_backend:
+  file:
+    path: '/config/users.yml'
+
+totp:
+  disable_reuse_security_policy: true
+
+webauthn:
+  disable: false
+  enable_passkey_login: true
+  display_name: 'Authelia'
+  attestation_conveyance_preference: 'indirect'
+  timeout: '60 seconds'
+  filtering:
+    permitted_aaguids: []
+    prohibited_aaguids: []
+    prohibit_backup_eligibility: false
+  selection_criteria:
+    attachment: ''
+    discoverability: 'required'
+    user_verification: 'preferred'
+  metadata:
+    enabled: false
+    validate_trust_anchor: true
+    validate_entry: false
+    validate_entry_permit_zero_aaguid: true
+    validate_status: true
+    validate_status_permitted: []
+    validate_status_prohibited: []
+
+identity_providers:
+  oidc:
+    enable_client_debug_messages: true
+    hmac_secret: 'IVPWBkAdJHje3uz7LtFTDU2pFUfh39Xm'
+    jwks:
+      - key: {{ secret "/pki/private.oidc.pem" | mindent 10 "|" | msquote }}
+        certificate_chain: {{ secret "/pki/public.oidc.chain.pem" | mindent 10 "|" | msquote }}
+access_control:
+  default_policy: 'deny'
+  rules:
+    - domain: 'home.example.com'
+      policy: 'bypass'
+    - domain: 'public.example.com'
+      policy: 'bypass'
+    - domain: 'admin.example.com'
+      policy: 'two_factor'
+    - domain: 'secure.example.com'
+      policy: 'two_factor'
+    - domain: 'singlefactor.example.com'
+      policy: 'one_factor'
+    - domain: 'oidc.example.com'
+      policy: 'two_factor'
+    - domain: 'oidc-public.example.com'
+      policy: 'bypass'
+...

internal/suites/OIDCConformance/users.yml

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2026 Authelia
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+users:
+  john:
+    displayname: "John Doe"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"  # yamllint disable-line rule:line-length
+    given_name: "John"
+    middle_name: "Richard"
+    family_name: "Doe"
+    nickname: "Johnny"
+    gender: "male"
+    birthdate: "1985-07-31"
+    website: "https://john.example.com"
+    profile: "https://john.example.com/profile"
+    picture: "https://john.example.com/picture.png"
+    zoneinfo: "Australia/Melbourne"
+    locale: "en-AU"
+    phone_number: "+610000000000"
+    phone_extension: "1234"
+    email: john.doe@authelia.com
+    address:
+      street_address: "123 Example Street"
+      locality: "Melbourne"
+      region: "Victoria"
+      postal_code: "3000"
+      country: "Australia"
+    groups:
+      - admins
+      - dev
+
+  harry:
+    displayname: "Harry Potter"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"  # yamllint disable-line rule:line-length
+    email: harry.potter@authelia.com
+    groups: []
+
+  bob:
+    displayname: "Bob Dylan"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"  # yamllint disable-line rule:line-length
+    email: bob.dylan@authelia.com
+    groups:
+      - dev
+
+  james:
+    displayname: "James Dean"
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"  # yamllint disable-line rule:line-length
+    email: james.dean@authelia.com
+...

internal/suites/conformance_api.go

@@ -0,0 +1,439 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/authelia/authelia/v4/internal/oidc/conformance"
+	"github.com/authelia/authelia/v4/internal/utils"
+)
+
+const (
+	conformanceWaitStateTimeout       = time.Second * 30
+	conformanceReadyInterval          = time.Second * 2
+	conformanceWaitStateRetryInterval = time.Millisecond * 500
+)
+
+// ConformancePlanModule is one entry in a conformance test plan's module list.
+type ConformancePlanModule struct {
+	TestModule string            `json:"testModule"`
+	Variant    map[string]string `json:"variant"`
+}
+
+// ConformanceCreatedPlan is the response to creating a conformance test plan.
+type ConformanceCreatedPlan struct {
+	ID      string                  `json:"id"`
+	Modules []ConformancePlanModule `json:"modules"`
+}
+
+// ConformanceTestInfo is the persisted state of a single conformance test module instance.
+type ConformanceTestInfo struct {
+	ID       string `json:"_id"`
+	TestName string `json:"testName"`
+	Status   string `json:"status"`
+	Result   string `json:"result"`
+}
+
+// ConformanceBrowserStatus describes the URLs a module is waiting for a browser to visit.
+type ConformanceBrowserStatus struct {
+	URLs    []string `json:"urls"`
+	Visited []string `json:"visited"`
+}
+
+// ConformanceLogEntry is one entry of a module's log. An entry with a non-empty Upload is an image placeholder which
+// has not been filled, and which is holding the module in WAITING.
+type ConformanceLogEntry struct {
+	Msg    string `json:"msg"`
+	Result string `json:"result"`
+	Src    string `json:"src"`
+	Upload string `json:"upload"`
+
+	Fields map[string]any `json:"-"`
+}
+
+// UnmarshalJSON decodes the typed fields and retains the entry whole.
+func (e *ConformanceLogEntry) UnmarshalJSON(data []byte) (err error) {
+	type entry ConformanceLogEntry
+
+	if err = json.Unmarshal(data, (*entry)(e)); err != nil {
+		return err
+	}
+
+	return json.Unmarshal(data, &e.Fields)
+}
+
+// ConformanceClient talks to the OpenID Foundation conformance suite's HTTP API. The suite runs with
+// fintechlabs.devmode enabled, which disables its OAuth login, so no credentials are sent.
+type ConformanceClient struct {
+	base   *url.URL
+	client *http.Client
+}
+
+// NewConformanceClient returns a ConformanceClient for the conformance suite at base.
+func NewConformanceClient(base string) (client *ConformanceClient, err error) {
+	var parsed *url.URL
+
+	if parsed, err = url.ParseRequestURI(base); err != nil {
+		return nil, fmt.Errorf("error parsing conformance base url: %w", err)
+	}
+
+	return &ConformanceClient{
+		base: parsed,
+		client: &http.Client{
+			Transport: NewHTTPTransport(),
+			Timeout:   conformanceWaitStateTimeout + time.Second*10,
+		},
+	}, nil
+}
+
+func (c *ConformanceClient) uri(query url.Values, elem ...string) string {
+	uri := c.base.JoinPath(append([]string{"api"}, elem...)...)
+
+	if query != nil {
+		uri.RawQuery = query.Encode()
+	}
+
+	return uri.String()
+}
+
+// LogDetailURL returns the human readable log page for a module, for inclusion in failure output.
+func (c *ConformanceClient) LogDetailURL(id string) string {
+	uri := c.base.JoinPath("log-detail.html")
+
+	query := uri.Query()
+	query.Set("log", id)
+	uri.RawQuery = query.Encode()
+
+	return uri.String()
+}
+
+type conformanceUnexpectedStatusError struct {
+	Method   string
+	URI      string
+	Body     []byte
+	Expected int
+	Status   int
+}
+
+// Error implements the error interface.
+func (e *conformanceUnexpectedStatusError) Error() string {
+	return fmt.Sprintf("%s %s: expected status %d but got %d: %s", e.Method, e.URI, e.Expected, e.Status, e.Body)
+}
+
+func (c *ConformanceClient) do(ctx context.Context, method, uri string, body io.Reader, contentType string, expected int, out any) (err error) {
+	data, err := c.doRaw(ctx, method, uri, body, contentType, expected)
+	if err != nil {
+		return err
+	}
+
+	if out == nil {
+		return nil
+	}
+
+	return json.Unmarshal(data, out)
+}
+
+func (c *ConformanceClient) doRaw(ctx context.Context, method, uri string, body io.Reader, contentType string, expected int) (data []byte, err error) {
+	var req *http.Request
+
+	if req, err = http.NewRequestWithContext(ctx, method, uri, body); err != nil {
+		return nil, err
+	}
+
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+
+	var resp *http.Response
+
+	if resp, err = c.client.Do(req); err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	if data, err = io.ReadAll(resp.Body); err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != expected {
+		return nil, &conformanceUnexpectedStatusError{Method: method, URI: uri, Body: data, Expected: expected, Status: resp.StatusCode}
+	}
+
+	return data, nil
+}
+
+// WaitReady blocks until the conformance server answers an API request or ctx expires. The JVM binds its port well
+// before Spring has finished starting, so a connection check is not sufficient.
+func (c *ConformanceClient) WaitReady(ctx context.Context) (err error) {
+	query := url.Values{}
+	query.Set("length", "1")
+
+	for {
+		if err = c.do(ctx, http.MethodGet, c.uri(query, "plan"), nil, "", http.StatusOK, nil); err == nil {
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("conformance suite was not ready: %w (last error: %v)", ctx.Err(), err)
+		case <-time.After(conformanceReadyInterval):
+		}
+	}
+}
+
+// CreatePlan creates a conformance test plan and returns its id and module list. The plan name and the variant are
+// parameters rather than being read off plan because [conformance.Plan] deliberately excludes both from its JSON
+// representation: they belong in the query string, not the body, and the caller is the one which knows them.
+func (c *ConformanceClient) CreatePlan(ctx context.Context, name string, variant *conformance.PlanVariant, plan *conformance.Plan) (created *ConformanceCreatedPlan, err error) {
+	if name == "" {
+		return nil, fmt.Errorf("error creating plan '%s': the conformance plan name is empty", plan.Alias)
+	}
+
+	body, err := json.Marshal(plan)
+	if err != nil {
+		return nil, err
+	}
+
+	query := url.Values{}
+	query.Set("planName", name)
+
+	if variant != nil {
+		var data []byte
+
+		if data, err = json.Marshal(variant); err != nil {
+			return nil, err
+		}
+
+		query.Set("variant", string(data))
+	}
+
+	created = &ConformanceCreatedPlan{}
+
+	if err = c.do(ctx, http.MethodPost, c.uri(query, "plan"), bytes.NewReader(body), "application/json", http.StatusCreated, created); err != nil {
+		return nil, err
+	}
+
+	if created.ID == "" {
+		return nil, fmt.Errorf("error creating plan '%s': the conformance suite returned no plan identifier", plan.Alias)
+	}
+
+	return created, nil
+}
+
+// CreateTest creates an instance of a plan's module and returns its id. The variant is sent alongside the plan even
+// though the endpoint documents it as applying to standalone tests: the upstream scripts/conformance.py sends both
+// together for plan modules, and it is what makes plans which repeat a module across variants work.
+func (c *ConformanceClient) CreateTest(ctx context.Context, planID, module string, variant map[string]string) (id string, err error) {
+	query := url.Values{}
+	query.Set("test", module)
+	query.Set("plan", planID)
+
+	if len(variant) != 0 {
+		var data []byte
+
+		if data, err = json.Marshal(variant); err != nil {
+			return "", err
+		}
+
+		query.Set("variant", string(data))
+	}
+
+	out := struct {
+		ID string `json:"id"`
+	}{}
+
+	if err = c.do(ctx, http.MethodPost, c.uri(query, "runner"), nil, "application/json", http.StatusCreated, &out); err != nil {
+		return "", err
+	}
+
+	if out.ID == "" {
+		return "", fmt.Errorf("error creating module '%s': the conformance suite returned no test identifier", module)
+	}
+
+	return out.ID, nil
+}
+
+// StartTest starts a module which is waiting in the CONFIGURED state.
+func (c *ConformanceClient) StartTest(ctx context.Context, id string) error {
+	return c.do(ctx, http.MethodPost, c.uri(nil, "runner", id), nil, "", http.StatusOK, nil)
+}
+
+// WaitState long polls until the module reaches one of states, or ctx expires. The server clamps each poll to 30
+// seconds and answers {"timeout":true} when it expires, so this loops. Once a module finishes it leaves the running
+// registry and the endpoint answers 404; the persisted status from Info is authoritative at that point. A transport
+// level failure - a 5xx response, a network error - is treated as transient and retried until ctx expires, so a
+// single blip during a long unattended run does not fail the poll outright. Any other status will not change on retry,
+// so it fails immediately.
+//
+// A 200 whose body is not the documented shape is not retried. A body which does not decode at all, and a body which
+// decodes but carries neither a state nor the timeout flag - an empty body, a proxy's own 200 page, a schema change
+// on a suite version bump - are both permanent for as long as the run lasts, so retrying either would only spin
+// against the server for the rest of the module budget from every plan at once. Both fail immediately with the body
+// echoed, which is also the only way the reason for a schema change ever reaches the run's output.
+func (c *ConformanceClient) WaitState(ctx context.Context, id string, states ...string) (state string, err error) {
+	query := url.Values{}
+	query.Set("states", strings.Join(states, ","))
+	query.Set("timeoutMs", strconv.FormatInt(conformanceWaitStateTimeout.Milliseconds(), 10))
+
+	for {
+		data, err := c.doRaw(ctx, http.MethodGet, c.uri(query, "runner", id, "wait-state"), nil, "", http.StatusOK)
+		if err != nil {
+			if ctx.Err() != nil {
+				return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err())
+			}
+
+			var statusErr *conformanceUnexpectedStatusError
+
+			if errors.As(err, &statusErr) {
+				switch {
+				case statusErr.Status == http.StatusNotFound:
+					return c.waitStateFromInfo(ctx, id, states)
+				case statusErr.Status < http.StatusInternalServerError:
+					return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, err)
+				}
+			}
+
+			select {
+			case <-ctx.Done():
+				return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err())
+			case <-time.After(conformanceWaitStateRetryInterval):
+			}
+
+			continue
+		}
+
+		out := struct {
+			State   string `json:"state"`
+			Timeout bool   `json:"timeout"`
+		}{}
+
+		if err = json.Unmarshal(data, &out); err != nil {
+			return "", fmt.Errorf("module '%s' wait-state poll answered with a body which is not JSON: %w: %s", id, err, data)
+		}
+
+		switch {
+		case out.State != "":
+			return out.State, nil
+		case out.Timeout:
+			// The poll expired without a transition. Fall through to the context check and poll again.
+		default:
+			return "", fmt.Errorf("module '%s' wait-state poll answered with neither a state nor a timeout: %s", id, data)
+		}
+
+		if ctx.Err() != nil {
+			return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err())
+		}
+	}
+}
+
+func (c *ConformanceClient) waitStateFromInfo(ctx context.Context, id string, states []string) (state string, err error) {
+	var info *ConformanceTestInfo
+
+	if info, err = c.Info(ctx, id); err != nil {
+		if ctx.Err() != nil {
+			return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err())
+		}
+
+		return "", err
+	}
+
+	if utils.IsStringInSlice(info.Status, states) {
+		return info.Status, nil
+	}
+
+	return "", fmt.Errorf("module '%s' is no longer running and its persisted status is '%s', which is not one of %v", id, info.Status, states)
+}
+
+// Info returns the persisted status and result of a module.
+func (c *ConformanceClient) Info(ctx context.Context, id string) (info *ConformanceTestInfo, err error) {
+	info = &ConformanceTestInfo{}
+
+	if err = c.do(ctx, http.MethodGet, c.uri(nil, "info", id), nil, "", http.StatusOK, info); err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// BrowserStatus returns the URLs a module is waiting to have visited.
+func (c *ConformanceClient) BrowserStatus(ctx context.Context, id string) (status *ConformanceBrowserStatus, err error) {
+	status = &ConformanceBrowserStatus{}
+
+	if err = c.do(ctx, http.MethodGet, c.uri(nil, "runner", "browser", id), nil, "", http.StatusOK, status); err != nil {
+		return nil, err
+	}
+
+	return status, nil
+}
+
+// Visit marks a URL as visited. The server matches by exact string equality, so uri must be the URL as it was given.
+func (c *ConformanceClient) Visit(ctx context.Context, id, uri string) error {
+	query := url.Values{}
+	query.Set("url", uri)
+
+	return c.do(ctx, http.MethodPost, c.uri(query, "runner", "browser", id, "visit"), nil, "", http.StatusNoContent, nil)
+}
+
+// Log returns a module's log entries.
+func (c *ConformanceClient) Log(ctx context.Context, id string) (entries []ConformanceLogEntry, err error) {
+	if err = c.do(ctx, http.MethodGet, c.uri(nil, "log", id), nil, "", http.StatusOK, &entries); err != nil {
+		return nil, err
+	}
+
+	return entries, nil
+}
+
+// UploadPlaceholder fills an image placeholder, releasing a module which is blocked in waitForPlaceholders().
+func (c *ConformanceClient) UploadPlaceholder(ctx context.Context, id, placeholder, dataURI string) error {
+	return c.do(ctx, http.MethodPost, c.uri(nil, "log", id, "images", placeholder), bytes.NewReader([]byte(dataURI)), "text/plain", http.StatusOK, nil)
+}
+
+// ExportPlanHTML writes a plan's HTML export to path, for collection as a CI artifact.
+func (c *ConformanceClient) ExportPlanHTML(ctx context.Context, planID, path string) (err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.uri(nil, "plan", "exporthtml", planID), nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected status %d exporting plan '%s'", resp.StatusCode, planID)
+	}
+
+	if err = os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	_, err = io.Copy(f, resp.Body)
+
+	return err
+}

internal/suites/conformance_api_test.go

@@ -0,0 +1,651 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/authelia/authelia/v4/internal/oidc/conformance"
+)
+
+const conformanceTestScreenshot = "data:image/png;base64,iVBORw0KGgo="
+
+func TestConformanceClient_CreatePlan(t *testing.T) {
+	var query string
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/plan", r.URL.Path)
+
+		query = r.URL.RawQuery
+
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"name":"oidcc-basic-certification-test-plan","id":"plan1","modules":[{"testModule":"oidcc-server","variant":{"response_type":"code"}}]}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	plan := &conformance.Plan{Alias: "alias"}
+
+	created, err := client.CreatePlan(context.Background(), "oidcc-basic-certification-test-plan", &conformance.PlanVariant{ServerMetadata: "discovery"}, plan)
+	require.NoError(t, err)
+
+	assert.Equal(t, "plan1", created.ID)
+	require.Len(t, created.Modules, 1)
+	assert.Equal(t, "oidcc-server", created.Modules[0].TestModule)
+	assert.Equal(t, "code", created.Modules[0].Variant["response_type"])
+	assert.Contains(t, query, "planName=oidcc-basic-certification-test-plan")
+	assert.Contains(t, query, "variant=")
+}
+
+func TestConformanceClient_CreatePlanRejectsAResponseWithoutAnIdentifier(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"name":"oidcc-basic-certification-test-plan","modules":[{"testModule":"oidcc-server"}]}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	_, err = client.CreatePlan(context.Background(), "oidcc-basic-certification-test-plan", nil, &conformance.Plan{Alias: "the-alias"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "the-alias")
+	assert.Contains(t, err.Error(), "returned no plan identifier")
+}
+
+func TestConformanceClient_WaitStateReturnsPersistedStatusWhenTestNoLongerRunning(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/runner/abc/wait-state":
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = w.Write([]byte(`{"error":"test not found"}`))
+		case "/api/info/abc":
+			_, _ = w.Write([]byte(`{"_id":"abc","testName":"oidcc-server","status":"FINISHED","result":"PASSED"}`))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	state, err := client.WaitState(ctx, "abc", "FINISHED")
+	require.NoError(t, err)
+	assert.Equal(t, "FINISHED", state)
+}
+
+func TestConformanceClient_WaitStateRetriesOnTimeout(t *testing.T) {
+	var calls int
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path)
+
+		calls++
+
+		if calls == 1 {
+			_, _ = w.Write([]byte(`{"timeout":true}`))
+
+			return
+		}
+
+		_, _ = w.Write([]byte(`{"state":"WAITING"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	state, err := client.WaitState(ctx, "abc", "WAITING")
+	require.NoError(t, err)
+	assert.Equal(t, "WAITING", state)
+	assert.Equal(t, 2, calls)
+}
+
+func TestConformanceClient_WaitStateRetriesOnTransientError(t *testing.T) {
+	var calls int
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/runner/abc/wait-state":
+			calls++
+
+			if calls == 1 {
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(`{"error":"internal error"}`))
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"state":"FINISHED"}`))
+		case "/api/info/abc":
+			t.Errorf("unexpected call to %s: a transient 5xx must not fall back to Info", r.URL.Path)
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	state, err := client.WaitState(ctx, "abc", "FINISHED")
+	require.NoError(t, err)
+	assert.Equal(t, "FINISHED", state)
+	assert.Equal(t, 2, calls)
+}
+
+func TestConformanceClient_WaitStateReturnsFramedErrorOnPersistentTransientFailure(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path)
+
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{"error":"internal error"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750)
+	defer cancel()
+
+	start := time.Now()
+
+	_, err = client.WaitState(ctx, "abc", "FINISHED")
+
+	elapsed := time.Since(start)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "abc")
+	assert.Contains(t, err.Error(), "FINISHED")
+	assert.Less(t, elapsed, time.Second*5, "WaitState should return promptly once ctx expires rather than hang")
+}
+
+func TestConformanceClient_CreateTestReportsAliasCollision(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusConflict)
+		_, _ = w.Write([]byte(`{"error":"alias already in use"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	_, err = client.CreateTest(context.Background(), "plan1", "oidcc-server", nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "alias already in use")
+}
+
+func TestConformanceClient_CreateTestRejectsAResponseWithoutAnIdentifier(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"name":"oidcc-server"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	id, err := client.CreateTest(context.Background(), "plan1", "oidcc-server", nil)
+	require.Error(t, err)
+	assert.Empty(t, id)
+	assert.Contains(t, err.Error(), "oidcc-server")
+	assert.Contains(t, err.Error(), "no test identifier")
+}
+
+func TestConformanceClient_WaitStateFailsFastOnAClientError(t *testing.T) {
+	for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden} {
+		t.Run(http.StatusText(status), func(t *testing.T) {
+			var calls atomic.Int32
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				calls.Add(1)
+				w.WriteHeader(status)
+			}))
+
+			defer server.Close()
+
+			client, err := NewConformanceClient(server.URL)
+			require.NoError(t, err)
+
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			defer cancel()
+
+			_, err = client.WaitState(ctx, "m1", "FINISHED")
+			require.Error(t, err)
+			require.NoError(t, ctx.Err(), "a client error is not going to change on retry, so it must not wait out the context")
+
+			var statusErr *conformanceUnexpectedStatusError
+
+			require.ErrorAs(t, err, &statusErr)
+			assert.Equal(t, status, statusErr.Status)
+			assert.Contains(t, err.Error(), "module 'm1'")
+			assert.Equal(t, int32(1), calls.Load())
+		})
+	}
+}
+
+func TestConformanceClient_LogExposesUnfilledPlaceholders(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/log/abc", r.URL.Path)
+
+		_, _ = w.Write([]byte(`[{"msg":"a"},{"msg":"upload me","upload":"ph1","result":"REVIEW"}]`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	entries, err := client.Log(context.Background(), "abc")
+	require.NoError(t, err)
+	require.Len(t, entries, 2)
+	assert.Equal(t, "", entries[0].Upload)
+	assert.Equal(t, "ph1", entries[1].Upload)
+}
+
+func TestConformanceClient_UploadPlaceholder(t *testing.T) {
+	var body []byte
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/log/abc/images/ph1", r.URL.Path)
+
+		body, _ = io.ReadAll(r.Body)
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	require.NoError(t, client.UploadPlaceholder(context.Background(), "abc", "ph1", conformanceTestScreenshot))
+	assert.Equal(t, conformanceTestScreenshot, string(body))
+}
+
+func TestConformanceClient_CreatePlanRejectsAnEmptyPlanName(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Errorf("the client must not post a nameless plan, got %s", r.URL.String())
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	_, err = client.CreatePlan(context.Background(), "", nil, &conformance.Plan{Alias: "alias"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "alias")
+	assert.Contains(t, err.Error(), "plan name is empty")
+}
+
+func TestConformanceClient_WaitStateFailsFastOnAnUnrecognizedBody(t *testing.T) {
+	testCases := []struct {
+		name     string
+		body     string
+		expected string
+	}{
+		{"ShouldFailOnAnEmptyJSONObject", `{}`, "neither a state nor a timeout"},
+		{"ShouldFailOnAFalseTimeout", `{"timeout":false}`, "neither a state nor a timeout"},
+		{"ShouldFailOnASchemaChange", `{"status":"FINISHED"}`, "neither a state nor a timeout"},
+		{"ShouldFailOnAMalformedBody", `<html>gateway</html>`, "is not JSON"},
+		{"ShouldFailOnAnEmptyBody", ``, "is not JSON"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var calls int
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path)
+
+				calls++
+
+				_, _ = w.Write([]byte(tc.body))
+			}))
+
+			defer server.Close()
+
+			client, err := NewConformanceClient(server.URL)
+			require.NoError(t, err)
+
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
+			start := time.Now()
+
+			_, err = client.WaitState(ctx, "abc", "FINISHED")
+
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.expected)
+			assert.Contains(t, err.Error(), "abc")
+			assert.Equal(t, 1, calls, "an unrecognized body must not be retried")
+			assert.Less(t, time.Since(start), time.Second, "an unrecognized body must fail fast rather than spin")
+		})
+	}
+}
+
+func TestNewConformanceClientRejectsAnInvalidURL(t *testing.T) {
+	_, err := NewConformanceClient("not a url")
+
+	assert.ErrorContains(t, err, "error parsing conformance base url")
+}
+
+func TestConformanceClient_WaitReady(t *testing.T) {
+	t.Run("ShouldReturnOnceTheServerAnswers", func(t *testing.T) {
+		var calls atomic.Int32
+
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "/api/plan", r.URL.Path)
+			assert.Equal(t, "1", r.URL.Query().Get("length"))
+
+			if calls.Add(1) == 1 {
+				w.WriteHeader(http.StatusServiceUnavailable)
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"data":[]}`))
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		require.NoError(t, client.WaitReady(ctx))
+		assert.Equal(t, int32(2), calls.Load(), "the server is asked again after it could not answer")
+	})
+
+	t.Run("ShouldGiveUpWhenTheContextExpires", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+		defer cancel()
+
+		err = client.WaitReady(ctx)
+
+		assert.ErrorContains(t, err, "conformance suite was not ready")
+		assert.ErrorContains(t, err, "expected status 200 but got 503")
+	})
+}
+
+func TestConformanceClient_Requests(t *testing.T) {
+	var (
+		mu       sync.Mutex
+		requests []string
+	)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+
+		requests = append(requests, r.Method+" "+r.URL.RequestURI())
+		mu.Unlock()
+
+		switch r.URL.Path {
+		case "/api/runner":
+			w.WriteHeader(http.StatusCreated)
+			_, _ = w.Write([]byte(`{"id":"m1"}`))
+		case "/api/runner/m1":
+			_, _ = w.Write([]byte(`{}`))
+		case "/api/runner/browser/m1/visit":
+			w.WriteHeader(http.StatusNoContent)
+		case "/api/plan/exporthtml/p1":
+			_, _ = w.Write([]byte("PK archive"))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+
+	id, err := client.CreateTest(ctx, "p1", "oidcc-server", map[string]string{"response_type": "code"})
+	require.NoError(t, err)
+	assert.Equal(t, "m1", id)
+
+	require.NoError(t, client.StartTest(ctx, "m1"))
+	require.NoError(t, client.Visit(ctx, "m1", "https://login.example.com:8080/api/oidc/authorization?client_id=a&state=b"))
+
+	archive := filepath.Join(t.TempDir(), "plans", "conformance-basic.zip")
+
+	require.NoError(t, client.ExportPlanHTML(ctx, "p1", archive))
+
+	data, err := os.ReadFile(archive)
+	require.NoError(t, err)
+	assert.Equal(t, "PK archive", string(data))
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.Equal(t, []string{
+		"POST /api/runner?plan=p1&test=oidcc-server&variant=%7B%22response_type%22%3A%22code%22%7D",
+		"POST /api/runner/m1",
+		"POST /api/runner/browser/m1/visit?url=https%3A%2F%2Flogin.example.com%3A8080%2Fapi%2Foidc%2Fauthorization%3Fclient_id%3Da%26state%3Db",
+		"GET /api/plan/exporthtml/p1",
+	}, requests)
+}
+
+func TestConformanceClient_ReportsServerErrors(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{"error":"broken"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+
+	_, err = client.CreatePlan(ctx, "oidcc-basic-certification-test-plan", nil, &conformance.Plan{Alias: "alias"})
+	assert.ErrorContains(t, err, "broken")
+
+	_, err = client.Info(ctx, "m1")
+	assert.ErrorContains(t, err, "broken")
+
+	_, err = client.BrowserStatus(ctx, "m1")
+	assert.ErrorContains(t, err, "broken")
+
+	_, err = client.Log(ctx, "m1")
+	assert.ErrorContains(t, err, "broken")
+
+	err = client.ExportPlanHTML(ctx, "p1", filepath.Join(t.TempDir(), "conformance-basic.zip"))
+	assert.EqualError(t, err, "unexpected status 500 exporting plan 'p1'")
+}
+
+func TestConformanceClient_ExportPlanHTMLReportsADirectoryItCannotCreate(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("PK archive"))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	file := filepath.Join(t.TempDir(), "file")
+	require.NoError(t, os.WriteFile(file, nil, 0600))
+
+	assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", filepath.Join(file, "conformance-basic.zip")))
+}
+
+func TestConformanceClient_LogRejectsAMalformedEntry(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`[{"msg":5}]`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	_, err = client.Log(context.Background(), "m1")
+	assert.Error(t, err)
+}
+
+func TestConformanceClient_WaitStateFromInfo(t *testing.T) {
+	testCases := []struct {
+		name     string
+		info     func(w http.ResponseWriter)
+		expected string
+	}{
+		{
+			"ShouldReportAPersistedStatusWhichIsNotWaitedFor",
+			func(w http.ResponseWriter) { _, _ = w.Write([]byte(`{"_id":"m1","status":"INTERRUPTED"}`)) },
+			"module 'm1' is no longer running and its persisted status is 'INTERRUPTED', which is not one of [FINISHED]",
+		},
+		{
+			"ShouldReportThatThePersistedStatusCouldNotBeRead",
+			func(w http.ResponseWriter) { w.WriteHeader(http.StatusInternalServerError) },
+			"expected status 200 but got 500",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				switch r.URL.Path {
+				case "/api/runner/m1/wait-state":
+					w.WriteHeader(http.StatusNotFound)
+				case "/api/info/m1":
+					tc.info(w)
+				default:
+					t.Errorf("unexpected path %s", r.URL.Path)
+				}
+			}))
+
+			defer server.Close()
+
+			client, err := NewConformanceClient(server.URL)
+			require.NoError(t, err)
+
+			_, err = client.WaitState(context.Background(), "m1", "FINISHED")
+
+			assert.ErrorContains(t, err, tc.expected)
+		})
+	}
+}
+
+func TestConformanceClient_WaitStateStopsWhenTheContextExpiresBetweenPolls(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(time.Millisecond * 20)
+
+		_, _ = w.Write([]byte(`{"timeout":true}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+	defer cancel()
+
+	_, err = client.WaitState(ctx, "m1", "FINISHED")
+
+	assert.ErrorContains(t, err, "module 'm1' did not reach one of [FINISHED]")
+}
+
+func TestConformanceClient_ExportPlanHTMLReportsWhatFailed(t *testing.T) {
+	t.Run("ShouldReportAServerWhichCannotBeReached", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		server.Close()
+
+		assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", filepath.Join(t.TempDir(), "conformance-basic.zip")))
+	})
+
+	t.Run("ShouldReportAnArchiveWhichCannotBeCreated", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write([]byte("PK archive"))
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", t.TempDir()), "the path is a directory")
+	})
+}
+
+func TestConformanceClient_WaitStateFromInfoStopsWhenTheContextExpires(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/runner/m1/wait-state" {
+			w.WriteHeader(http.StatusNotFound)
+
+			return
+		}
+
+		time.Sleep(time.Millisecond * 200)
+
+		_, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`))
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+	defer cancel()
+
+	_, err = client.WaitState(ctx, "m1", "FINISHED")
+
+	assert.ErrorContains(t, err, "module 'm1' did not reach one of [FINISHED]")
+}

internal/suites/conformance_browser.go

@@ -0,0 +1,556 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
+	log "github.com/sirupsen/logrus"
+)
+
+// ConformancePageState is what the browser is looking at part way through a conformance module's authorization flow.
+type ConformancePageState int
+
+const (
+	// ConformancePageUnknown is a page the driver has no action for.
+	ConformancePageUnknown ConformancePageState = iota
+
+	// ConformancePageFirstFactor is Authelia's first factor sign in form.
+	ConformancePageFirstFactor
+
+	// ConformancePageConsent is Authelia's OpenID Connect 1.0 consent decision stage.
+	ConformancePageConsent
+
+	// ConformancePageAutheliaError is Authelia reporting an error rather than presenting a stage.
+	ConformancePageAutheliaError
+
+	// ConformancePageCallback is the conformance suite's own callback, which ends a leg of the flow.
+	ConformancePageCallback
+)
+
+const (
+	conformanceSelectorFirstFactor             = "#first-factor-stage"
+	conformanceSelectorConsent                 = "#openid-consent-decision-stage"
+	conformanceSelectorConsentAccept           = "#openid-consent-accept"
+	conformanceSelectorConsentReauthentication = "#openid-consent-prompt-login"
+	conformanceSelectorConsentPassword         = "#openid-consent-prompt-login #password-textfield"
+	conformanceSelectorConsentAuthenticate     = "#openid-consent-authenticate"
+	conformanceSelectorUsername                = "#username-textfield"
+	conformanceSelectorPassword                = "#password-textfield"
+	conformanceSelectorSignIn                  = "#sign-in-button"
+	conformanceCallbackPathPrefix              = "/test/a/"
+	conformanceSelectorAutheliaError           = `.notification[data-type="error"]`
+	conformanceSelectorCompletionError         = `[data-testid="openid-completion-outcome"][data-outcome="error"]`
+	conformanceElementTimeout                  = time.Second * 2
+	conformanceDriveInterval                   = time.Millisecond * 250
+	conformanceSettleTimeout                   = time.Second * 30
+	conformanceLegPatience                     = time.Second * 30
+	conformancePageTimeout                     = time.Second * 30
+	conformanceSignInAttempts                  = 3
+	conformanceConsentAttempts                 = 3
+	conformanceScreenshotLimit                 = 500 * 1024
+	conformanceCoverageBinding                 = "__autheliaConformanceCoverage"
+	conformanceCoverageScript                  = `addEventListener('beforeunload', () => { if (window.__coverage__) window.` + conformanceCoverageBinding + `(JSON.stringify(window.__coverage__)); });`
+)
+
+// ConformanceClassifyPage decides what the browser is looking at. It takes the observations rather than the page so
+// that the decision is testable without a browser. The callback wins over every stage selector, because a stage left
+// in the DOM of the document being navigated away from would otherwise be read as the current page.
+func ConformanceClassifyPage(uri string, hasFirstFactor, hasConsent, hasError bool) ConformancePageState {
+	switch {
+	case conformanceIsCallback(uri):
+		return ConformancePageCallback
+	case hasConsent:
+		return ConformancePageConsent
+	case hasFirstFactor:
+		return ConformancePageFirstFactor
+	case hasError:
+		return ConformancePageAutheliaError
+	default:
+		return ConformancePageUnknown
+	}
+}
+
+func conformanceIsCallback(uri string) bool {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(u.Path, conformanceCallbackPathPrefix)
+}
+
+// ConformanceLeg is what the driver saw over the course of one browser leg of a module's authorization flow. It is
+// recorded as the leg is driven rather than probed afterwards: [ConformanceBrowser.Drive] only returns once the flow
+// has left Authelia for the conformance suite's callback, so anything asked of the live DOM at that point is a
+// question about the callback page and not about what Authelia did. Being a plain value also makes the module
+// overrides which consume it testable without a browser.
+type ConformanceLeg struct {
+	// Index is the zero based position of this leg among the module's browser legs. Modules such as
+	// oidcc-prompt-login perform two authorization round trips, and only the second one carries the parameter under
+	// test, so an expectation about re-authentication belongs to Index 1 rather than to every leg.
+	Index int
+
+	// FirstFactor is whether Authelia presented its first factor sign in form at any point during the leg.
+	FirstFactor bool
+
+	// Consent is whether Authelia presented its OpenID Connect 1.0 consent decision stage at any point during the leg.
+	Consent bool
+
+	// Reauthentication is whether Authelia asked for the password again during the leg. This is the signal a
+	// prompt=login or max_age expectation turns on, not FirstFactor: Authelia asks on the consent decision form
+	// rather than by returning to the sign in page.
+	Reauthentication bool
+
+	// AutheliaError is whether the leg ended on Authelia reporting an error rather than reaching the callback.
+	AutheliaError bool
+
+	// ErrorURL is the page the leg ended on when AutheliaError is set. Authelia's completion view carries the error in
+	// its query, so this is what describes the error to a report.
+	ErrorURL string
+
+	// LoginScreenshot is the last page during the leg which asked for the password, captured with it entered and
+	// before it was submitted: the consent form's re-authentication step, or the sign in form.
+	LoginScreenshot string
+
+	// ErrorScreenshot is the Authelia error page the leg ended on.
+	ErrorScreenshot string
+}
+
+// ConformanceBrowser is one conformance plan's isolated browser context. Every method returns an error rather than
+// failing a test, because the driver runs on a goroutine which is not the test's.
+type ConformanceBrowser struct {
+	session   *RodSession
+	incognito *rod.Browser
+	page      *rod.Page
+	username  string
+	password  string
+
+	trace        *ConformanceTrace
+	stopCoverage func()
+}
+
+// NewConformanceBrowser returns a ConformanceBrowser with its own cookie jar.
+func NewConformanceBrowser(session *RodSession) (browser *ConformanceBrowser, err error) {
+	incognito, err := session.WebDriver.Incognito()
+	if err != nil {
+		return nil, fmt.Errorf("error creating an isolated browser context: %w", err)
+	}
+
+	page, err := incognito.Page(proto.TargetCreateTarget{})
+	if err != nil {
+		_ = incognito.Close()
+
+		return nil, fmt.Errorf("error creating a tab: %w", err)
+	}
+
+	browser = &ConformanceBrowser{
+		session:   session,
+		incognito: incognito,
+		page:      page,
+		username:  testUsername,
+		password:  testPassword,
+	}
+
+	if err = browser.collectCoverage(); err != nil {
+		_ = incognito.Close()
+
+		return nil, fmt.Errorf("error arranging for frontend coverage to be collected: %w", err)
+	}
+
+	return browser, nil
+}
+
+func (b *ConformanceBrowser) collectCoverage() (err error) {
+	if err = (proto.RuntimeAddBinding{Name: conformanceCoverageBinding}).Call(b.page); err != nil {
+		return err
+	}
+
+	if _, err = b.page.EvalOnNewDocument(conformanceCoverageScript); err != nil {
+		return err
+	}
+
+	page, cancel := b.page.WithCancel()
+
+	wait := page.EachEvent(func(e *proto.RuntimeBindingCalled) {
+		if e.Name != conformanceCoverageBinding {
+			return
+		}
+
+		if err := writeCoverage(coverageDir, e.Payload); err != nil {
+			log.Errorf("Error writing coverage: %v", err)
+		}
+	})
+
+	done := make(chan struct{})
+
+	go func() {
+		wait()
+		close(done)
+	}()
+
+	b.stopCoverage = func() {
+		cancel()
+		<-done
+	}
+
+	return nil
+}
+
+// Close disposes of the browser context. The document the browser ends on is never left, so its coverage is collected
+// directly, and any coverage still being written is finished before the context goes.
+func (b *ConformanceBrowser) Close() {
+	if b.incognito == nil {
+		return
+	}
+
+	b.session.collectCoverage(b.page.Timeout(conformancePageTimeout))
+
+	if b.stopCoverage != nil {
+		b.stopCoverage()
+	}
+
+	_ = b.incognito.Close()
+
+	b.incognito = nil
+}
+
+// ClearCookies discards every cookie in this context, for the modules whose instructions are to remove any cookies
+// received from the provider before proceeding.
+func (b *ConformanceBrowser) ClearCookies() error {
+	return proto.NetworkClearBrowserCookies{}.Call(b.page.Timeout(conformancePageTimeout))
+}
+
+func (b *ConformanceBrowser) has(selector string) bool {
+	has, _, err := b.page.Timeout(conformanceElementTimeout).Has(selector)
+
+	return err == nil && has
+}
+
+func (b *ConformanceBrowser) hasAutheliaError() bool {
+	return b.has(conformanceSelectorAutheliaError) || b.has(conformanceSelectorCompletionError)
+}
+
+func (b *ConformanceBrowser) url() string {
+	info, err := b.page.Timeout(conformancePageTimeout).Info()
+	if err != nil {
+		return ""
+	}
+
+	return info.URL
+}
+
+func (b *ConformanceBrowser) title() string {
+	info, err := b.page.Timeout(conformancePageTimeout).Info()
+	if err != nil {
+		return ""
+	}
+
+	return info.Title
+}
+
+func (b *ConformanceBrowser) settle(ctx context.Context, selector, startURL string) error {
+	deadline := time.Now().Add(conformanceSettleTimeout)
+
+	for time.Now().Before(deadline) {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
+		if !b.has(selector) || b.url() != startURL {
+			return nil
+		}
+
+		time.Sleep(conformanceDriveInterval)
+	}
+
+	return fmt.Errorf("the form at '%s' was submitted but '%s' was still on screen %s later", startURL, selector, conformanceSettleTimeout)
+}
+
+// Drive navigates to uri and works the flow through to the conformance suite's callback. It is deliberately reactive:
+// it acts on whatever page appears rather than on a script per module, so that a module added by a future conformance
+// suite release runs instead of failing.
+func (b *ConformanceBrowser) Drive(ctx context.Context, index int, uri string) (leg ConformanceLeg, err error) {
+	leg = ConformanceLeg{Index: index}
+
+	loaded := b.page.Timeout(conformancePageTimeout).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded)
+
+	if err = b.session.doNavigate(b.page.Timeout(conformancePageTimeout), uri); err != nil {
+		return leg, fmt.Errorf("error navigating to '%s': %w", uri, err)
+	}
+
+	loaded()
+
+	attempts, consents := 0, 0
+
+	progress := newConformanceProgress(conformanceLegPatience, time.Now())
+
+	for {
+		if ctx.Err() != nil {
+			return leg, fmt.Errorf("the flow beginning at '%s' did not reach the conformance callback, the last page was '%s': %w", uri, b.url(), ctx.Err())
+		}
+
+		pageURL := b.url()
+
+		switch ConformanceClassifyPage(pageURL, b.has(conformanceSelectorFirstFactor), b.has(conformanceSelectorConsent), b.hasAutheliaError()) {
+		case ConformancePageCallback:
+			return leg, nil
+		case ConformancePageFirstFactor:
+			leg.FirstFactor = true
+
+			if attempts++; attempts > conformanceSignInAttempts {
+				return leg, fmt.Errorf("the sign in form at '%s' was still present after %d attempts", pageURL, conformanceSignInAttempts)
+			}
+
+			var screenshot string
+
+			if screenshot, err = b.submitSignIn(ctx, pageURL, attempts); err != nil {
+				return leg, err
+			}
+
+			leg.LoginScreenshot = conformanceLatest(leg.LoginScreenshot, screenshot)
+
+			progress.observe(time.Now(), pageURL, true)
+		case ConformancePageConsent:
+			leg.Consent = true
+
+			if consents++; consents > conformanceConsentAttempts {
+				return leg, fmt.Errorf("the consent decision form at '%s' was still present after %d attempts", pageURL, conformanceConsentAttempts)
+			}
+
+			var (
+				reauthentication bool
+				screenshot       string
+			)
+
+			if reauthentication, screenshot, err = b.submitConsent(ctx, pageURL, consents); err != nil {
+				return leg, err
+			}
+
+			leg.Reauthentication = leg.Reauthentication || reauthentication
+			leg.LoginScreenshot = conformanceLatest(leg.LoginScreenshot, screenshot)
+
+			progress.observe(time.Now(), pageURL, true)
+		case ConformancePageAutheliaError:
+			leg.AutheliaError, leg.ErrorURL, leg.ErrorScreenshot = true, pageURL, b.screenshot()
+
+			return leg, nil
+		case ConformancePageUnknown:
+			if progress.observe(time.Now(), pageURL, false) {
+				return leg, fmt.Errorf("the flow beginning at '%s' stopped at '%s' (%q), a page the driver has no action for, and it did not change for %s",
+					uri, pageURL, b.title(), conformanceLegPatience)
+			}
+
+			time.Sleep(conformanceDriveInterval)
+		}
+	}
+}
+
+func (b *ConformanceBrowser) signIn() (screenshot string, err error) {
+	if err = b.input(conformanceSelectorUsername, b.username); err != nil {
+		return "", err
+	}
+
+	if err = b.input(conformanceSelectorPassword, b.password); err != nil {
+		return "", err
+	}
+
+	screenshot = b.screenshot()
+
+	return screenshot, b.click(conformanceSelectorSignIn)
+}
+
+func (b *ConformanceBrowser) screenshot() string {
+	uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) {
+		req := &proto.PageCaptureScreenshot{Format: format}
+
+		if quality != 0 {
+			req.Quality = &quality
+		}
+
+		return b.page.Timeout(conformancePageTimeout).Screenshot(false, req)
+	})
+	if err != nil {
+		log.Warnf("Conformance driver could not capture a screenshot of '%s': %v", b.url(), err)
+
+		return ""
+	}
+
+	return uri
+}
+
+func conformanceScreenshotDataURI(capture func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error)) (uri string, err error) {
+	for _, attempt := range []struct {
+		format  proto.PageCaptureScreenshotFormat
+		quality int
+		mime    string
+	}{
+		{proto.PageCaptureScreenshotFormatPng, 0, "image/png"},
+		{proto.PageCaptureScreenshotFormatJpeg, 80, "image/jpeg"},
+		{proto.PageCaptureScreenshotFormatJpeg, 50, "image/jpeg"},
+	} {
+		var data []byte
+
+		if data, err = capture(attempt.format, attempt.quality); err != nil {
+			return "", err
+		}
+
+		if len(data) <= conformanceScreenshotLimit {
+			return "data:" + attempt.mime + ";base64," + base64.StdEncoding.EncodeToString(data), nil
+		}
+	}
+
+	return "", fmt.Errorf("the screenshot does not fit the conformance suite's %dKB upload limit in any format", conformanceScreenshotLimit/1024)
+}
+
+func conformanceLatest(current, next string) string {
+	if next == "" {
+		return current
+	}
+
+	return next
+}
+
+func (b *ConformanceBrowser) input(selector, value string) (err error) {
+	element, err := b.page.Timeout(elementLocateTimeout).Element(selector)
+	if err != nil {
+		return fmt.Errorf("error locating '%s': %w", selector, err)
+	}
+
+	if err = element.SelectAllText(); err != nil {
+		return fmt.Errorf("error selecting the text of '%s': %w", selector, err)
+	}
+
+	if err = element.Input(value); err != nil {
+		return fmt.Errorf("error typing into '%s': %w", selector, err)
+	}
+
+	return nil
+}
+
+func (b *ConformanceBrowser) click(selector string) (err error) {
+	element, err := b.page.Timeout(elementLocateTimeout).Element(selector)
+	if err != nil {
+		return fmt.Errorf("error locating '%s': %w", selector, err)
+	}
+
+	if err = element.Click(proto.InputMouseButtonLeft, 1); err != nil {
+		return fmt.Errorf("error clicking '%s': %w", selector, err)
+	}
+
+	return nil
+}
+
+type conformanceProgress struct {
+	patience time.Duration
+	url      string
+	deadline time.Time
+}
+
+func newConformanceProgress(patience time.Duration, now time.Time) *conformanceProgress {
+	return &conformanceProgress{patience: patience, deadline: now.Add(patience)}
+}
+
+func (p *conformanceProgress) observe(now time.Time, url string, recognized bool) (stalled bool) {
+	if url != p.url {
+		p.url, p.deadline = url, now.Add(p.patience)
+
+		return false
+	}
+
+	if recognized {
+		p.deadline = now.Add(p.patience)
+
+		return false
+	}
+
+	return now.After(p.deadline)
+}
+
+func (b *ConformanceBrowser) submitSignIn(ctx context.Context, pageURL string, attempt int) (screenshot string, err error) {
+	b.trace.Logf("Signing in at '%s' (attempt %d)", pageURL, attempt)
+
+	if screenshot, err = b.signIn(); err != nil {
+		return "", fmt.Errorf("error signing in at '%s': %w", pageURL, err)
+	}
+
+	if err = b.settle(ctx, conformanceSelectorFirstFactor, pageURL); err != nil {
+		return "", fmt.Errorf("error signing in at '%s': %w", pageURL, err)
+	}
+
+	return screenshot, nil
+}
+
+func (b *ConformanceBrowser) submitConsent(ctx context.Context, pageURL string, attempt int) (reauthentication bool, screenshot string, err error) {
+	b.trace.Logf("Accepting the consent decision form at '%s' (attempt %d)", pageURL, attempt)
+
+	if err = b.click(conformanceSelectorConsentAccept); err != nil {
+		return false, "", fmt.Errorf("error accepting consent at '%s': %w", pageURL, err)
+	}
+
+	if reauthentication, err = b.awaitConsentAccepted(ctx, pageURL); err != nil {
+		return false, "", err
+	}
+
+	if !reauthentication {
+		return false, "", nil
+	}
+
+	b.trace.Logf("Answering the consent form's re-authentication step at '%s'", pageURL)
+
+	if err = b.input(conformanceSelectorConsentPassword, b.password); err != nil {
+		return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err)
+	}
+
+	screenshot = b.screenshot()
+
+	if err = b.click(conformanceSelectorConsentAuthenticate); err != nil {
+		return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err)
+	}
+
+	if err = b.settle(ctx, conformanceSelectorConsentReauthentication, pageURL); err != nil {
+		return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err)
+	}
+
+	return true, screenshot, nil
+}
+
+func (b *ConformanceBrowser) awaitConsentAccepted(ctx context.Context, pageURL string) (reauthentication bool, err error) {
+	deadline := time.Now().Add(conformanceSettleTimeout)
+
+	for time.Now().Before(deadline) {
+		if ctx.Err() != nil {
+			return false, ctx.Err()
+		}
+
+		reauthentication, done := ConformanceConsentProgress(b.has(conformanceSelectorConsentReauthentication), b.has(conformanceSelectorConsent), b.url() != pageURL)
+		if done {
+			return reauthentication, nil
+		}
+
+		time.Sleep(conformanceDriveInterval)
+	}
+
+	return false, fmt.Errorf("the consent decision form at '%s' was accepted but neither left nor asked for the password %s later", pageURL, conformanceSettleTimeout)
+}
+
+// ConformanceConsentProgress reports what accepting the consent decision achieved.
+func ConformanceConsentProgress(hasReauthentication, hasStage, urlChanged bool) (reauthentication, done bool) {
+	switch {
+	case hasReauthentication:
+		return true, true
+	case !hasStage || urlChanged:
+		return false, true
+	default:
+		return false, false
+	}
+}

internal/suites/conformance_browser_driver_test.go

@@ -0,0 +1,372 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/launcher"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceBrowser_Drive(t *testing.T) {
+	driver := newConformanceTestBrowser(t)
+	portal := conformanceFakePortal(t)
+
+	drive := func(t *testing.T, index int, path string) ConformanceLeg {
+		t.Helper()
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+		defer cancel()
+
+		leg, err := driver.Drive(ctx, index, portal.URL+path)
+		require.NoError(t, err)
+
+		return leg
+	}
+
+	t.Run("ShouldSignInThenAnswerTheConsentFormsReauthentication", func(t *testing.T) {
+		leg := drive(t, 1, "/authorize-reauthenticate")
+
+		assert.Equal(t, 1, leg.Index)
+		assert.True(t, leg.FirstFactor)
+		assert.True(t, leg.Consent)
+		assert.True(t, leg.Reauthentication)
+		assert.False(t, leg.AutheliaError)
+		assert.True(t, strings.HasPrefix(leg.LoginScreenshot, "data:image/png;base64,"), "the password step is captured")
+		assert.Empty(t, leg.ErrorScreenshot)
+	})
+
+	t.Run("ShouldAcceptConsentWhichAsksForNoPassword", func(t *testing.T) {
+		leg := drive(t, 0, "/consent")
+
+		assert.True(t, leg.Consent)
+		assert.False(t, leg.Reauthentication)
+		assert.False(t, leg.FirstFactor)
+		assert.Empty(t, leg.LoginScreenshot)
+	})
+
+	t.Run("ShouldDriveALegWhichStartsOnTheLastLegsCallback", func(t *testing.T) {
+		drive(t, 0, "/consent")
+
+		leg := drive(t, 1, "/error")
+
+		assert.True(t, leg.AutheliaError)
+	})
+
+	t.Run("ShouldEndTheLegOnAnAutheliaErrorPage", func(t *testing.T) {
+		leg := drive(t, 0, "/error")
+
+		assert.True(t, leg.AutheliaError)
+		assert.Equal(t, portal.URL+"/error", leg.ErrorURL)
+		assert.True(t, strings.HasPrefix(leg.ErrorScreenshot, "data:image/png;base64,"), "the error page is captured")
+	})
+
+	t.Run("ShouldHandCoverageOverAsThePageIsLeft", func(t *testing.T) {
+		before, _ := os.ReadDir(coverageDir)
+
+		drive(t, 0, "/coverage")
+
+		assert.Eventually(t, func() bool {
+			after, _ := os.ReadDir(coverageDir)
+
+			return len(after) > len(before)
+		}, time.Second*5, time.Millisecond*50)
+	})
+
+	t.Run("ShouldClearTheContextsCookies", func(t *testing.T) {
+		drive(t, 0, "/authorize")
+
+		cookies, err := driver.page.Cookies(nil)
+		require.NoError(t, err)
+		require.NotEmpty(t, cookies)
+
+		require.NoError(t, driver.ClearCookies())
+
+		cookies, err = driver.page.Cookies(nil)
+		require.NoError(t, err)
+		assert.Empty(t, cookies)
+	})
+
+	t.Run("ShouldReportANavigationWhichFailed", func(t *testing.T) {
+		_, err := driver.Drive(context.Background(), 0, "http://127.0.0.1:1/")
+
+		assert.ErrorContains(t, err, "error navigating to 'http://127.0.0.1:1/'")
+	})
+
+	t.Run("ShouldStopWhenTheContextHasExpired", func(t *testing.T) {
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+
+		_, err := driver.Drive(ctx, 0, portal.URL+"/error")
+
+		assert.ErrorContains(t, err, "did not reach the conformance callback")
+	})
+}
+
+func TestConformanceRunner_VisitURLsDrivesEachLeg(t *testing.T) {
+	driver := newConformanceTestBrowser(t)
+	portal := conformanceFakePortal(t)
+
+	var (
+		mu      sync.Mutex
+		visited []string
+	)
+
+	visit := func(status int) http.HandlerFunc {
+		return func(w http.ResponseWriter, r *http.Request) {
+			mu.Lock()
+
+			visited = append(visited, r.URL.Query().Get("url"))
+			mu.Unlock()
+
+			w.WriteHeader(status)
+		}
+	}
+
+	t.Run("ShouldVisitALegOnceItIsDriven", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{"/api/runner/browser/m1/visit": visit(http.StatusNoContent)})
+
+		runner := NewConformanceRunner(client, driver, "plan1", nil)
+		runner.trace = NewConformanceTrace(true)
+
+		legs := &conformanceLegs{visited: map[string]bool{}}
+
+		progressed, err := runner.visitURLs(context.Background(), "m1", ConformanceOverride{Screenshot: ConformanceScreenshotErrorPage}, legs, []string{portal.URL + "/error"})
+
+		require.NoError(t, err)
+		assert.True(t, progressed)
+		assert.Equal(t, portal.URL+"/error", legs.errorURL)
+		assert.True(t, strings.HasPrefix(legs.screenshot, "data:image/png;base64,"))
+		assert.Equal(t, []string{portal.URL + "/error"}, visited)
+		assert.Len(t, runner.trace.Lines(), 2, "the leg is traced as it starts and as it ends")
+	})
+
+	t.Run("ShouldNotVisitALegWhoseAssertionFailed", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{})
+
+		legs := &conformanceLegs{visited: map[string]bool{}}
+
+		_, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{Assert: conformanceAssertErrorPage}, legs, []string{portal.URL + "/consent"})
+
+		assert.EqualError(t, err, "expected Authelia to show an error page rather than redirect to the client but it did not")
+	})
+
+	t.Run("ShouldReportAVisitWhichFailed", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{"/api/runner/browser/m1/visit": visit(http.StatusInternalServerError)})
+
+		legs := &conformanceLegs{visited: map[string]bool{}}
+
+		_, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{}, legs, []string{portal.URL + "/consent-reauthenticate"})
+
+		assert.ErrorContains(t, err, "expected status 204 but got 500")
+	})
+
+	t.Run("ShouldReportALegWhichCouldNotBeDriven", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{})
+
+		legs := &conformanceLegs{visited: map[string]bool{}}
+
+		_, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{}, legs, []string{"http://127.0.0.1:1/"})
+
+		assert.ErrorContains(t, err, "error navigating")
+	})
+}
+
+func TestConformanceBrowser_AfterClose(t *testing.T) {
+	driver := newConformanceTestBrowser(t)
+
+	driver.Close()
+
+	assert.Empty(t, driver.url())
+	assert.Empty(t, driver.title())
+	assert.False(t, driver.has(conformanceSelectorFirstFactor))
+	assert.Empty(t, driver.screenshot())
+
+	assert.NotPanics(t, driver.Close, "closing twice is harmless")
+}
+
+func TestConformanceRunner_WithABrowser(t *testing.T) {
+	driver := newConformanceTestBrowser(t)
+
+	t.Run("ShouldClearCookiesForAModuleWhichAsksForIt", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+			"/api/runner":               conformanceReply(http.StatusCreated, `{"id":"m1"}`),
+			"/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`),
+			"/api/info/m1":              conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`),
+		})
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		var outcomes []ConformanceOutcome
+
+		for outcome := range NewConformanceRunner(client, driver, "plan1", []ConformancePlanModule{{TestModule: "oidcc-display-page"}}).Run(ctx) {
+			outcomes = append(outcomes, outcome)
+		}
+
+		require.Len(t, outcomes, 1)
+		require.NoError(t, outcomes[0].Err)
+		assert.Equal(t, "PASSED", outcomes[0].Result)
+	})
+
+	t.Run("ShouldReportALegWhichCouldNotBeDriven", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+			"/api/runner/browser/m1": conformanceReply(http.StatusOK, `{"urls":["http://127.0.0.1:1/"]}`),
+		})
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err := NewConformanceRunner(client, driver, "plan1", nil).interact(ctx, "m1", ConformanceOverride{})
+
+		assert.ErrorContains(t, err, "error navigating to 'http://127.0.0.1:1/'")
+	})
+}
+
+func conformanceFakePortal(t *testing.T) *httptest.Server {
+	t.Helper()
+
+	const callback = "/test/a/alias/callback?code=abc"
+
+	page := func(w http.ResponseWriter, body string) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = fmt.Fprintf(w, "<!DOCTYPE html><html><head><title>Login - Authelia</title></head><body>%s</body></html>", body)
+	}
+
+	mux := http.NewServeMux()
+
+	for _, variant := range []struct {
+		suffix           string
+		reauthentication bool
+	}{
+		{"", false},
+		{"-reauthenticate", true},
+	} {
+		mux.HandleFunc("/authorize"+variant.suffix, func(w http.ResponseWriter, r *http.Request) {
+			http.Redirect(w, r, "/login"+variant.suffix, http.StatusFound)
+		})
+
+		mux.HandleFunc("/login"+variant.suffix, func(w http.ResponseWriter, _ *http.Request) {
+			page(w, fmt.Sprintf(`<div id="first-factor-stage">
+<input id="username-textfield"><input id="password-textfield" type="password">
+<button id="sign-in-button" onclick="signIn()">Sign in</button></div>
+<script>
+document.cookie = 'authelia_session=session; path=/';
+
+function signIn() {
+  if (document.getElementById('username-textfield').value === %q &&
+    document.getElementById('password-textfield').value === %q) location.href = %q;
+}
+</script>`, testUsername, testPassword, "/consent"+variant.suffix))
+		})
+
+		mux.HandleFunc("/consent"+variant.suffix, func(w http.ResponseWriter, _ *http.Request) {
+			page(w, fmt.Sprintf(`<div id="openid-consent-decision-stage">
+<button id="openid-consent-accept" onclick="accept()">Accept</button></div>
+<script>
+function accept() {
+  if (%t) {
+    const stage = document.getElementById('openid-consent-decision-stage');
+    stage.innerHTML = '<div id="openid-consent-prompt-login"><input id="password-textfield" type="password"></div>' +
+      '<button id="openid-consent-authenticate" onclick="authenticate()">Authenticate</button>';
+
+    return;
+  }
+
+  location.href = %q;
+}
+
+function authenticate() {
+  if (document.querySelector('#openid-consent-prompt-login #password-textfield').value === %q) location.href = %q;
+}
+</script>`, variant.reauthentication, callback, testPassword, callback))
+		})
+	}
+
+	mux.HandleFunc("/error", func(w http.ResponseWriter, _ *http.Request) {
+		page(w, `<div data-testid="openid-completion-outcome" data-outcome="error">An error occurred processing the request</div>`)
+	})
+
+	mux.HandleFunc("/coverage", func(w http.ResponseWriter, _ *http.Request) {
+		page(w, fmt.Sprintf(`<script>window.__coverage__ = {"/node/src/app/src/index.tsx": {"s": {"0": 1}}};
+setTimeout(() => { location.href = %q; }, 100);</script>`, callback))
+	})
+
+	mux.HandleFunc("/test/a/alias/callback", func(w http.ResponseWriter, _ *http.Request) {
+		page(w, "callback")
+	})
+
+	server := httptest.NewServer(mux)
+
+	t.Cleanup(server.Close)
+
+	return server
+}
+
+func newConformanceTestBrowser(t *testing.T) *ConformanceBrowser {
+	t.Helper()
+
+	if testing.Short() {
+		t.Skip("skipping browser test in short mode")
+	}
+
+	preserveCoverageDir(t)
+
+	path, err := GetBrowserPath()
+	require.NoError(t, err)
+
+	l := launcher.New().Bin(path).Headless(true)
+	t.Cleanup(l.Cleanup)
+
+	browser := rod.New().ControlURL(l.MustLaunch()).MustConnect()
+
+	t.Cleanup(func() { _ = browser.Close() })
+
+	driver, err := NewConformanceBrowser(&RodSession{WebDriver: browser})
+	require.NoError(t, err)
+
+	t.Cleanup(driver.Close)
+
+	return driver
+}
+
+func preserveCoverageDir(t *testing.T) {
+	t.Helper()
+
+	existed := map[string]bool{}
+
+	entries, err := os.ReadDir(coverageDir)
+	dirExisted := err == nil
+
+	for _, entry := range entries {
+		existed[entry.Name()] = true
+	}
+
+	t.Cleanup(func() {
+		entries, _ := os.ReadDir(coverageDir)
+
+		for _, entry := range entries {
+			if !existed[entry.Name()] {
+				_ = os.Remove(filepath.Join(coverageDir, entry.Name()))
+			}
+		}
+
+		if !dirExisted {
+			_ = os.Remove(coverageDir)
+		}
+	})
+}

internal/suites/conformance_browser_test.go

@@ -0,0 +1,171 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/go-rod/rod/lib/proto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceClassifyPage(t *testing.T) {
+	testCases := []struct {
+		name                                string
+		url                                 string
+		firstFactor, consent, autheliaError bool
+		expected                            ConformancePageState
+	}{
+		{
+			"ShouldDetectTheCallbackByPath",
+			"https://conformance.example.com:8443/test/a/conformance-basic-authelia1a2b3c4d/callback?code=abc",
+			false, false, false,
+			ConformancePageCallback,
+		},
+		{
+			"ShouldPreferTheCallbackOverAnyStageStillInTheDOM",
+			"https://conformance.example.com:8443/test/a/alias/callback",
+			true, false, false,
+			ConformancePageCallback,
+		},
+		{
+			"ShouldNotMistakeAnAuthorizationRequestCarryingTheCallbackForTheCallback",
+			"https://login.example.com:8080/api/oidc/authorization?client_id=abc&redirect_uri=https://conformance.example.com:8443/test/a/alias/callback",
+			false, false, false,
+			ConformancePageUnknown,
+		},
+		{
+			"ShouldDetectTheFirstFactorPage",
+			"https://login.example.com:8080/",
+			true, false, false,
+			ConformancePageFirstFactor,
+		},
+		{
+			"ShouldDetectTheConsentPage",
+			"https://login.example.com:8080/consent/openid/decision",
+			false, true, false,
+			ConformancePageConsent,
+		},
+		{
+			"ShouldDetectAnAutheliaError",
+			"https://login.example.com:8080/",
+			false, false, true,
+			ConformancePageAutheliaError,
+		},
+		{
+			"ShouldPreferTheFirstFactorPageOverACoexistingErrorToast",
+			"https://login.example.com:8080/",
+			true, false, true,
+			ConformancePageFirstFactor,
+		},
+		{
+			"ShouldPreferTheConsentPageOverACoexistingErrorToast",
+			"https://login.example.com:8080/consent/openid/decision",
+			false, true, true,
+			ConformancePageConsent,
+		},
+		{
+			"ShouldReportUnknownWhenNothingMatches",
+			"https://login.example.com:8080/",
+			false, false, false,
+			ConformancePageUnknown,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, ConformanceClassifyPage(tc.url, tc.firstFactor, tc.consent, tc.autheliaError))
+		})
+	}
+}
+
+func TestConformanceConsentProgress(t *testing.T) {
+	t.Run("ShouldTreatTheRevealedPasswordFieldAsTheReauthenticationStep", func(t *testing.T) {
+		reauthentication, done := ConformanceConsentProgress(true, true, false)
+		assert.True(t, reauthentication)
+		assert.True(t, done)
+	})
+
+	t.Run("ShouldTreatTheStageClearingAsTheFlowLeaving", func(t *testing.T) {
+		reauthentication, done := ConformanceConsentProgress(false, false, false)
+		assert.False(t, reauthentication)
+		assert.True(t, done)
+	})
+
+	t.Run("ShouldTreatAChangedURLAsTheFlowLeaving", func(t *testing.T) {
+		reauthentication, done := ConformanceConsentProgress(false, true, true)
+		assert.False(t, reauthentication)
+		assert.True(t, done)
+	})
+
+	t.Run("ShouldKeepWaitingWhileTheFormIsStillOnItsDecisionStep", func(t *testing.T) {
+		reauthentication, done := ConformanceConsentProgress(false, true, false)
+		assert.False(t, reauthentication)
+		assert.False(t, done)
+	})
+
+	t.Run("ShouldPreferTheReauthenticationStepOverTheFormLeaving", func(t *testing.T) {
+		reauthentication, done := ConformanceConsentProgress(true, false, true)
+		assert.True(t, reauthentication)
+		assert.True(t, done)
+	})
+}
+
+func TestConformanceScreenshotDataURI(t *testing.T) {
+	small, large := []byte("small"), make([]byte, conformanceScreenshotLimit+1)
+
+	t.Run("ShouldPreferAPNGWhichFits", func(t *testing.T) {
+		uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) {
+			assert.Equal(t, proto.PageCaptureScreenshotFormatPng, format)
+
+			return small, nil
+		})
+
+		require.NoError(t, err)
+		assert.Equal(t, "data:image/png;base64,"+base64.StdEncoding.EncodeToString(small), uri)
+	})
+
+	t.Run("ShouldFallBackToAJPEGWhenThePNGIsTooLarge", func(t *testing.T) {
+		var asked []string
+
+		uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) {
+			asked = append(asked, fmt.Sprintf("%s:%d", format, quality))
+
+			if format == proto.PageCaptureScreenshotFormatPng {
+				return large, nil
+			}
+
+			return small, nil
+		})
+
+		require.NoError(t, err)
+		assert.Equal(t, "data:image/jpeg;base64,"+base64.StdEncoding.EncodeToString(small), uri)
+		assert.Equal(t, []string{"png:0", "jpeg:80"}, asked)
+	})
+
+	t.Run("ShouldFailWhenNothingFits", func(t *testing.T) {
+		_, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) {
+			return large, nil
+		})
+
+		assert.ErrorContains(t, err, "500KB")
+	})
+
+	t.Run("ShouldFailWhenTheCaptureFails", func(t *testing.T) {
+		_, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) {
+			return nil, errors.New("target closed")
+		})
+
+		assert.ErrorContains(t, err, "target closed")
+	})
+}
+
+func TestConformanceIsCallbackRejectsAnUnparsableURL(t *testing.T) {
+	assert.False(t, conformanceIsCallback("%zz/test/a/alias/callback"))
+}

internal/suites/conformance_compose_test.go

@@ -0,0 +1,162 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.yaml.in/yaml/v4"
+)
+
+func TestOIDCConformanceComposeSuppliesOIDCLoginPlaceholders(t *testing.T) {
+	data, err := os.ReadFile("OIDCConformance/compose.yml")
+	require.NoError(t, err)
+
+	compose := struct {
+		Services map[string]struct {
+			Environment map[string]string `yaml:"environment"`
+		} `yaml:"services"`
+	}{}
+
+	require.NoError(t, yaml.Unmarshal(data, &compose))
+
+	server, ok := compose.Services["conformance-server"]
+	require.True(t, ok, "the compose file must define a conformance-server service")
+
+	for _, name := range []string{"OIDC_GOOGLE_CLIENTID", "OIDC_GOOGLE_SECRET", "OIDC_GITLAB_CLIENTID", "OIDC_GITLAB_SECRET"} {
+		value, ok := server.Environment[name]
+
+		assert.Truef(t, ok, "%s must be set: the image's entrypoint interpolates it unconditionally, so leaving it unset overrides the suite's own default with an empty system property", name)
+		assert.NotEmptyf(t, value, "%s must not be empty: Spring rejects an empty client id while building OAuth2ClientProperties, which fails the application context before devmode is consulted", name)
+	}
+}
+
+func TestOIDCConformanceComposeMongoDB(t *testing.T) {
+	type service struct {
+		Environment map[string]string `yaml:"environment"`
+		DependsOn   []string          `yaml:"depends_on"`
+	}
+
+	read := func(t *testing.T, path string) map[string]service {
+		t.Helper()
+
+		data, err := os.ReadFile(path)
+		require.NoError(t, err)
+
+		compose := struct {
+			Services map[string]service `yaml:"services"`
+		}{}
+
+		require.NoError(t, yaml.Unmarshal(data, &compose))
+
+		return compose.Services
+	}
+
+	t.Run("ShouldLeaveTheDatabaseToItsOwnComposeFile", func(t *testing.T) {
+		services := read(t, "OIDCConformance/compose.yml")
+
+		assert.NotContains(t, services, "conformance-mongodb")
+		assert.NotContains(t, services["conformance-server"].DependsOn, "conformance-mongodb")
+		assert.Equal(t, "${"+oidcConformanceMongoDBHostEnv+":-conformance-mongodb}", services["conformance-server"].Environment["MONGODB_HOST"],
+			"the server uses the database the suite was pointed at, and its own otherwise")
+	})
+
+	t.Run("ShouldRunTheDatabaseFromItsOwnComposeFile", func(t *testing.T) {
+		services := read(t, oidcConformanceMongoDBComposeFile)
+
+		assert.Contains(t, services, "conformance-mongodb")
+		assert.Contains(t, services["conformance-server"].DependsOn, "conformance-mongodb")
+	})
+}
+
+func TestOIDCConformanceComposeFiles(t *testing.T) {
+	assert.Contains(t, oidcConformanceComposeFiles(""), "internal/suites/"+oidcConformanceMongoDBComposeFile)
+	assert.Contains(t, oidcConformanceLogServices(""), "conformance-mongodb")
+
+	assert.NotContains(t, oidcConformanceComposeFiles("mongodb.example.com"), "internal/suites/"+oidcConformanceMongoDBComposeFile,
+		"a database the suite is pointed at replaces its own")
+	assert.NotContains(t, oidcConformanceLogServices("mongodb.example.com"), "conformance-mongodb", "there is no container to print the logs of")
+
+	for _, host := range []string{"", "mongodb.example.com"} {
+		assert.Contains(t, oidcConformanceComposeFiles(host), "internal/suites/OIDCConformance/compose.yml")
+		assert.Contains(t, oidcConformanceLogServices(host), "conformance-server")
+	}
+}
+
+func TestOIDCConformanceServedByThePortal(t *testing.T) {
+	base, err := url.ParseRequestURI(oidcConformanceBaseURL)
+	require.NoError(t, err)
+
+	conf, err := os.ReadFile("example/compose/nginx/portal/nginx.conf")
+	require.NoError(t, err)
+
+	nginx := string(conf)
+
+	t.Run("ShouldListenOnThePortTheBaseURLUses", func(t *testing.T) {
+		assert.Containsf(t, nginx, "listen "+base.Port()+" ssl;",
+			"the portal must terminate TLS on %s, which is the port %s resolves to", base.Port(), oidcConformanceBaseURL)
+	})
+
+	t.Run("ShouldServeTheConformanceHost", func(t *testing.T) {
+		assert.Equal(t, 3, strings.Count(nginx, `server_name ~^conformance\.example([0-9])*\.com$;`),
+			"one server block per listener the published nginx image exposed")
+		assert.Equal(t, 3, strings.Count(nginx, "set $upstream_endpoint http://conformance-server:8080;"),
+			"the portal proxies straight to the server now that the published nginx image is gone")
+	})
+
+	t.Run("ShouldExposeTheMutualTLSAndFAPIListeners", func(t *testing.T) {
+		// 8444 and 8445 carry no traffic yet, but a profile that reaches for them should find them rather than
+		// fail part way through a plan, and they are the reason the published image existed at all.
+		assert.Contains(t, nginx, "listen 8444 ssl;")
+		assert.Contains(t, nginx, "listen 8445 ssl;")
+
+		assert.Equal(t, 2, strings.Count(nginx, "ssl_verify_client optional_no_ca;"),
+			"the mutual TLS listeners ask for a client certificate but leave verifying it to the suite")
+		assert.Contains(t, nginx, "ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384;",
+			"8445 narrows the ciphers to the FAPI 2.0 Final set")
+	})
+
+	t.Run("ShouldReproduceWhatThePublishedImageSet", func(t *testing.T) {
+		// The suite depends on each of these: large headers carry its request objects, the long timeouts outlast
+		// requests that block up to 60s on a callback, and it rebuilds its own base URL from the forwarded headers.
+		for _, directive := range []string{
+			"client_header_buffer_size 32k;",
+			"large_client_header_buffers 4 32k;",
+			"proxy_read_timeout 120s;",
+			"proxy_send_timeout 120s;",
+			"proxy_set_header  X-Forwarded-Host $host;",
+			"proxy_set_header  X-Forwarded-Port $server_port;",
+			"proxy_set_header  X-Ssl-Cert $ssl_client_cert;",
+		} {
+			assert.Equalf(t, 3, strings.Count(nginx, directive),
+				"the published nginx image set %s on every listener and the suite relies on it", directive)
+		}
+
+		// Only the listener that does not ask for a client certificate tells the suite so.
+		assert.Equal(t, 1, strings.Count(nginx, "proxy_set_header  X-Test-Mtls-Called-On-Wrong-Host $mtls_wrong_host;"))
+		assert.Equal(t, 1, strings.Count(nginx, "ssl_verify_client off;"))
+	})
+
+	t.Run("ShouldResolveTheConformanceHostToThePortal", func(t *testing.T) {
+		entries := map[string]string{}
+
+		for _, entry := range HostEntries() {
+			entries[entry.Domain] = entry.IP
+		}
+
+		require.Contains(t, entries, base.Hostname())
+		assert.Equal(t, entries["login.example.com"], entries[base.Hostname()],
+			"the conformance host is served by the portal, so it resolves where every other portal domain does")
+	})
+
+	t.Run("ShouldRunThePortal", func(t *testing.T) {
+		assert.Contains(t, oidcConformanceComposeFiles(""), "internal/suites/example/compose/nginx/portal/compose.yml")
+	})
+}

internal/suites/conformance_diagnostics.go

@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+const (
+	conformanceDiagnosticEntries = 20
+	conformanceDiagnosticValue   = 2000
+)
+
+var (
+	conformanceDiagnosticResults = map[string]bool{"FAILURE": true, "WARNING": true, "REVIEW": true}
+	conformanceDiagnosticSkip    = map[string]bool{
+		"msg": true, "result": true, "src": true, "requirements": true,
+		"_id": true, "testId": true, "testOwner": true, "time": true,
+		"blockId": true, "updatedAt": true, "img": true, "upload": true,
+	}
+)
+
+// ConformanceDiagnostics renders the graded entries of a module's log for a failure message. Each entry keeps its
+// condition's own arguments, which is where a conformance failure actually explains itself: the message says a claim
+// was missing, the arguments say which one.
+func ConformanceDiagnostics(entries []ConformanceLogEntry) string {
+	builder := &strings.Builder{}
+
+	reported := 0
+
+	for _, entry := range entries {
+		if !conformanceDiagnosticResults[entry.Result] {
+			continue
+		}
+
+		if reported == conformanceDiagnosticEntries {
+			fmt.Fprintf(builder, "\n  ... further entries omitted, see the exported plan archive for the complete log")
+
+			break
+		}
+
+		reported++
+
+		fmt.Fprintf(builder, "\n  [%s] %s", entry.Result, entry.Msg)
+
+		if entry.Src != "" {
+			fmt.Fprintf(builder, "\n    condition: %s", entry.Src)
+		}
+
+		if requirements := conformanceDiagnosticField(entry.Fields["requirements"]); requirements != "" {
+			fmt.Fprintf(builder, "\n    requirements: %s", requirements)
+		}
+
+		for _, name := range conformanceDiagnosticNames(entry.Fields) {
+			fmt.Fprintf(builder, "\n    %s: %s", name, conformanceDiagnosticField(entry.Fields[name]))
+		}
+	}
+
+	if reported == 0 {
+		return ""
+	}
+
+	return builder.String()
+}
+
+func conformanceDiagnosticNames(fields map[string]any) (names []string) {
+	for name := range fields {
+		if conformanceDiagnosticSkip[name] {
+			continue
+		}
+
+		names = append(names, name)
+	}
+
+	sort.Strings(names)
+
+	return names
+}
+
+func conformanceDiagnosticField(value any) string {
+	if value == nil {
+		return ""
+	}
+
+	rendered, ok := value.(string)
+
+	if !ok {
+		data, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Sprintf("%v", value)
+		}
+
+		rendered = string(data)
+	}
+
+	if len(rendered) > conformanceDiagnosticValue {
+		return rendered[:conformanceDiagnosticValue] + fmt.Sprintf("... (%d bytes truncated)", len(rendered)-conformanceDiagnosticValue)
+	}
+
+	return rendered
+}
+
+// ConformanceAcceptedWarningReport renders a module's accepted WARNING for its subtest's log: why it is accepted,
+// followed by the graded entries of the module's log, so the warning which was accepted is visible rather than only
+// the fact that one was.
+func ConformanceAcceptedWarningReport(module, status, reason string, entries []ConformanceLogEntry) string {
+	report := fmt.Sprintf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s.", module, status, reason)
+
+	if detail := ConformanceDiagnostics(entries); detail != "" {
+		return report + " Conformance log entries:" + detail
+	}
+
+	return report + " The module logged no warning entries."
+}

internal/suites/conformance_diagnostics_test.go

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceDiagnostics(t *testing.T) {
+	scopeFailure := `[
+	  {"result":"SUCCESS","src":"CallUserInfoEndpoint","msg":"Userinfo endpoint response was 200"},
+	  {"result":"INFO","src":"ExtractUserInfo","msg":"Extracted userinfo"},
+	  {"result":"WARNING","src":"VerifyScopesReturnedInUserInfoClaims","msg":"'claims' in userinfo doesn't contain all scope items of scope in authorization request","requirements":["OIDCC-5.4"],
+	   "expected_scope_items":["name","given_name","birthdate"],"actual_scope_items":["name"],"missing_items":["given_name","birthdate"],
+	   "_id":"abc","testId":"t1","testOwner":"anon","time":1789021616378}
+	]`
+
+	var entries []ConformanceLogEntry
+
+	require.NoError(t, json.Unmarshal([]byte(scopeFailure), &entries))
+
+	out := ConformanceDiagnostics(entries)
+
+	t.Run("ShouldReportTheGradedEntry", func(t *testing.T) {
+		assert.Contains(t, out, "[WARNING]")
+		assert.Contains(t, out, "doesn't contain all scope items")
+		assert.Contains(t, out, "condition: VerifyScopesReturnedInUserInfoClaims")
+		assert.Contains(t, out, `requirements: ["OIDCC-5.4"]`)
+	})
+
+	t.Run("ShouldReportTheConditionArgumentsThatExplainIt", func(t *testing.T) {
+		assert.Contains(t, out, `missing_items: ["given_name","birthdate"]`)
+		assert.Contains(t, out, `expected_scope_items: ["name","given_name","birthdate"]`)
+		assert.Contains(t, out, `actual_scope_items: ["name"]`)
+	})
+
+	t.Run("ShouldOmitUngradedNarrativeAndBookkeeping", func(t *testing.T) {
+		assert.NotContains(t, out, "Userinfo endpoint response was 200")
+		assert.NotContains(t, out, "Extracted userinfo")
+		assert.NotContains(t, out, "testOwner")
+		assert.NotContains(t, out, "1789021616378")
+	})
+
+	t.Run("ShouldOrderFieldsStablyAcrossRuns", func(t *testing.T) {
+		for i := 0; i < 8; i++ {
+			var repeat []ConformanceLogEntry
+
+			require.NoError(t, json.Unmarshal([]byte(scopeFailure), &repeat))
+			assert.Equal(t, out, ConformanceDiagnostics(repeat))
+		}
+	})
+}
+
+func TestConformanceDiagnosticsReportsNothingWhenEveryEntryPassed(t *testing.T) {
+	entries := []ConformanceLogEntry{
+		{Result: "SUCCESS", Msg: "fine"},
+		{Result: "INFO", Msg: "also fine"},
+	}
+
+	assert.Empty(t, ConformanceDiagnostics(entries))
+}
+
+func TestConformanceDiagnosticsTruncatesAValueLargeEnoughToBuryTheRest(t *testing.T) {
+	entries := []ConformanceLogEntry{{
+		Result: "FAILURE",
+		Msg:    "token endpoint returned an error",
+		Fields: map[string]any{"result": "FAILURE", "msg": "token endpoint returned an error", "http_response_body": strings.Repeat("x", conformanceDiagnosticValue+500)},
+	}}
+
+	out := ConformanceDiagnostics(entries)
+
+	assert.Contains(t, out, "500 bytes truncated")
+	assert.Less(t, len(out), conformanceDiagnosticValue+400)
+}
+
+func TestConformanceDiagnosticsCapsTheNumberOfEntries(t *testing.T) {
+	entries := make([]ConformanceLogEntry, 0, conformanceDiagnosticEntries+5)
+
+	for i := 0; i < conformanceDiagnosticEntries+5; i++ {
+		entries = append(entries, ConformanceLogEntry{Result: "FAILURE", Msg: "failed"})
+	}
+
+	out := ConformanceDiagnostics(entries)
+
+	assert.Equal(t, conformanceDiagnosticEntries, strings.Count(out, "[FAILURE]"))
+	assert.Contains(t, out, "further entries omitted")
+}
+
+func TestConformanceAcceptedWarningReport(t *testing.T) {
+	entries := []ConformanceLogEntry{
+		{Msg: "Checked the id_token", Result: "SUCCESS", Src: "ValidateIdToken"},
+		{Msg: "acr value in id_token is not (one of the) requested values", Result: "WARNING", Src: "ValidateIdTokenACRClaimAgainstAcrValuesRequest",
+			Fields: map[string]any{"requirements": []any{"OIDCC-3.1.2.1", "OIDCC-15.1"}, "id_token_acr": "0"}},
+	}
+
+	report := ConformanceAcceptedWarningReport("oidcc-ensure-request-with-acr-values-succeeds", "FINISHED", "the acr is the fallback", entries)
+
+	assert.Contains(t, report, "Module 'oidcc-ensure-request-with-acr-values-succeeds' finished with the accepted result 'WARNING' and the status 'FINISHED': the acr is the fallback.")
+	assert.Contains(t, report, "[WARNING] acr value in id_token is not (one of the) requested values")
+	assert.Contains(t, report, "condition: ValidateIdTokenACRClaimAgainstAcrValuesRequest")
+	assert.Contains(t, report, "id_token_acr: 0")
+	assert.NotContains(t, report, "Checked the id_token", "only the graded entries are reported")
+
+	assert.Equal(t, "Module 'oidcc-server' finished with the accepted result 'WARNING' and the status 'FINISHED': reason. The module logged no warning entries.",
+		ConformanceAcceptedWarningReport("oidcc-server", "FINISHED", "reason", nil))
+}
+
+func TestConformanceDiagnosticsOmitsTheConformanceUIsBookkeeping(t *testing.T) {
+	detail := ConformanceDiagnostics([]ConformanceLogEntry{
+		{Msg: "acr value in id_token is not (one of the) requested values", Result: "WARNING",
+			Fields: map[string]any{"blockId": "a1c507", "updatedAt": float64(1789035070157), "id_token_acr": "0"}},
+	})
+
+	assert.Contains(t, detail, "id_token_acr: 0")
+	assert.NotContains(t, detail, "blockId")
+	assert.NotContains(t, detail, "updatedAt")
+}
+
+func TestConformanceDiagnosticFieldFallsBackForAValueJSONCannotEncode(t *testing.T) {
+	assert.Equal(t, "(1+2i)", conformanceDiagnosticField(complex(1, 2)))
+}

internal/suites/conformance_generate_test.go

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOIDCConformanceGenerateReportsAFileItCannotWrite(t *testing.T) {
+	t.Setenv("SUITE_TMP_PATH", filepath.Join(t.TempDir(), "missing"))
+
+	assert.Error(t, oidcConformanceGenerate())
+	assert.Error(t, oidcConformanceWriteJSON(SuiteTmpPath(oidcConformancePlansFile), []OIDCConformancePlanFile{}))
+}
+
+func TestOIDCConformanceReadPlans(t *testing.T) {
+	t.Run("ShouldReportPlansWhichWereNeverGenerated", func(t *testing.T) {
+		t.Setenv("SUITE_TMP_PATH", t.TempDir())
+
+		_, err := oidcConformanceReadPlans()
+
+		assert.ErrorContains(t, err, "error reading the generated conformance plans, was the suite set up?")
+	})
+
+	t.Run("ShouldReportPlansWhichCannotBeDecoded", func(t *testing.T) {
+		dir := t.TempDir()
+
+		t.Setenv("SUITE_TMP_PATH", dir)
+
+		require.NoError(t, os.WriteFile(filepath.Join(dir, oidcConformancePlansFile), []byte("{"), 0600))
+
+		_, err := oidcConformanceReadPlans()
+
+		assert.Error(t, err)
+	})
+}

internal/suites/conformance_names.go

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"sort"
+	"strings"
+	"unicode"
+)
+
+const conformanceModulePrefix = "oidcc-"
+
+// ConformanceSubtestNames returns the PascalCase subtest name of each module, index aligned with modules. A module
+// which a plan lists more than once has its variant appended so the names within a plan stay unique, which is what
+// makes go test able to address a single one with -run.
+func ConformanceSubtestNames(modules []ConformancePlanModule) (names []string) {
+	counts := make(map[string]int, len(modules))
+
+	for _, module := range modules {
+		counts[module.TestModule]++
+	}
+
+	names = make([]string, len(modules))
+
+	for i, module := range modules {
+		name := conformancePascalCase(strings.TrimPrefix(module.TestModule, conformanceModulePrefix))
+
+		if counts[module.TestModule] > 1 {
+			name += conformanceVariantSuffix(module.Variant)
+		}
+
+		names[i] = name
+	}
+
+	return names
+}
+
+func conformanceVariantSuffix(variant map[string]string) (suffix string) {
+	keys := make([]string, 0, len(variant))
+
+	for key := range variant {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		suffix += conformancePascalCase(variant[key])
+	}
+
+	return suffix
+}
+
+func conformancePascalCase(value string) string {
+	builder := &strings.Builder{}
+
+	for _, part := range strings.FieldsFunc(value, func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	}) {
+		runes := []rune(strings.ToLower(part))
+		runes[0] = unicode.ToUpper(runes[0])
+
+		builder.WriteString(string(runes))
+	}
+
+	return builder.String()
+}

internal/suites/conformance_names_test.go

@@ -0,0 +1,192 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConformanceSubtestNames(t *testing.T) {
+	testCases := []struct {
+		name     string
+		have     []ConformancePlanModule
+		expected []string
+	}{
+		{
+			"ShouldStripPrefixAndPascalCase",
+			[]ConformancePlanModule{{TestModule: "oidcc-server"}},
+			[]string{"Server"},
+		},
+		{
+			"ShouldPascalCaseEverySegment",
+			[]ConformancePlanModule{{TestModule: "oidcc-ensure-request-without-nonce-succeeds-for-code-flow"}},
+			[]string{"EnsureRequestWithoutNonceSucceedsForCodeFlow"},
+		},
+		{
+			"ShouldKeepDigits",
+			[]ConformancePlanModule{{TestModule: "oidcc-max-age-10000"}},
+			[]string{"MaxAge10000"},
+		},
+		{
+			"ShouldHandleModulesWithoutThePrefix",
+			[]ConformancePlanModule{{TestModule: "oidcc-config-certification"}, {TestModule: "discovery-issuer-not-matching-config"}},
+			[]string{"ConfigCertification", "DiscoveryIssuerNotMatchingConfig"},
+		},
+		{
+			"ShouldNotSuffixUniqueModules",
+			[]ConformancePlanModule{
+				{TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code"}},
+				{TestModule: "oidcc-scope-address", Variant: map[string]string{"response_type": "code"}},
+			},
+			[]string{"Server", "ScopeAddress"},
+		},
+		{
+			"ShouldSuffixRepeatedModulesWithTheirVariant",
+			[]ConformancePlanModule{
+				{TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code id_token"}},
+				{TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code id_token token"}},
+			},
+			[]string{"ServerCodeIdToken", "ServerCodeIdTokenToken"},
+		},
+		{
+			"ShouldOrderVariantKeysDeterministically",
+			[]ConformancePlanModule{
+				{TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code", "response_mode": "form_post"}},
+				{TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code", "response_mode": "query"}},
+			},
+			[]string{"ServerFormPostCode", "ServerQueryCode"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, ConformanceSubtestNames(tc.have))
+		})
+	}
+}
+
+func TestConformanceResultAccepted(t *testing.T) {
+	testCases := []struct {
+		name     string
+		module   string
+		result   string
+		expected bool
+	}{
+		{"ShouldAcceptPassedFromAnyModule", "oidcc-server", "PASSED", true},
+		{"ShouldAcceptReviewFromAnyModule", "oidcc-server", "REVIEW", true},
+		{"ShouldAcceptSkippedFromAnyModule", "oidcc-server", "SKIPPED", true},
+		{"ShouldAcceptAWarningFromAListedModule", "oidcc-ensure-request-with-acr-values-succeeds", "WARNING", true},
+		{"ShouldRejectAWarningFromAnUnlistedModule", "oidcc-scope-profile", "WARNING", false},
+		{"ShouldRejectAFailureFromAListedModule", "oidcc-ensure-request-with-acr-values-succeeds", "FAILED", false},
+		{"ShouldRejectAFailure", "oidcc-server", "FAILED", false},
+		{"ShouldRejectAnUnknownResult", "oidcc-server", "", false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, conformanceResultAccepted(tc.module, tc.result))
+		})
+	}
+
+	for module, reason := range conformanceAcceptedWarnings {
+		assert.NotEmptyf(t, reason, "the WARNING accepted from '%s' must say why it is expected", module)
+	}
+}
+
+func TestConformanceArchivePath(t *testing.T) {
+	t.Run("ShouldWriteWhereTheCIArtifactPathCollectsIt", func(t *testing.T) {
+		t.Setenv("CI", "true")
+
+		assert.Equal(t, filepath.Join("..", "..", "oidc-conformance-plans", "conformance-basic.zip"), conformanceArchivePath("basic"))
+	})
+
+	t.Run("ShouldWriteToTheTemporaryDirectoryOutsideCI", func(t *testing.T) {
+		t.Setenv("CI", "")
+
+		assert.Equal(t, filepath.Join(os.TempDir(), "authelia-suites-oidc-conformance-plans", composeProjectName(), "conformance-basic.zip"), conformanceArchivePath("basic"))
+	})
+}
+
+func TestBuildkiteLogMarkers(t *testing.T) {
+	t.Run("ShouldMarkGroupsInBuildkiteWhenDebugging", func(t *testing.T) {
+		t.Setenv("BUILDKITE", "true")
+		t.Setenv("SUITE_DEBUG", "true")
+
+		assert.Equal(t, "--- OIDC Conformance Plan: basic", buildkiteGroup("OIDC Conformance Plan: basic"))
+		assert.Equal(t, "^^^ +++", buildkiteExpandGroup())
+	})
+
+	t.Run("ShouldMarkNothingInBuildkiteWhenNotDebugging", func(t *testing.T) {
+		t.Setenv("BUILDKITE", "true")
+		t.Setenv("SUITE_DEBUG", "")
+
+		assert.Empty(t, buildkiteGroup("OIDC Conformance Plan: basic"))
+		assert.Empty(t, buildkiteExpandGroup())
+	})
+
+	t.Run("ShouldMarkNothingOutsideBuildkite", func(t *testing.T) {
+		t.Setenv("BUILDKITE", "")
+		t.Setenv("SUITE_DEBUG", "true")
+
+		assert.Empty(t, buildkiteGroup("OIDC Conformance Plan: basic"))
+		assert.Empty(t, buildkiteExpandGroup())
+	})
+}
+
+func TestConformancePlanSummary(t *testing.T) {
+	assert.Equal(t, "Plan 'basic': 45 accepted, 2 failed, 1 skipped", conformancePlanSummary("basic", 45, 2, 1))
+}
+
+func TestConformanceArchiveReference(t *testing.T) {
+	t.Run("ShouldLinkTheJobsArtifactsInBuildkite", func(t *testing.T) {
+		t.Setenv("CI", "true")
+		t.Setenv("BUILDKITE_BUILD_URL", "https://buildkite.com/authelia/authelia/builds/58611")
+		t.Setenv("BUILDKITE_JOB_ID", "0192")
+
+		assert.Equal(t, "oidc-conformance-plans/conformance-basic.zip in the job's artifacts: https://buildkite.com/authelia/authelia/builds/58611/waterfall?jid=0192&tab=artifacts",
+			conformanceArchiveReference("basic", "https://conformance.example.com:8443/log-detail.html?log=m1"))
+	})
+
+	t.Run("ShouldPointAtTheLiveLogElsewhere", func(t *testing.T) {
+		t.Setenv("CI", "")
+		t.Setenv("BUILDKITE_BUILD_URL", "")
+		t.Setenv("BUILDKITE_JOB_ID", "")
+
+		assert.Equal(t, conformanceArchivePath("basic")+", or the live log while the suite is up: https://conformance.example.com:8443/log-detail.html?log=m1",
+			conformanceArchiveReference("basic", "https://conformance.example.com:8443/log-detail.html?log=m1"))
+	})
+}
+
+func TestOIDCConformanceRelease(t *testing.T) {
+	assert.Equal(t, "v4.39.24 (commit 7498f635a5f3)", oidcConformanceRelease("7498f635a5f3", "v4.39.24"))
+	assert.Equal(t, "commit 7498f635a5f3", oidcConformanceRelease("7498f635a5f3", ""))
+	assert.Empty(t, oidcConformanceRelease("", ""), "without a commit the builder's own version is left in place")
+}
+
+func TestConformanceOutcomeAccepted(t *testing.T) {
+	testCases := []struct {
+		name     string
+		outcome  ConformanceOutcome
+		expected bool
+	}{
+		{"ShouldAcceptAFinishedPass", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "PASSED"}, true},
+		{"ShouldAcceptAFinishedListedWarning", ConformanceOutcome{Module: "oidcc-ensure-request-with-acr-values-succeeds", Status: "FINISHED", Result: "WARNING"}, true},
+		{"ShouldRejectAnInterruptedReview", ConformanceOutcome{Module: "oidcc-server", Status: "INTERRUPTED", Result: "REVIEW"}, false},
+		{"ShouldRejectAnInterruptedPass", ConformanceOutcome{Module: "oidcc-server", Status: "INTERRUPTED", Result: "PASSED"}, false},
+		{"ShouldRejectAFinishedFailure", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "FAILED"}, false},
+		{"ShouldRejectASuiteError", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "PASSED", Err: errors.New("stalled")}, false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, conformanceOutcomeAccepted(tc.outcome))
+		})
+	}
+}

internal/suites/conformance_progress_test.go

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConformanceProgress(t *testing.T) {
+	patience := time.Second * 30
+	start := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+
+	t.Run("ShouldStallOnAnUnrecognizedPageThatNeverMoves", func(t *testing.T) {
+		progress := newConformanceProgress(patience, start)
+
+		assert.False(t, progress.observe(start, "https://stuck.example.com", false))
+		assert.False(t, progress.observe(start.Add(time.Second*29), "https://stuck.example.com", false))
+		assert.True(t, progress.observe(start.Add(time.Second*31), "https://stuck.example.com", false))
+	})
+
+	t.Run("ShouldNotStallWhileTheURLKeepsChanging", func(t *testing.T) {
+		progress := newConformanceProgress(patience, start)
+
+		for i, at := range []time.Duration{0, time.Second * 25, time.Second * 50, time.Second * 75} {
+			assert.Falsef(t, progress.observe(start.Add(at), "https://hop.example.com/"+string(rune('a'+i)), false), "hop %d", i)
+		}
+	})
+
+	t.Run("ShouldNotStallOnAPageTheDriverIsActingOn", func(t *testing.T) {
+		progress := newConformanceProgress(patience, start)
+
+		assert.False(t, progress.observe(start, "https://login.example.com/consent", true))
+		assert.False(t, progress.observe(start.Add(time.Second*45), "https://login.example.com/consent", true))
+		assert.False(t, progress.observe(start.Add(time.Second*60), "https://login.example.com/consent", true))
+	})
+
+	t.Run("ShouldStallOnceAPageStopsBeingActedOn", func(t *testing.T) {
+		progress := newConformanceProgress(patience, start)
+
+		assert.False(t, progress.observe(start, "https://login.example.com/consent", true))
+		assert.False(t, progress.observe(start.Add(time.Second*10), "https://login.example.com/consent", false))
+		assert.True(t, progress.observe(start.Add(time.Second*41), "https://login.example.com/consent", false))
+	})
+}

internal/suites/conformance_runner.go

@@ -0,0 +1,568 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	conformanceReauthenticationLeg  = 1
+	conformanceStatusConfigured     = "CONFIGURED"
+	conformanceStatusWaiting        = "WAITING"
+	conformanceStatusFinished       = "FINISHED"
+	conformanceStatusInterrupted    = "INTERRUPTED"
+	conformanceModuleTimeout        = time.Minute * 5
+	conformancePlaceholderTimeout   = time.Second * 30
+	conformanceUploadSettleInterval = time.Second
+	conformanceUploadSettleTimeout  = time.Second * 10
+	conformancePlaceholderInterval  = time.Second
+	conformanceOutcomeInfoTimeout   = time.Second * 10
+	conformanceErrorSettleTimeout   = time.Second * 30
+)
+
+var (
+	conformanceTerminalStates = []string{conformanceStatusFinished, conformanceStatusInterrupted}
+
+	conformanceOverrides = map[string]ConformanceOverride{
+		// The second authorization carries prompt=login, so Authelia must ask for credentials again despite the session.
+		"oidcc-prompt-login": {Assert: conformanceAssertReauthentication, Screenshot: ConformanceScreenshotReauthentication},
+
+		// The max age one module sets max_age=1 with a second of delay must force re-authentication and an auth_time claim.
+		"oidcc-max-age-1": {Assert: conformanceAssertReauthentication, Screenshot: ConformanceScreenshotReauthentication},
+
+		// The max age 1000 module sets max_age=10000 is longer than the session has existed, so the provider must not ask
+		// again.
+		"oidcc-max-age-10000": {Assert: conformanceAssertNoReauthentication},
+
+		// The redirect_uri is not registered, so Authelia must show its own error page rather than send the browser to it.
+		// This is the page the module's screenshot placeholder asks for.
+		"oidcc-ensure-registered-redirect-uri": {Assert: conformanceAssertErrorPage, Screenshot: ConformanceScreenshotErrorPage},
+
+		// The request object carries a registered redirect_uri and the query an unregistered one. The module accepts either
+		// Authelia using the request object's and completing the flow, or showing its error page, and only asks for a
+		// screenshot of the latter.
+		"oidcc-ensure-request-object-with-redirect-uri": {Screenshot: ConformanceScreenshotErrorPage},
+
+		// Every module whose upstream summary says to remove any cookies received from the provider, and no others. The
+		// plans also carry oidcc-registration-logo-uri, -policy-uri and -tos-uri under that instruction, but those are
+		// Dynamic OP modules and none of the profiles Authelia is certified for includes them.
+		//
+		// oidcc-prompt-none-not-logged-in is the one where a stale session is not merely untidy: prompt=none against a
+		// live session returns a code, and the module is asserting that the provider errors instead, so it fails.
+		"oidcc-display-page":              {ClearCookies: true},
+		"oidcc-display-popup":             {ClearCookies: true},
+		"oidcc-login-hint":                {ClearCookies: true},
+		"oidcc-prompt-none-not-logged-in": {ClearCookies: true},
+		"oidcc-ui-locales":                {ClearCookies: true},
+	}
+
+	conformanceUnattended = map[string]string{
+		"oidcc-server-rotate-keys": "requires the provider's signing keys to be rotated by hand while the module waits",
+	}
+)
+
+// ConformanceOutcome is the result of one module, produced off the test goroutine and asserted on it.
+type ConformanceOutcome struct {
+	Name    string
+	Module  string
+	ID      string
+	Result  string
+	Status  string
+	LogURL  string
+	Skipped bool
+	Reason  string
+	Err     error
+	Trace   []string
+}
+
+// ConformanceOverride is the per module behavior which differs from the reactive default.
+type ConformanceOverride struct {
+	// ClearCookies discards the context's cookies before the module's first browser leg, for the modules whose
+	// instructions are to remove any cookies received from the provider before proceeding.
+	ClearCookies bool
+
+	// Assert stands in for the screenshot the conformance suite would otherwise want. It is handed what the driver
+	// observed over one browser leg, once that leg has been driven to its end, and its error becomes the module's
+	// verdict. It is given the record rather than the browser because by the time a leg ends the browser is on the
+	// conformance suite's callback, so nothing about Authelia's pages can be established from the live DOM any more.
+	// The record carries the leg's index, so an expectation which belongs to the second authorization round trip is
+	// not applied to the first.
+	Assert func(leg ConformanceLeg) error
+
+	// Screenshot is the page this module's screenshot placeholder asks for. A placeholder is only filled with a capture
+	// of that page, so one from a module without it is left unfilled and the module stalls.
+	Screenshot ConformanceScreenshot
+}
+
+// ConformanceScreenshot is a page a module's screenshot placeholder asks for.
+type ConformanceScreenshot int
+
+const (
+	// ConformanceScreenshotNone is a module which asks for no screenshot.
+	ConformanceScreenshotNone ConformanceScreenshot = iota
+
+	// ConformanceScreenshotReauthentication is Authelia asking for the password again on the second authorization.
+	ConformanceScreenshotReauthentication
+
+	// ConformanceScreenshotErrorPage is Authelia's error page.
+	ConformanceScreenshotErrorPage
+)
+
+func (o ConformanceOverride) screenshotOf(leg ConformanceLeg) string {
+	switch o.Screenshot {
+	case ConformanceScreenshotReauthentication:
+		if leg.Index == conformanceReauthenticationLeg {
+			return leg.LoginScreenshot
+		}
+	case ConformanceScreenshotErrorPage:
+		return leg.ErrorScreenshot
+	}
+
+	return ""
+}
+
+func conformanceAssertReauthentication(leg ConformanceLeg) error {
+	if leg.Index != conformanceReauthenticationLeg {
+		return nil
+	}
+
+	if !leg.Reauthentication && !leg.FirstFactor {
+		return errors.New("expected Authelia to ask for the password again on the second authorization but it did not")
+	}
+
+	return nil
+}
+
+func conformanceAssertNoReauthentication(leg ConformanceLeg) error {
+	if leg.Index != conformanceReauthenticationLeg {
+		return nil
+	}
+
+	if leg.Reauthentication || leg.FirstFactor {
+		return errors.New("expected Authelia to reuse the existing session on the second authorization but it asked for the password again")
+	}
+
+	return nil
+}
+
+func conformanceAssertErrorPage(leg ConformanceLeg) error {
+	if !leg.AutheliaError {
+		return errors.New("expected Authelia to show an error page rather than redirect to the client but it did not")
+	}
+
+	return nil
+}
+
+// ConformanceRunner walks one plan's modules in order.
+type ConformanceRunner struct {
+	client  *ConformanceClient
+	browser *ConformanceBrowser
+	planID  string
+	modules []ConformancePlanModule
+	names   []string
+
+	// debug is whether each module's trace is recorded.
+	debug bool
+
+	// trace is the trace of the module being run. Modules run one at a time, so it is replaced as each begins.
+	trace *ConformanceTrace
+}
+
+// NewConformanceRunner returns a ConformanceRunner for one plan. browser may be nil only when no module in modules
+// reaches the WAITING state, which in practice means only in tests.
+func NewConformanceRunner(client *ConformanceClient, browser *ConformanceBrowser, planID string, modules []ConformancePlanModule) *ConformanceRunner {
+	return &ConformanceRunner{
+		client:  client,
+		browser: browser,
+		planID:  planID,
+		modules: modules,
+		names:   ConformanceSubtestNames(modules),
+	}
+}
+
+// Run walks the plan on its own goroutine and returns the channel its outcomes arrive on, one per module in plan
+// order. The channel closes when the plan is done. Nothing here touches [testing.T]: an outcome carries its error so the
+// test goroutine can be the one that fails.
+func (r *ConformanceRunner) Run(ctx context.Context) <-chan ConformanceOutcome {
+	outcomes := make(chan ConformanceOutcome, len(r.modules))
+
+	go func() {
+		defer close(outcomes)
+
+		for i, module := range r.modules {
+			outcomes <- r.run(ctx, r.names[i], module)
+		}
+	}()
+
+	return outcomes
+}
+
+func (r *ConformanceRunner) run(ctx context.Context, name string, module ConformancePlanModule) (outcome ConformanceOutcome) {
+	outcome = ConformanceOutcome{Name: name, Module: module.TestModule}
+
+	r.trace = NewConformanceTrace(r.debug)
+
+	if r.browser != nil {
+		r.browser.trace = r.trace
+	}
+
+	defer func() {
+		outcome.Trace = r.trace.Lines()
+	}()
+
+	if reason, ok := conformanceUnattended[module.TestModule]; ok {
+		outcome.Skipped, outcome.Reason = true, reason
+
+		return outcome
+	}
+
+	parent := ctx
+
+	ctx, cancel := context.WithTimeout(ctx, conformanceModuleTimeout)
+	defer cancel()
+
+	override := conformanceOverrides[module.TestModule]
+
+	id, state, err := r.createAndAdvance(ctx, module, override)
+
+	r.trace.Logf("Module '%s' created as '%s' in state '%s'", module.TestModule, id, state)
+
+	if id != "" {
+		outcome.ID, outcome.LogURL = id, r.client.LogDetailURL(id)
+	}
+
+	if err != nil {
+		outcome.Err = err
+
+		if id != "" {
+			r.settleAfterError(parent, id)
+		}
+
+		return outcome
+	}
+
+	if state == conformanceStatusWaiting {
+		if err = r.interact(ctx, id, override); err != nil {
+			outcome.Err = err
+		}
+	}
+
+	if outcome.Err == nil {
+		if _, err = r.client.WaitState(ctx, id, conformanceTerminalStates...); err != nil {
+			outcome.Err = err
+		}
+	}
+
+	if outcome.Err != nil {
+		r.settleAfterError(parent, id)
+	}
+
+	infoCtx, infoCancel := context.WithTimeout(context.WithoutCancel(ctx), conformanceOutcomeInfoTimeout)
+	defer infoCancel()
+
+	info, err := r.client.Info(infoCtx, id)
+	if err != nil {
+		if outcome.Err == nil {
+			outcome.Err = err
+		}
+
+		return outcome
+	}
+
+	outcome.Status, outcome.Result = info.Status, info.Result
+
+	return outcome
+}
+
+func (r *ConformanceRunner) settleAfterError(parent context.Context, id string) {
+	ctx, cancel := context.WithTimeout(parent, conformanceErrorSettleTimeout)
+	defer cancel()
+
+	state, err := r.client.WaitState(ctx, id, append([]string{conformanceStatusConfigured, conformanceStatusWaiting}, conformanceTerminalStates...)...)
+
+	r.trace.Logf("Module '%s' settled in state '%s' after the suite failed it (%v)", id, state, err)
+}
+
+func (r *ConformanceRunner) createAndAdvance(ctx context.Context, module ConformancePlanModule, override ConformanceOverride) (id, state string, err error) {
+	if id, err = r.client.CreateTest(ctx, r.planID, module.TestModule, module.Variant); err != nil {
+		return "", "", fmt.Errorf("error creating module '%s': %w", module.TestModule, err)
+	}
+
+	if override.ClearCookies && r.browser != nil {
+		if err = r.browser.ClearCookies(); err != nil {
+			return id, "", err
+		}
+	}
+
+	if state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusConfigured, conformanceStatusWaiting}, conformanceTerminalStates...)...); err != nil {
+		return id, "", err
+	}
+
+	if state != conformanceStatusConfigured {
+		return id, state, nil
+	}
+
+	if err = r.client.StartTest(ctx, id); err != nil {
+		return id, "", err
+	}
+
+	state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusWaiting}, conformanceTerminalStates...)...)
+
+	return id, state, err
+}
+
+func (r *ConformanceRunner) interact(ctx context.Context, id string, override ConformanceOverride) (err error) {
+	if r.browser == nil {
+		return errors.New("the module needs a browser but the runner has none")
+	}
+
+	legs := &conformanceLegs{visited: map[string]bool{}}
+
+	var deadline time.Time
+
+	for {
+		if ctx.Err() != nil {
+			return fmt.Errorf("module '%s' did not leave the WAITING state: %w", id, ctx.Err())
+		}
+
+		status, err := r.client.BrowserStatus(ctx, id)
+		if err != nil {
+			return err
+		}
+
+		if deadline.IsZero() {
+			deadline = time.Now().Add(conformancePlaceholderTimeout)
+		}
+
+		progressed, err := r.visitURLs(ctx, id, override, legs, status.URLs)
+		if err != nil {
+			return err
+		}
+
+		filled, pending, err := r.fillPlaceholders(ctx, id, legs.screenshot)
+		if err != nil {
+			return err
+		}
+
+		legs.pendingPlaceholder = pending
+
+		done, settled, err := r.advance(ctx, id, filled)
+		if err != nil {
+			return err
+		}
+
+		if done {
+			return nil
+		}
+
+		if progressed || filled || settled {
+			deadline = time.Now().Add(conformancePlaceholderTimeout)
+		} else if time.Now().After(deadline) {
+			return legs.stalled(id)
+		}
+
+		time.Sleep(conformancePlaceholderInterval)
+	}
+}
+
+func (r *ConformanceRunner) advance(ctx context.Context, id string, filled bool) (done, settled bool, err error) {
+	var state string
+
+	if state, err = r.moduleState(ctx, id, filled); err != nil {
+		return false, false, err
+	}
+
+	switch state {
+	case conformanceStatusFinished, conformanceStatusInterrupted:
+		return true, false, nil
+	case conformanceStatusWaiting:
+		return false, false, nil
+	}
+
+	r.trace.Logf("Module '%s' is in state '%s' between browser legs", id, state)
+
+	if state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusWaiting}, conformanceTerminalStates...)...); err != nil {
+		return false, false, err
+	}
+
+	return state != conformanceStatusWaiting, true, nil
+}
+
+type conformanceLegs struct {
+	visited            map[string]bool
+	count              int
+	errorURL           string
+	screenshot         string
+	pendingPlaceholder bool
+}
+
+func (l *conformanceLegs) stalled(id string) error {
+	if l.pendingPlaceholder {
+		return fmt.Errorf("module '%s' is waiting for an evidence image, but no leg showed the page it asks for", id)
+	}
+
+	if l.errorURL != "" {
+		return &ConformanceErrorPageStallError{ID: id, URL: l.errorURL}
+	}
+
+	return fmt.Errorf("module '%s' stayed in the WAITING state with nothing left to visit or fill", id)
+}
+
+// ConformanceErrorPageStallError is a module left waiting for the flow to return to the client after Authelia ended
+// it on an error page, which the module had no placeholder for.
+type ConformanceErrorPageStallError struct {
+	ID  string
+	URL string
+}
+
+// Error names the module and what Authelia's error page reported.
+func (e *ConformanceErrorPageStallError) Error() string {
+	return fmt.Sprintf("module '%s' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: %s",
+		e.ID, conformanceDescribeErrorPage(e.URL))
+}
+
+func conformanceDescribeErrorPage(uri string) string {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return uri
+	}
+
+	query := u.Query()
+
+	var parts []string
+
+	for _, field := range []struct{ label, key string }{
+		{"error", "error"},
+		{"description", "error_description"},
+		{"hint", "error_hint"},
+		{"debug", "error_debug"},
+	} {
+		if value := query.Get(field.key); value != "" {
+			parts = append(parts, fmt.Sprintf("%s '%s'", field.label, value))
+		}
+	}
+
+	if len(parts) == 0 {
+		return uri
+	}
+
+	return strings.Join(parts, ", ")
+}
+
+func (r *ConformanceRunner) visitURLs(ctx context.Context, id string, override ConformanceOverride, legs *conformanceLegs, urls []string) (progressed bool, err error) {
+	for _, uri := range urls {
+		if legs.visited[uri] {
+			continue
+		}
+
+		legs.visited[uri] = true
+		progressed = true
+
+		index := legs.count
+		legs.count++
+
+		r.trace.Logf("Driving leg %d at '%s'", index, uri)
+
+		leg, err := r.browser.Drive(ctx, index, uri)
+		if err != nil {
+			return progressed, err
+		}
+
+		r.trace.Logf("Finished leg %d: first factor %t, consent %t, reauthentication %t, error page %t",
+			index, leg.FirstFactor, leg.Consent, leg.Reauthentication, leg.AutheliaError)
+
+		legs.errorURL = leg.ErrorURL
+		legs.screenshot = conformanceLatest(legs.screenshot, override.screenshotOf(leg))
+
+		if override.Assert != nil {
+			if err = override.Assert(leg); err != nil {
+				return progressed, err
+			}
+		}
+
+		if err = r.client.Visit(ctx, id, uri); err != nil {
+			return progressed, err
+		}
+	}
+
+	return progressed, nil
+}
+
+func (r *ConformanceRunner) fillPlaceholders(ctx context.Context, id, screenshot string) (filled, pending bool, err error) {
+	entries, err := r.client.Log(ctx, id)
+	if err != nil {
+		return false, false, err
+	}
+
+	for _, entry := range entries {
+		if entry.Upload == "" {
+			continue
+		}
+
+		if screenshot == "" {
+			pending = true
+
+			continue
+		}
+
+		r.trace.Logf("Module '%s' uploading evidence image '%s'", id, entry.Upload)
+
+		if err = r.client.UploadPlaceholder(ctx, id, entry.Upload, screenshot); err != nil {
+			return filled, pending, err
+		}
+
+		filled = true
+	}
+
+	return filled, pending, nil
+}
+
+func (r *ConformanceRunner) awaitPlaceholderSettled(ctx context.Context, id string, interval, timeout time.Duration) (current string, err error) {
+	deadline := time.Now().Add(timeout)
+
+	for {
+		var info *ConformanceTestInfo
+
+		if info, err = r.client.Info(ctx, id); err != nil {
+			return "", fmt.Errorf("error reading the status of module '%s' while waiting for it to take up its evidence image: %w", id, err)
+		}
+
+		if info.Status != conformanceStatusWaiting || time.Now().After(deadline) {
+			return info.Status, nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("module '%s' did not leave the %s state after its evidence image was uploaded: %w", id, conformanceStatusWaiting, ctx.Err())
+		case <-time.After(interval):
+		}
+	}
+}
+
+func (r *ConformanceRunner) moduleState(ctx context.Context, id string, filled bool) (state string, err error) {
+	if filled {
+		if state, err = r.awaitPlaceholderSettled(ctx, id, conformanceUploadSettleInterval, conformanceUploadSettleTimeout); err != nil {
+			return "", err
+		}
+
+		r.trace.Logf("Module '%s' is in state '%s' after its evidence image was uploaded", id, state)
+
+		return state, nil
+	}
+
+	var info *ConformanceTestInfo
+
+	if info, err = r.client.Info(ctx, id); err != nil {
+		return "", err
+	}
+
+	return info.Status, nil
+}

internal/suites/conformance_runner_errors_test.go

@@ -0,0 +1,305 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceRunner_ReportsTheStepWhichFailed(t *testing.T) {
+	created := conformanceReply(http.StatusCreated, `{"id":"m1"}`)
+
+	testCases := []struct {
+		name     string
+		routes   map[string]http.HandlerFunc
+		expected string
+		id       string
+	}{
+		{
+			"ShouldReportAModuleWhichCouldNotBeCreated",
+			map[string]http.HandlerFunc{"/api/runner": conformanceReply(http.StatusInternalServerError, "broken")},
+			"error creating module 'oidcc-server'",
+			"",
+		},
+		{
+			"ShouldReportAModuleWhichCouldNotBeWaitedOn",
+			map[string]http.HandlerFunc{
+				"/api/runner":               created,
+				"/api/runner/m1/wait-state": conformanceReply(http.StatusBadRequest, "bad"),
+			},
+			"expected status 200 but got 400",
+			"m1",
+		},
+		{
+			"ShouldReportAConfiguredModuleWhichCouldNotBeStarted",
+			map[string]http.HandlerFunc{
+				"/api/runner":               created,
+				"/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"CONFIGURED"}`),
+				"POST /api/runner/m1":       conformanceReply(http.StatusInternalServerError, "broken"),
+			},
+			"expected status 200 but got 500",
+			"m1",
+		},
+		{
+			"ShouldReportAModuleWhoseStatusCouldNotBeRead",
+			map[string]http.HandlerFunc{
+				"/api/runner":               created,
+				"/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`),
+				"/api/info/m1":              conformanceReply(http.StatusInternalServerError, "broken"),
+			},
+			"expected status 200 but got 500",
+			"m1",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			outcome := conformanceRunOne(t, newConformanceFakeClient(t, tc.routes))
+
+			require.Error(t, outcome.Err)
+			assert.ErrorContains(t, outcome.Err, tc.expected)
+			assert.Equal(t, tc.id, outcome.ID)
+		})
+	}
+}
+
+func TestConformanceRunner_StartsAConfiguredModule(t *testing.T) {
+	var started atomic.Bool
+
+	client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+		"/api/runner": conformanceReply(http.StatusCreated, `{"id":"m1"}`),
+		"/api/runner/m1/wait-state": func(w http.ResponseWriter, r *http.Request) {
+			if strings.Contains(r.URL.Query().Get("states"), conformanceStatusConfigured) {
+				_, _ = w.Write([]byte(`{"state":"CONFIGURED"}`))
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"state":"FINISHED"}`))
+		},
+		"POST /api/runner/m1": func(w http.ResponseWriter, _ *http.Request) {
+			started.Store(true)
+
+			_, _ = w.Write([]byte(`{}`))
+		},
+		"/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`),
+	})
+
+	outcome := conformanceRunOne(t, client)
+
+	require.NoError(t, outcome.Err)
+	assert.True(t, started.Load())
+	assert.Equal(t, "PASSED", outcome.Result)
+}
+
+func TestConformanceRunner_InteractReportsTheCallWhichFailed(t *testing.T) {
+	status := conformanceReply(http.StatusOK, `{"urls":[]}`)
+	log := conformanceReply(http.StatusOK, `[]`)
+
+	testCases := []struct {
+		name     string
+		routes   map[string]http.HandlerFunc
+		expected string
+	}{
+		{
+			"ShouldReportTheBrowserStatus",
+			map[string]http.HandlerFunc{"/api/runner/browser/m1": conformanceReply(http.StatusInternalServerError, "status broken")},
+			"status broken",
+		},
+		{
+			"ShouldReportTheLog",
+			map[string]http.HandlerFunc{
+				"/api/runner/browser/m1": status,
+				"/api/log/m1":            conformanceReply(http.StatusInternalServerError, "log broken"),
+			},
+			"log broken",
+		},
+		{
+			"ShouldReportTheStatus",
+			map[string]http.HandlerFunc{
+				"/api/runner/browser/m1": status,
+				"/api/log/m1":            log,
+				"/api/info/m1":           conformanceReply(http.StatusInternalServerError, "info broken"),
+			},
+			"info broken",
+		},
+		{
+			"ShouldReportTheWaitForAModuleBetweenLegs",
+			map[string]http.HandlerFunc{
+				"/api/runner/browser/m1":    status,
+				"/api/log/m1":               log,
+				"/api/info/m1":              conformanceReply(http.StatusOK, `{"_id":"m1","status":"RUNNING"}`),
+				"/api/runner/m1/wait-state": conformanceReply(http.StatusBadRequest, "wait broken"),
+			},
+			"wait broken",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := NewConformanceRunner(newConformanceFakeClient(t, tc.routes), &ConformanceBrowser{}, "plan1", nil)
+
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
+			assert.ErrorContains(t, runner.interact(ctx, "m1", ConformanceOverride{}), tc.expected)
+		})
+	}
+}
+
+func TestConformanceRunner_FillPlaceholdersReportsAnUploadWhichFailed(t *testing.T) {
+	client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+		"/api/log/m1":            conformanceReply(http.StatusOK, `[{"msg":"upload","upload":"ph1","result":"REVIEW"}]`),
+		"/api/log/m1/images/ph1": conformanceReply(http.StatusInternalServerError, "upload broken"),
+	})
+
+	filled, pending, err := NewConformanceRunner(client, nil, "plan1", nil).fillPlaceholders(context.Background(), "m1", conformanceTestScreenshot)
+
+	assert.ErrorContains(t, err, "upload broken")
+	assert.False(t, filled)
+	assert.False(t, pending)
+}
+
+func TestConformanceRunner_InteractStopsWhenTheContextHasExpired(t *testing.T) {
+	runner := NewConformanceRunner(nil, &ConformanceBrowser{}, "plan1", nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	assert.ErrorContains(t, runner.interact(ctx, "m1", ConformanceOverride{}), "module 'm1' did not leave the WAITING state")
+}
+
+func TestConformanceRunner_InteractPollsAWaitingModuleAgain(t *testing.T) {
+	var infos atomic.Int32
+
+	client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+		"/api/runner/browser/m1": conformanceReply(http.StatusOK, `{"urls":[]}`),
+		"/api/log/m1":            conformanceReply(http.StatusOK, `[]`),
+		"/api/info/m1": func(w http.ResponseWriter, _ *http.Request) {
+			if infos.Add(1) == 1 {
+				_, _ = w.Write([]byte(`{"_id":"m1","status":"WAITING"}`))
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`))
+		},
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	require.NoError(t, NewConformanceRunner(client, &ConformanceBrowser{}, "plan1", nil).interact(ctx, "m1", ConformanceOverride{}))
+	assert.Equal(t, int32(2), infos.Load())
+}
+
+func TestConformanceRunner_ModuleState(t *testing.T) {
+	t.Run("ShouldSettleAfterAnEvidenceImage", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+			"/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED"}`),
+		})
+
+		runner := NewConformanceRunner(client, nil, "plan1", nil)
+		runner.trace = NewConformanceTrace(true)
+
+		state, err := runner.moduleState(context.Background(), "m1", true)
+
+		require.NoError(t, err)
+		assert.Equal(t, conformanceStatusFinished, state)
+		require.Len(t, runner.trace.Lines(), 1)
+		assert.Contains(t, runner.trace.Lines()[0], "Module 'm1' is in state 'FINISHED' after its evidence image was uploaded")
+	})
+
+	t.Run("ShouldReportAStatusWhichCouldNotBeReadWhileSettling", func(t *testing.T) {
+		client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+			"/api/info/m1": conformanceReply(http.StatusInternalServerError, "broken"),
+		})
+
+		_, err := NewConformanceRunner(client, nil, "plan1", nil).moduleState(context.Background(), "m1", true)
+
+		assert.ErrorContains(t, err, "while waiting for it to take up its evidence image")
+	})
+}
+
+func TestConformanceDescribeErrorPageFallsBackForAnUnparsableURL(t *testing.T) {
+	assert.Equal(t, "%zz", conformanceDescribeErrorPage("%zz"))
+}
+
+func TestConformanceRunner_HandsTheModuleTraceToItsBrowser(t *testing.T) {
+	client := newConformanceFakeClient(t, map[string]http.HandlerFunc{
+		"/api/runner":               conformanceReply(http.StatusCreated, `{"id":"m1"}`),
+		"/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`),
+		"/api/info/m1":              conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`),
+	})
+
+	browser := &ConformanceBrowser{}
+	runner := NewConformanceRunner(client, browser, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}})
+	runner.debug = true
+
+	for outcome := range runner.Run(context.Background()) {
+		require.NoError(t, outcome.Err)
+	}
+
+	assert.Same(t, runner.trace, browser.trace, "what the driver does is recorded in the module's trace")
+}
+
+func newConformanceFakeClient(t *testing.T, routes map[string]http.HandlerFunc) *ConformanceClient {
+	t.Helper()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		route, ok := routes[r.Method+" "+r.URL.Path]
+		if !ok {
+			route, ok = routes[r.URL.Path]
+		}
+
+		if !ok {
+			t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+
+			return
+		}
+
+		route(w, r)
+	}))
+
+	t.Cleanup(server.Close)
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	return client
+}
+
+func conformanceReply(status int, body string) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}
+}
+
+func conformanceRunOne(t *testing.T, client *ConformanceClient) ConformanceOutcome {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	var outcomes []ConformanceOutcome
+
+	for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) {
+		outcomes = append(outcomes, outcome)
+	}
+
+	require.Len(t, outcomes, 1)
+
+	return outcomes[0]
+}

internal/suites/conformance_runner_test.go

@@ -0,0 +1,659 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceRunner_ReportsOutcomesInPlanOrder(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch { //nolint:staticcheck // a tagged switch would obscure the multi-condition cases below.
+		case r.URL.Path == "/api/runner":
+			w.WriteHeader(http.StatusCreated)
+			_, _ = w.Write([]byte(`{"id":"` + r.URL.Query().Get("test") + `-id"}`)) //nolint:gosec // test double echoing a query param set by this same test, not attacker controlled.
+		case r.URL.Path == "/api/runner/oidcc-server-id/wait-state",
+			r.URL.Path == "/api/runner/oidcc-scope-address-id/wait-state":
+			_, _ = w.Write([]byte(`{"state":"FINISHED"}`))
+		case r.URL.Path == "/api/info/oidcc-server-id":
+			_, _ = w.Write([]byte(`{"_id":"oidcc-server-id","status":"FINISHED","result":"PASSED"}`))
+		case r.URL.Path == "/api/info/oidcc-scope-address-id":
+			_, _ = w.Write([]byte(`{"_id":"oidcc-scope-address-id","status":"FINISHED","result":"WARNING"}`))
+		case r.URL.Path == "/api/log/oidcc-server-id", r.URL.Path == "/api/log/oidcc-scope-address-id":
+			_, _ = w.Write([]byte(`[]`))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	modules := []ConformancePlanModule{{TestModule: "oidcc-server"}, {TestModule: "oidcc-scope-address"}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+	defer cancel()
+
+	var outcomes []ConformanceOutcome
+
+	for outcome := range NewConformanceRunner(client, nil, "plan1", modules).Run(ctx) {
+		outcomes = append(outcomes, outcome)
+	}
+
+	require.Len(t, outcomes, 2)
+
+	assert.Equal(t, "Server", outcomes[0].Name)
+	assert.Equal(t, "PASSED", outcomes[0].Result)
+	require.NoError(t, outcomes[0].Err)
+
+	assert.Equal(t, "ScopeAddress", outcomes[1].Name)
+	assert.Equal(t, "WARNING", outcomes[1].Result)
+}
+
+func TestConformanceRunner_SkipsUnattendedModules(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Errorf("the runner must not contact the server for a skipped module, got %s", r.URL.Path)
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	modules := []ConformancePlanModule{{TestModule: "oidcc-server-rotate-keys"}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	var outcomes []ConformanceOutcome
+
+	for outcome := range NewConformanceRunner(client, nil, "plan1", modules).Run(ctx) {
+		outcomes = append(outcomes, outcome)
+	}
+
+	require.Len(t, outcomes, 1)
+	assert.True(t, outcomes[0].Skipped)
+	assert.Equal(t, "ServerRotateKeys", outcomes[0].Name)
+	assert.NotEmpty(t, outcomes[0].Reason)
+}
+
+func TestConformanceRunner_FillPlaceholdersUploadsOnlyUnfilledEntries(t *testing.T) {
+	t.Run("UploadsOnlyTheEntryWithAnUnfilledPlaceholder", func(t *testing.T) {
+		var (
+			uploadCount  int
+			uploadedID   string
+			uploadedBody []byte
+		)
+
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/api/log/m1":
+				_, _ = w.Write([]byte(`[{"msg":"info","result":"INFO"},{"msg":"upload","upload":"ph1","result":"REVIEW"}]`))
+			case "/api/log/m1/images/ph1":
+				uploadCount++
+				uploadedID = "ph1"
+				uploadedBody, _ = io.ReadAll(r.Body)
+
+				_, _ = w.Write([]byte(`{}`))
+			default:
+				t.Errorf("unexpected path %s", r.URL.Path)
+			}
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		defer cancel()
+
+		filled, pending, err := runner.fillPlaceholders(ctx, "m1", conformanceTestScreenshot)
+		require.NoError(t, err)
+
+		assert.True(t, filled)
+		assert.False(t, pending)
+		assert.Equal(t, 1, uploadCount)
+		assert.Equal(t, "ph1", uploadedID)
+		assert.Equal(t, conformanceTestScreenshot, string(uploadedBody))
+	})
+
+	t.Run("UploadsNothingWhenNoPlaceholderIsUnfilled", func(t *testing.T) {
+		var uploadCalled bool
+
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/api/log/m1":
+				_, _ = w.Write([]byte(`[{"msg":"info","result":"INFO"}]`))
+			case "/api/log/m1/images/ph1":
+				uploadCalled = true
+
+				_, _ = w.Write([]byte(`{}`))
+			default:
+				t.Errorf("unexpected path %s", r.URL.Path)
+			}
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		defer cancel()
+
+		filled, pending, err := runner.fillPlaceholders(ctx, "m1", conformanceTestScreenshot)
+		require.NoError(t, err)
+
+		assert.False(t, filled)
+		assert.False(t, pending)
+		assert.False(t, uploadCalled)
+	})
+
+	t.Run("WaitsForTheScreenshotBeforeUploading", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/api/log/m1":
+				_, _ = w.Write([]byte(`[{"msg":"upload","upload":"ph1","result":"REVIEW"}]`))
+			default:
+				t.Errorf("unexpected path %s", r.URL.Path)
+			}
+		}))
+
+		defer server.Close()
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		defer cancel()
+
+		filled, pending, err := runner.fillPlaceholders(ctx, "m1", "")
+		require.NoError(t, err)
+
+		assert.False(t, filled)
+		assert.True(t, pending)
+	})
+}
+
+func TestConformanceOverrideAssertions(t *testing.T) {
+	testCases := []struct {
+		name     string
+		assert   func(leg ConformanceLeg) error
+		leg      ConformanceLeg
+		expected string
+	}{
+		{
+			"ShouldIgnoreTheFirstLegWhenReauthenticationIsExpected",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 0, FirstFactor: false},
+			"",
+		},
+		{
+			"ShouldIgnoreTheFirstLegEvenWhenItPrompted",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 0, FirstFactor: true},
+			"",
+		},
+		{
+			"ShouldPassWhenTheSecondLegPrompted",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 1, FirstFactor: true},
+			"",
+		},
+		{
+			"ShouldFailWhenTheSecondLegDidNotPrompt",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 1, FirstFactor: false},
+			"expected Authelia to ask for the password again on the second authorization but it did not",
+		},
+		{
+			"ShouldPassWhenTheSecondLegReauthenticatedOnTheConsentForm",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 1, Consent: true, Reauthentication: true},
+			"",
+		},
+		{
+			"ShouldFailWhenTheSecondLegShowedConsentWithoutAskingForThePassword",
+			conformanceAssertReauthentication,
+			ConformanceLeg{Index: 1, Consent: true, Reauthentication: false},
+			"expected Authelia to ask for the password again on the second authorization but it did not",
+		},
+		{
+			"ShouldFailWhenTheSecondLegReauthenticatedOnTheConsentFormButShouldNotHave",
+			conformanceAssertNoReauthentication,
+			ConformanceLeg{Index: 1, Consent: true, Reauthentication: true},
+			"expected Authelia to reuse the existing session on the second authorization but it asked for the password again",
+		},
+		{
+			"ShouldIgnoreTheFirstLegWhenNoReauthenticationIsExpected",
+			conformanceAssertNoReauthentication,
+			ConformanceLeg{Index: 0, FirstFactor: true},
+			"",
+		},
+		{
+			"ShouldPassWhenTheSecondLegDidNotPrompt",
+			conformanceAssertNoReauthentication,
+			ConformanceLeg{Index: 1, FirstFactor: false},
+			"",
+		},
+		{
+			"ShouldFailWhenTheSecondLegPrompted",
+			conformanceAssertNoReauthentication,
+			ConformanceLeg{Index: 1, FirstFactor: true},
+			"expected Authelia to reuse the existing session on the second authorization but it asked for the password again",
+		},
+		{
+			"ShouldNotBeConfusedByConsentOnTheSecondLeg",
+			conformanceAssertNoReauthentication,
+			ConformanceLeg{Index: 1, FirstFactor: false, Consent: true},
+			"",
+		},
+		{
+			"ShouldPassWhenTheLegEndedOnAnAutheliaErrorPage",
+			conformanceAssertErrorPage,
+			ConformanceLeg{Index: 0, AutheliaError: true},
+			"",
+		},
+		{
+			"ShouldFailWhenTheLegReachedTheCallbackInsteadOfAnErrorPage",
+			conformanceAssertErrorPage,
+			ConformanceLeg{Index: 0},
+			"expected Authelia to show an error page rather than redirect to the client but it did not",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.assert(tc.leg)
+
+			if tc.expected == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tc.expected)
+			}
+		})
+	}
+}
+
+func TestConformanceOverrides_AreWiredToTheirAssertions(t *testing.T) {
+	for module, expected := range map[string]ConformanceLeg{
+		"oidcc-prompt-login":  {Index: conformanceReauthenticationLeg, FirstFactor: false},
+		"oidcc-max-age-1":     {Index: conformanceReauthenticationLeg, FirstFactor: false},
+		"oidcc-max-age-10000": {Index: conformanceReauthenticationLeg, FirstFactor: true},
+
+		"oidcc-ensure-registered-redirect-uri": {Index: 0},
+	} {
+		override, ok := conformanceOverrides[module]
+
+		require.Truef(t, ok, "the '%s' module has no override", module)
+		require.NotNilf(t, override.Assert, "the '%s' module has no assertion", module)
+
+		assert.Errorf(t, override.Assert(expected), "the '%s' module's assertion cannot fail", module)
+	}
+}
+
+func TestConformanceRunner_VisitURLsDoesNotRedriveOrRecountAVisitedURL(t *testing.T) {
+	legs := &conformanceLegs{visited: map[string]bool{"https://conformance.example.com/first": true}, count: 1}
+
+	override := ConformanceOverride{Assert: func(leg ConformanceLeg) error {
+		return fmt.Errorf("the assertion must not run for an already driven leg, got %+v", leg)
+	}}
+
+	runner := NewConformanceRunner(nil, nil, "plan1", nil)
+
+	progressed, err := runner.visitURLs(context.Background(), "m1", override, legs, []string{"https://conformance.example.com/first"})
+	require.NoError(t, err)
+
+	assert.False(t, progressed)
+	assert.Equal(t, 1, legs.count)
+}
+
+func TestConformanceOverrides_ClearCookiesMatchesTheModulesThatRequireIt(t *testing.T) {
+	expected := []string{
+		"oidcc-display-page",
+		"oidcc-display-popup",
+		"oidcc-login-hint",
+		"oidcc-prompt-none-not-logged-in",
+		"oidcc-ui-locales",
+	}
+
+	var actual []string
+
+	for module, override := range conformanceOverrides {
+		if override.ClearCookies {
+			actual = append(actual, module)
+		}
+	}
+
+	sort.Strings(actual)
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestConformanceRunner_TreatsAnInterruptedModuleAsFinished(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/runner":
+			w.WriteHeader(http.StatusCreated)
+			_, _ = w.Write([]byte(`{"id":"m1"}`))
+		case "/api/runner/m1/wait-state":
+			if !strings.Contains(r.URL.Query().Get("states"), conformanceStatusInterrupted) {
+				time.Sleep(time.Millisecond * 10)
+
+				_, _ = w.Write([]byte(`{"timeout":true}`))
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"state":"INTERRUPTED"}`))
+		case "/api/info/m1":
+			_, _ = w.Write([]byte(`{"_id":"m1","status":"INTERRUPTED","result":"FAILED"}`))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	var outcomes []ConformanceOutcome
+
+	for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-max-age-1"}}).Run(ctx) {
+		outcomes = append(outcomes, outcome)
+	}
+
+	require.Len(t, outcomes, 1)
+	require.NoError(t, outcomes[0].Err)
+
+	assert.Equal(t, conformanceStatusInterrupted, outcomes[0].Status)
+	assert.Equal(t, "FAILED", outcomes[0].Result)
+}
+
+func TestConformanceRunner_InteractKeepsDrivingAModuleWhichIsRunningBetweenLegs(t *testing.T) {
+	var (
+		mu                  sync.Mutex
+		browserPolls, infos int
+	)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+
+		switch r.URL.Path {
+		case "/api/runner/browser/m1":
+			browserPolls++
+
+			_, _ = w.Write([]byte(`{"urls":[]}`))
+		case "/api/log/m1":
+			_, _ = w.Write([]byte(`[]`))
+		case "/api/info/m1":
+			infos++
+
+			if infos == 1 {
+				_, _ = w.Write([]byte(`{"_id":"m1","status":"RUNNING"}`))
+
+				return
+			}
+
+			_, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`))
+		case "/api/runner/m1/wait-state":
+			assert.Equal(t, "WAITING,FINISHED,INTERRUPTED", r.URL.Query().Get("states"))
+
+			_, _ = w.Write([]byte(`{"state":"WAITING"}`))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	runner := NewConformanceRunner(client, &ConformanceBrowser{}, "plan1", nil)
+
+	require.NoError(t, runner.interact(ctx, "m1", ConformanceOverride{}))
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.Equal(t, 2, browserPolls, "the browser status must be read again once the module is back in WAITING")
+}
+
+func TestConformanceRunner_ReportsTheStatusOfAModuleWhichRanOutOfTime(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/runner":
+			w.WriteHeader(http.StatusCreated)
+			_, _ = w.Write([]byte(`{"id":"m1"}`))
+		case "/api/runner/m1/wait-state":
+			if strings.Contains(r.URL.Query().Get("states"), conformanceStatusConfigured) {
+				_, _ = w.Write([]byte(`{"state":"FINISHED"}`))
+
+				return
+			}
+
+			time.Sleep(time.Millisecond * 10)
+
+			_, _ = w.Write([]byte(`{"timeout":true}`))
+		case "/api/info/m1":
+			_, _ = w.Write([]byte(`{"_id":"m1","status":"WAITING"}`))
+		default:
+			t.Errorf("unexpected path %s", r.URL.Path)
+		}
+	}))
+
+	defer server.Close()
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	var outcomes []ConformanceOutcome
+
+	for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) {
+		outcomes = append(outcomes, outcome)
+	}
+
+	require.Len(t, outcomes, 1)
+	require.Error(t, outcomes[0].Err)
+
+	assert.Equal(t, conformanceStatusWaiting, outcomes[0].Status)
+}
+
+func TestConformanceLegs_Stalled(t *testing.T) {
+	t.Run("ShouldNameAutheliasErrorWhenTheLastLegEndedOnItsErrorPage", func(t *testing.T) {
+		legs := &conformanceLegs{errorURL: "https://login.example.com:8080/consent/completion?error=invalid_request_object&error_description=The+request+parameter+contains+an+invalid+Request+Object.&error_hint=Could+not+be+validated.&error_debug=Expected+typ+JWT."}
+
+		err := legs.stalled("m1")
+
+		var stall *ConformanceErrorPageStallError
+
+		require.ErrorAs(t, err, &stall)
+		assert.EqualError(t, err, "module 'm1' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: "+
+			"error 'invalid_request_object', description 'The request parameter contains an invalid Request Object.', hint 'Could not be validated.', debug 'Expected typ JWT.'")
+	})
+
+	t.Run("ShouldFallBackToTheURLWhenTheErrorPageCarriesNoParameters", func(t *testing.T) {
+		legs := &conformanceLegs{errorURL: "https://login.example.com:8080/"}
+
+		assert.EqualError(t, legs.stalled("m1"), "module 'm1' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: https://login.example.com:8080/")
+	})
+
+	t.Run("ShouldReportAPlainStallOtherwise", func(t *testing.T) {
+		legs := &conformanceLegs{}
+
+		err := legs.stalled("m1")
+
+		var stall *ConformanceErrorPageStallError
+
+		assert.False(t, errors.As(err, &stall))
+		assert.EqualError(t, err, "module 'm1' stayed in the WAITING state with nothing left to visit or fill")
+	})
+}
+
+func TestConformanceOverride_ScreenshotOf(t *testing.T) {
+	leg := func(index int) ConformanceLeg {
+		return ConformanceLeg{Index: index, LoginScreenshot: fmt.Sprintf("login-%d", index), ErrorScreenshot: fmt.Sprintf("error-%d", index)}
+	}
+
+	reauthentication := ConformanceOverride{Screenshot: ConformanceScreenshotReauthentication}
+
+	assert.Empty(t, reauthentication.screenshotOf(leg(0)))
+	assert.Equal(t, "login-1", reauthentication.screenshotOf(leg(1)))
+
+	errorPage := ConformanceOverride{Screenshot: ConformanceScreenshotErrorPage}
+
+	assert.Equal(t, "error-0", errorPage.screenshotOf(leg(0)))
+	assert.Empty(t, errorPage.screenshotOf(ConformanceLeg{Index: 0, LoginScreenshot: "login-0"}))
+
+	assert.Empty(t, ConformanceOverride{}.screenshotOf(leg(1)), "a module with no screenshot kind is given none")
+}
+
+func TestConformanceOverrides_ScreenshotEachPlaceholderModule(t *testing.T) {
+	expected := map[string]ConformanceScreenshot{
+		"oidcc-prompt-login": ConformanceScreenshotReauthentication,
+		"oidcc-max-age-1":    ConformanceScreenshotReauthentication,
+
+		"oidcc-ensure-registered-redirect-uri":          ConformanceScreenshotErrorPage,
+		"oidcc-ensure-request-object-with-redirect-uri": ConformanceScreenshotErrorPage,
+	}
+
+	actual := map[string]ConformanceScreenshot{}
+
+	for module, override := range conformanceOverrides {
+		if override.Screenshot != ConformanceScreenshotNone {
+			actual[module] = override.Screenshot
+		}
+	}
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestConformanceOverrides_RequestObjectModules(t *testing.T) {
+	_, ok := conformanceOverrides["oidcc-unsigned-request-object-supported-correctly-or-rejected-as-unsupported"]
+	assert.False(t, ok)
+
+	override := conformanceOverrides["oidcc-ensure-request-object-with-redirect-uri"]
+
+	assert.Nil(t, override.Assert, "either outcome is accepted by the module")
+	assert.Equal(t, "error", override.screenshotOf(ConformanceLeg{AutheliaError: true, ErrorScreenshot: "error"}))
+	assert.Empty(t, override.screenshotOf(ConformanceLeg{}), "a leg which reached the client has no error page to capture")
+}
+
+func TestConformanceLegs_StalledOnAPlaceholderWithoutAScreenshot(t *testing.T) {
+	legs := &conformanceLegs{pendingPlaceholder: true, errorURL: "https://login.example.com:8080/consent/completion?error=invalid_request"}
+
+	err := legs.stalled("m1")
+
+	var stall *ConformanceErrorPageStallError
+
+	assert.False(t, errors.As(err, &stall), "a module waiting on its placeholder is not stranded for want of one")
+	assert.EqualError(t, err, "module 'm1' is waiting for an evidence image, but no leg showed the page it asks for")
+}
+
+func TestConformanceRunner_LetsAFailedModuleFinishBeforeMovingOn(t *testing.T) {
+	testCases := []struct {
+		name     string
+		settle   string
+		expected string
+	}{
+		{"ShouldWaitForAModuleWhichIsFinishing", `{"state":"FINISHED"}`, "FINISHED"},
+		{"ShouldNotWaitOnAModuleWhichIsWaitingOnTheSuite", `{"state":"WAITING"}`, "WAITING"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				mu      sync.Mutex
+				settled []string
+			)
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				defer mu.Unlock()
+
+				switch r.URL.Path {
+				case "/api/runner":
+					w.WriteHeader(http.StatusCreated)
+					_, _ = w.Write([]byte(`{"id":"m1"}`))
+				case "/api/runner/m1/wait-state":
+					states := r.URL.Query().Get("states")
+
+					if states == "CONFIGURED,WAITING,FINISHED,INTERRUPTED" && len(settled) == 0 && !strings.Contains(r.URL.RawQuery, "settle") {
+						settled = append(settled, "create:"+states)
+
+						_, _ = w.Write([]byte(`{"state":"WAITING"}`))
+
+						return
+					}
+
+					settled = append(settled, "settle:"+states)
+
+					_, _ = w.Write([]byte(tc.settle))
+				case "/api/info/m1":
+					_, _ = w.Write([]byte(`{"_id":"m1","status":"` + tc.expected + `","result":"FAILED"}`))
+				default:
+					t.Errorf("unexpected path %s", r.URL.Path)
+				}
+			}))
+
+			defer server.Close()
+
+			client, err := NewConformanceClient(server.URL)
+			require.NoError(t, err)
+
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
+			var outcomes []ConformanceOutcome
+
+			for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) {
+				outcomes = append(outcomes, outcome)
+			}
+
+			require.Len(t, outcomes, 1)
+			require.Error(t, outcomes[0].Err)
+			assert.Equal(t, tc.expected, outcomes[0].Status)
+
+			mu.Lock()
+			defer mu.Unlock()
+
+			assert.Equal(t, []string{"create:CONFIGURED,WAITING,FINISHED,INTERRUPTED", "settle:CONFIGURED,WAITING,FINISHED,INTERRUPTED"}, settled)
+		})
+	}
+}

internal/suites/conformance_selectors_test.go

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceSelectorsExistInThePortal(t *testing.T) {
+	portal := conformancePortalSource(t)
+
+	for _, tc := range []struct {
+		name     string
+		selector string
+	}{
+		{"FirstFactorStage", conformanceSelectorFirstFactor},
+		{"ConsentStage", conformanceSelectorConsent},
+		{"ConsentAccept", conformanceSelectorConsentAccept},
+		{"ConsentAuthenticate", conformanceSelectorConsentAuthenticate},
+		{"ConsentReauthenticationField", conformanceSelectorConsentReauthentication},
+		{"ConsentReauthenticationPassword", conformanceSelectorConsentPassword},
+		{"Username", conformanceSelectorUsername},
+		{"Password", conformanceSelectorPassword},
+		{"SignIn", conformanceSelectorSignIn},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			for _, id := range conformanceSelectorIdentifiers(tc.selector) {
+				assert.Containsf(t, portal, id, "the portal no longer renders anything with the id %q, which %q depends on", id, tc.selector)
+			}
+		})
+	}
+
+	t.Run("AutheliaErrorToast", func(t *testing.T) {
+		assert.Contains(t, portal, "notification", "the toast no longer carries the notification class")
+		assert.Contains(t, portal, "data-[type=error]", "the toast no longer distinguishes an error by data-type")
+	})
+
+	t.Run("AutheliaCompletionError", func(t *testing.T) {
+		assert.Contains(t, portal, `data-testid={"openid-completion-outcome"}`, "the completion view no longer carries its outcome test id")
+		assert.Contains(t, portal, "data-outcome={outcome}", "the completion view no longer exposes its outcome")
+		assert.Contains(t, portal, `error ? "error"`, "the completion view no longer names the error outcome 'error'")
+	})
+}
+
+func conformanceSelectorIdentifiers(selector string) (ids []string) {
+	for _, part := range strings.Fields(selector) {
+		if !strings.HasPrefix(part, "#") {
+			continue
+		}
+
+		ids = append(ids, strings.TrimPrefix(part, "#"))
+	}
+
+	return ids
+}
+
+func conformancePortalSource(t *testing.T) string {
+	t.Helper()
+
+	builder := &strings.Builder{}
+
+	root, err := os.OpenRoot("../../web/src")
+	require.NoError(t, err)
+
+	defer root.Close()
+
+	require.NoError(t, fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if entry.IsDir() || !strings.HasSuffix(path, ".tsx") {
+			return nil
+		}
+
+		data, err := root.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		builder.Write(data)
+
+		return nil
+	}))
+
+	require.NotEmpty(t, builder.String(), "no portal sources were read, so this test proves nothing")
+
+	return builder.String()
+}

internal/suites/conformance_settle_test.go

@@ -0,0 +1,106 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceRunner_AwaitPlaceholderSettledReturnsAsSoonAsTheStatusChanges(t *testing.T) {
+	client, calls, closeServer := conformanceSettleServer(t, 2)
+	defer closeServer()
+
+	runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+	start := time.Now()
+
+	status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Millisecond*20, time.Second*2)
+	require.NoError(t, err)
+
+	assert.Equal(t, "FINISHED", status)
+	assert.Equal(t, int32(3), calls.Load(), "polls until it changes and then stops")
+	assert.Less(t, time.Since(start), time.Second, "does not wait out the whole window once it has an answer")
+}
+
+func TestConformanceRunner_AwaitPlaceholderSettledChecksBeforeWaitingAtAll(t *testing.T) {
+	client, calls, closeServer := conformanceSettleServer(t, 0)
+	defer closeServer()
+
+	runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+	start := time.Now()
+
+	status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Second*5, time.Second*30)
+	require.NoError(t, err)
+
+	assert.Equal(t, "FINISHED", status)
+	assert.Equal(t, int32(1), calls.Load())
+	assert.Less(t, time.Since(start), time.Second, "a module which already moved on is not made to wait for the interval")
+}
+
+func TestConformanceRunner_AwaitPlaceholderSettledGivesUpAtTheTimeoutAndReportsWhatItSaw(t *testing.T) {
+	client, calls, closeServer := conformanceSettleServer(t, 1000)
+	defer closeServer()
+
+	runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+	status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Millisecond*20, time.Millisecond*100)
+	require.NoError(t, err)
+
+	assert.Equal(t, conformanceStatusWaiting, status, "reports the status it last saw rather than erroring")
+	assert.Greater(t, calls.Load(), int32(2), "polled repeatedly across the window")
+}
+
+func TestConformanceRunner_AwaitPlaceholderSettledStopsWhenTheContextExpires(t *testing.T) {
+	client, _, closeServer := conformanceSettleServer(t, 1000)
+	defer closeServer()
+
+	runner := NewConformanceRunner(client, nil, "plan1", nil)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50)
+	defer cancel()
+
+	_, err := runner.awaitPlaceholderSettled(ctx, "m1", time.Millisecond*20, time.Minute)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "m1")
+	assert.Contains(t, err.Error(), "evidence image")
+}
+
+func TestConformanceUploadSettleMatchesTheIntendedCadence(t *testing.T) {
+	assert.Equal(t, time.Second, conformanceUploadSettleInterval)
+	assert.Equal(t, time.Second*10, conformanceUploadSettleTimeout)
+}
+
+func conformanceSettleServer(t *testing.T, waits int32) (client *ConformanceClient, calls *atomic.Int32, close func()) {
+	t.Helper()
+
+	calls = &atomic.Int32{}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/info/m1", r.URL.Path)
+
+		status := "FINISHED"
+
+		if calls.Add(1) <= waits {
+			status = "WAITING"
+		}
+
+		_, _ = fmt.Fprintf(w, `{"_id":"m1","status":%q,"result":"REVIEW"}`, status)
+	}))
+
+	client, err := NewConformanceClient(server.URL)
+	require.NoError(t, err)
+
+	return client, calls, server.Close
+}

internal/suites/conformance_trace.go

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"fmt"
+	"time"
+)
+
+// ConformanceTrace records what the suite did for one module, for that module's subtest to log. The runner works on
+// its own goroutine, which outlives any one subtest, so it records rather than logging directly. It records nothing
+// unless the suite was asked to debug.
+type ConformanceTrace struct {
+	enabled bool
+	lines   []string
+}
+
+// NewConformanceTrace returns a ConformanceTrace which records only when enabled.
+func NewConformanceTrace(enabled bool) *ConformanceTrace {
+	return &ConformanceTrace{enabled: enabled}
+}
+
+// Logf records a line, stamped with the time in UTC so it can be lined up with the container logs.
+func (t *ConformanceTrace) Logf(format string, args ...any) {
+	if t == nil || !t.enabled {
+		return
+	}
+
+	t.lines = append(t.lines, time.Now().UTC().Format("15:04:05.000")+" "+fmt.Sprintf(format, args...))
+}
+
+// Lines returns the recorded lines.
+func (t *ConformanceTrace) Lines() []string {
+	if t == nil {
+		return nil
+	}
+
+	return t.lines
+}

internal/suites/conformance_trace_test.go

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConformanceTrace(t *testing.T) {
+	t.Run("ShouldRecordNothingUnlessDebugging", func(t *testing.T) {
+		trace := NewConformanceTrace(false)
+
+		trace.Logf("driving leg %d", 0)
+
+		assert.Empty(t, trace.Lines())
+	})
+
+	t.Run("ShouldRecordEachLineStampedWithTheTime", func(t *testing.T) {
+		trace := NewConformanceTrace(true)
+
+		trace.Logf("driving leg %d", 0)
+		trace.Logf("finished leg %d", 0)
+
+		lines := trace.Lines()
+
+		require.Len(t, lines, 2)
+		assert.Regexp(t, regexp.MustCompile(`^\d{2}:\d{2}:\d{2}\.\d{3} driving leg 0$`), lines[0])
+		assert.Regexp(t, regexp.MustCompile(`^\d{2}:\d{2}:\d{2}\.\d{3} finished leg 0$`), lines[1])
+	})
+
+	t.Run("ShouldBeSafeWithoutATrace", func(t *testing.T) {
+		var trace *ConformanceTrace
+
+		assert.NotPanics(t, func() { trace.Logf("driving leg %d", 0) })
+		assert.Empty(t, trace.Lines())
+	})
+}
+
+func TestSuiteDebug(t *testing.T) {
+	t.Setenv("SUITE_DEBUG", "true")
+	assert.True(t, suiteDebug())
+
+	t.Setenv("SUITE_DEBUG", "")
+	assert.False(t, suiteDebug())
+}
+
+func TestConformanceRunner_TracesEachModuleOnlyWhenDebugging(t *testing.T) {
+	for _, debug := range []bool{false, true} {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/api/runner":
+				w.WriteHeader(http.StatusCreated)
+				_, _ = w.Write([]byte(`{"id":"m1"}`))
+			case "/api/runner/m1/wait-state":
+				_, _ = w.Write([]byte(`{"state":"FINISHED"}`))
+			case "/api/info/m1":
+				_, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED","result":"PASSED"}`))
+			default:
+				t.Errorf("unexpected path %s", r.URL.Path)
+			}
+		}))
+
+		client, err := NewConformanceClient(server.URL)
+		require.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+
+		runner := NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}})
+		runner.debug = debug
+
+		var outcomes []ConformanceOutcome
+
+		for outcome := range runner.Run(ctx) {
+			outcomes = append(outcomes, outcome)
+		}
+
+		cancel()
+		server.Close()
+
+		require.Len(t, outcomes, 1)
+
+		if debug {
+			require.NotEmpty(t, outcomes[0].Trace, "a debugging run records what the module did")
+			assert.Contains(t, outcomes[0].Trace[0], "created as 'm1' in state 'FINISHED'")
+		} else {
+			assert.Empty(t, outcomes[0].Trace, "a run which is not debugging records nothing")
+		}
+	}
+}

internal/suites/conformance_users_test.go

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/authelia/authelia/v4/internal/authentication"
+)
+
+func TestOIDCConformanceUserSuppliesEveryStandardClaim(t *testing.T) {
+	database := authentication.NewFileUserDatabase("OIDCConformance/users.yml", false, false, nil)
+
+	require.NoError(t, database.Load())
+
+	user, err := database.GetUserDetails(testUsername)
+	require.NoError(t, err)
+
+	details := user.ToExtendedUserDetails()
+
+	// scope=profile, less the claims Authelia derives rather than reads: preferred_username comes from the username
+	// and updated_at is generated.
+	assert.NotEmpty(t, details.GetDisplayName(), "backs the 'name' claim")
+	assert.NotEmpty(t, details.GetGivenName(), "backs the 'given_name' claim")
+	assert.NotEmpty(t, details.GetFamilyName(), "backs the 'family_name' claim")
+	assert.NotEmpty(t, details.GetMiddleName(), "backs the 'middle_name' claim")
+	assert.NotEmpty(t, details.GetNickname(), "backs the 'nickname' claim")
+	assert.NotEmpty(t, details.GetProfile(), "backs the 'profile' claim")
+	assert.NotEmpty(t, details.GetPicture(), "backs the 'picture' claim")
+	assert.NotEmpty(t, details.GetWebsite(), "backs the 'website' claim")
+	assert.NotEmpty(t, details.GetGender(), "backs the 'gender' claim")
+	assert.NotEmpty(t, details.GetBirthdate(), "backs the 'birthdate' claim")
+	assert.NotEmpty(t, details.GetZoneInfo(), "backs the 'zoneinfo' claim")
+	assert.NotEmpty(t, details.GetLocale(), "backs the 'locale' claim")
+
+	// scope=email, less email_verified which Authelia always reports.
+	assert.NotEmpty(t, details.GetEmails(), "backs the 'email' claim")
+
+	// scope=phone. Authelia emits phone_number_verified only when a number is present, so the number carries both
+	// claims of this scope.
+	assert.NotEmpty(t, details.GetPhoneNumberRFC3966(), "backs the 'phone_number' and 'phone_number_verified' claims")
+
+	// scope=address, whose single claim is an object Authelia assembles from these five.
+	assert.NotEmpty(t, details.GetStreetAddress(), "backs the 'address' claim")
+	assert.NotEmpty(t, details.GetLocality(), "backs the 'address' claim")
+	assert.NotEmpty(t, details.GetRegion(), "backs the 'address' claim")
+	assert.NotEmpty(t, details.GetPostalCode(), "backs the 'address' claim")
+	assert.NotEmpty(t, details.GetCountry(), "backs the 'address' claim")
+}

internal/suites/coverage_test.go

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteCoverage(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), ".nyc_output")
+
+	require.NoError(t, writeCoverage(dir, `{"/node/src/app/src/index.tsx":{"path":"/node/src/app/src/index.tsx"}}`))
+	require.NoError(t, writeCoverage(dir, `{}`))
+
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	require.Len(t, entries, 2, "each document's coverage is written to a file of its own, so the report merges them rather than one replacing another")
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	web := strings.TrimSuffix(wd, "internal/suites") + "web/"
+
+	var found bool
+
+	for _, entry := range entries {
+		assert.True(t, strings.HasPrefix(entry.Name(), "coverage-") && strings.HasSuffix(entry.Name(), ".json"), entry.Name())
+
+		data, err := os.ReadFile(filepath.Join(dir, entry.Name()))
+		require.NoError(t, err)
+
+		if strings.Contains(string(data), "index.tsx") {
+			found = true
+
+			assert.NotContains(t, string(data), "/node/src/app/", "the paths the frontend was built at must be rewritten to this checkout")
+			assert.Contains(t, string(data), web+"src/index.tsx")
+		}
+	}
+
+	assert.True(t, found)
+}
+
+func TestConformanceCoverageScript(t *testing.T) {
+	assert.Contains(t, conformanceCoverageScript, "'beforeunload'")
+	assert.Contains(t, conformanceCoverageScript, "window."+conformanceCoverageBinding+"(")
+	assert.Contains(t, conformanceCoverageScript, "window.__coverage__")
+}
+
+func TestWriteCoverageReportsADirectoryItCannotCreate(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "file")
+
+	require.NoError(t, os.WriteFile(file, nil, 0600))
+
+	assert.Error(t, writeCoverage(filepath.Join(file, ".nyc_output"), `{}`))
+}
+
+func TestEmitBuildkiteMarker(t *testing.T) {
+	capture := func(t *testing.T, marker string) string {
+		t.Helper()
+
+		r, w, err := os.Pipe()
+		require.NoError(t, err)
+
+		stdout := os.Stdout
+		os.Stdout = w
+
+		emitBuildkiteMarker(marker)
+
+		os.Stdout = stdout
+
+		require.NoError(t, w.Close())
+
+		data, err := io.ReadAll(r)
+		require.NoError(t, err)
+
+		return string(data)
+	}
+
+	assert.Equal(t, "--- OIDC Conformance Plan: basic\n", capture(t, "--- OIDC Conformance Plan: basic"))
+	assert.Empty(t, capture(t, ""), "no marker is written outside Buildkite")
+}

internal/suites/example/compose/nginx/portal/compose.yml

@@ -16,6 +16,7 @@ services:
           - secure.example.com
           - login.example.com
           - duo.example.com
+          - conformance.example.com
         # Set the IP to be able to query on port 443
         ipv4_address: ${SUITE_SUBNET:-192.168.240}.100
 ...

internal/suites/example/compose/nginx/portal/nginx.conf

@@ -450,6 +450,142 @@ http {
         }
     }
 
+    # OpenID Connect 1.0 conformance suite. This replaces the nginx image the conformance project publishes, whose
+    # configuration is baked in and only answers on that container's own address. The three listeners below mirror
+    # that image: 8443 is the ordinary one, 8444 asks for a client certificate so the mutual TLS tests have somewhere
+    # to land, and 8445 narrows the ciphers to the FAPI 2.0 set. Only 8443 is exercised today, but a suite that grows
+    # a profile reaching for the other two finds them where it expects rather than failing part way through a plan.
+    #
+    # Each listener repeats the proxy body rather than sharing one: nginx would need an include file mounted beside
+    # this one to factor it out, and the ports differ in the handshake, not in what they forward. The buffer sizes,
+    # timeouts and forwarded headers are what the published image sets and the suite depends on, so they are
+    # reproduced here rather than left to the defaults the blocks above share.
+    server {
+        listen 8443 ssl;
+        server_name ~^conformance\.example([0-9])*\.com$;
+
+        # The mutual TLS tests have to arrive on a listener that asks for a client certificate, which this one does
+        # not, so the suite is told it reached the wrong host and reports that rather than timing out.
+        ssl_verify_client off;
+
+        resolver 127.0.0.11 ipv6=off;
+        set $upstream_endpoint http://conformance-server:8080;
+
+        ssl_certificate     /pki/public.chain.pem;
+        ssl_certificate_key /pki/private.pem;
+        ssl_protocols       TLSv1.2 TLSv1.3;
+
+        # The suite sends request objects and authorization requests that overflow the default 1k header buffer.
+        client_header_buffer_size 32k;
+        large_client_header_buffers 4 32k;
+
+        location / {
+            # The suite has requests that block waiting for callbacks for up to 60s, which the default 60s proxy
+            # timeout races against and turns into sporadic 504s.
+            proxy_read_timeout 120s;
+            proxy_send_timeout 120s;
+
+            set $mtls_wrong_host "";
+            if ($request_uri ~* "^/test-mtls") {
+                set $mtls_wrong_host "true";
+            }
+            proxy_set_header  X-Test-Mtls-Called-On-Wrong-Host $mtls_wrong_host;
+
+            # Host is deliberately $host rather than $http_host: the suite rebuilds its own base URL from
+            # X-Forwarded-Host and X-Forwarded-Port, so a port carried in either of the first two would be doubled.
+            proxy_set_header  Host $host;
+            proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header  X-Forwarded-Proto $scheme;
+            proxy_set_header  X-Forwarded-Host $host;
+            proxy_set_header  X-Forwarded-Port $server_port;
+            proxy_set_header  X-Forwarded-Uri $request_uri;
+            proxy_set_header  X-Ssl-Cipher $ssl_cipher;
+            proxy_set_header  X-Ssl-Protocol $ssl_protocol;
+            proxy_set_header  X-Ssl-Cert $ssl_client_cert;
+            proxy_set_header  Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme";
+
+            proxy_pass_request_headers on;
+            proxy_pass        $upstream_endpoint;
+        }
+    }
+
+    # Mutual TLS. The client certificate is requested but not checked against a CA: the suite inspects it itself out
+    # of X-Ssl-Cert, which is why this is optional_no_ca rather than on.
+    server {
+        listen 8444 ssl;
+        server_name ~^conformance\.example([0-9])*\.com$;
+
+        ssl_verify_client optional_no_ca;
+
+        resolver 127.0.0.11 ipv6=off;
+        set $upstream_endpoint http://conformance-server:8080;
+
+        ssl_certificate     /pki/public.chain.pem;
+        ssl_certificate_key /pki/private.pem;
+        ssl_protocols       TLSv1.2 TLSv1.3;
+
+        client_header_buffer_size 32k;
+        large_client_header_buffers 4 32k;
+
+        location / {
+            proxy_read_timeout 120s;
+            proxy_send_timeout 120s;
+
+            proxy_set_header  Host $host;
+            proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header  X-Forwarded-Proto $scheme;
+            proxy_set_header  X-Forwarded-Host $host;
+            proxy_set_header  X-Forwarded-Port $server_port;
+            proxy_set_header  X-Forwarded-Uri $request_uri;
+            proxy_set_header  X-Ssl-Cipher $ssl_cipher;
+            proxy_set_header  X-Ssl-Protocol $ssl_protocol;
+            proxy_set_header  X-Ssl-Cert $ssl_client_cert;
+            proxy_set_header  Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme";
+
+            proxy_pass_request_headers on;
+            proxy_pass        $upstream_endpoint;
+        }
+    }
+
+    # The FAPI 2.0 Final security profile cipher set.
+    server {
+        listen 8445 ssl;
+        server_name ~^conformance\.example([0-9])*\.com$;
+
+        ssl_verify_client optional_no_ca;
+        ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384;
+        ssl_prefer_server_ciphers on;
+
+        resolver 127.0.0.11 ipv6=off;
+        set $upstream_endpoint http://conformance-server:8080;
+
+        ssl_certificate     /pki/public.chain.pem;
+        ssl_certificate_key /pki/private.pem;
+        ssl_protocols       TLSv1.2 TLSv1.3;
+
+        client_header_buffer_size 32k;
+        large_client_header_buffers 4 32k;
+
+        location / {
+            proxy_read_timeout 120s;
+            proxy_send_timeout 120s;
+
+            proxy_set_header  Host $host;
+            proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header  X-Forwarded-Proto $scheme;
+            proxy_set_header  X-Forwarded-Host $host;
+            proxy_set_header  X-Forwarded-Port $server_port;
+            proxy_set_header  X-Forwarded-Uri $request_uri;
+            proxy_set_header  X-Ssl-Cipher $ssl_cipher;
+            proxy_set_header  X-Ssl-Protocol $ssl_protocol;
+            proxy_set_header  X-Ssl-Cert $ssl_client_cert;
+            proxy_set_header  Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme";
+
+            proxy_pass_request_headers on;
+            proxy_pass        $upstream_endpoint;
+        }
+    }
+
     # Fake Web Mail used to receive emails sent by Authelia.
     server {
         listen 8080 ssl;

internal/suites/hosts.go

@@ -57,6 +57,9 @@ func HostEntries() []HostEntry {
 		{Domain: "oidc.example.com", IP: portal},
 		{Domain: "oidc-public.example.com", IP: portal},
 
+		// OpenID Connect 1.0 conformance suite.
+		{Domain: "conformance.example.com", IP: portal},
+
 		// For Traefik suite.
 		{Domain: "traefik.example.com", IP: portal},
 

internal/suites/pngdiff.go

@@ -119,7 +119,7 @@ func AssertVisualSnapshot(t *testing.T, repoRoot, name string, screenshot []byte
 	}
 
 	if diff.Percentage <= tolerancePercentage {
-		t.Logf("snapshot %s differs by %d/%d pixels (%.2f%%) — within tolerance (%.2f%%)",
+		t.Logf("snapshot %s differs by %d/%d pixels (%.2f%%) - within tolerance (%.2f%%)",
 			baselinePath, diff.DifferingPixels, diff.TotalPixels, diff.Percentage, tolerancePercentage)
 
 		_ = os.Remove(actualPath)

internal/suites/suite_activedirectory.go

@@ -45,11 +45,12 @@ func init() {
 
 	GlobalRegistry.Register(activedirectorySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     120 * time.Second,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia against an Active Directory implementation of LDAP, backed by a Samba domain controller.",
 	})
 }

internal/suites/suite_bypass_all.go

@@ -44,12 +44,12 @@ func init() {
 
 	GlobalRegistry.Register(bypassAllSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     1 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description:     "This suite has been created to test Authelia with a bypass policy on all resources",
 	})
 }

internal/suites/suite_caddy.go

@@ -58,11 +58,12 @@ func init() {
 
 	GlobalRegistry.Register(caddySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     150 * time.Second,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia as an authorization server behind the Caddy reverse proxy.",
 	})
 }

internal/suites/suite_cli.go

@@ -44,11 +44,13 @@ func init() {
 
 	GlobalRegistry.Register(cliSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     3 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description: `This suite has been created to test the Authelia command line interface, covering the
+configuration validation, password hashing, certificate generation, storage and access control subcommands.`,
 	})
 }

internal/suites/suite_docker.go

@@ -43,13 +43,12 @@ func init() {
 
 	GlobalRegistry.Register(dockerSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     1 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
-
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite has been created to test the distributable version of Authelia
 It's often useful to test this one before the Kube one.`,
 	})

internal/suites/suite_duo_push.go

@@ -47,13 +47,12 @@ func init() {
 
 	GlobalRegistry.Register(duoPushSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     4 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
-
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite has been created to test Authelia against
 the Duo API for push notifications. It allows a user to validate second factor
 with a mobile phone.`,

internal/suites/suite_envoy.go

@@ -58,11 +58,12 @@ func init() {
 
 	GlobalRegistry.Register(envoySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     150 * time.Second,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia as an authorization server behind the Envoy reverse proxy.",
 	})
 }

internal/suites/suite_haproxy.go

@@ -58,11 +58,12 @@ func init() {
 
 	GlobalRegistry.Register(haproxySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia as an authorization server behind the HAProxy reverse proxy.",
 	})
 }

internal/suites/suite_high_availability.go

@@ -48,12 +48,12 @@ func init() {
 
 	GlobalRegistry.Register(highAvailabilitySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     6 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite is made to test Authelia in a *complete*
 environment, that is, with all components making Authelia highly available.`,
 	})

internal/suites/suite_kubernetes.go

@@ -102,10 +102,10 @@ func init() {
 
 	GlobalRegistry.Register(kubernetesSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    12 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description:     "This suite has been created to test Authelia in a Kubernetes context and using Traefik as the ingress controller.",
 	})
 }

internal/suites/suite_ldap.go

@@ -46,11 +46,13 @@ func init() {
 
 	GlobalRegistry.Register(ldapSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     150 * time.Second,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		TearDownTimeout: 1 * time.Minute,
+		Description: `This suite has been created to test Authelia against an OpenLDAP backend, including the
+phpLDAPadmin administration interface.`,
 	})
 }

internal/suites/suite_mariadb.go

@@ -46,10 +46,12 @@ func init() {
 
 	GlobalRegistry.Register(mariadbSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
+		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia with MariaDB as the storage backend and LDAP as the user backend.",
 	})
 }

internal/suites/suite_multi_cookie_domain.go

@@ -44,12 +44,12 @@ func init() {
 
 	GlobalRegistry.Register(multiCookieDomainSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
-		TearDown:        teardown,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     4 * time.Minute,
-		TearDownTimeout: 2 * time.Minute,
+		TearDown:        teardown,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite is used to test Authelia in a multi cookie domain
 configuration with in-memory sessions and a local sqlite db stored on disk`,
 	})

internal/suites/suite_mysql.go

@@ -46,10 +46,12 @@ func init() {
 
 	GlobalRegistry.Register(mysqlSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
+		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia with MySQL as the storage backend and LDAP as the user backend.",
 	})
 }

internal/suites/suite_network_acl.go

@@ -46,12 +46,12 @@ func init() {
 
 	GlobalRegistry.Register(networkACLSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     1 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite has been created to test Authelia with basic feature in a non highly-available setup.
 Authelia basically use an in-memory cache to store user sessions and persist data on disk instead
 of using a remote database. Also, the user accounts are stored in file-based database.`,

internal/suites/suite_oidc.go

@@ -52,11 +52,13 @@ func init() {
 
 	GlobalRegistry.Register(oidcSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description: `This suite has been created to test Authelia as an OpenID Connect 1.0 Provider using a
+client application registered against it, with Redis backed sessions.`,
 	})
 }

internal/suites/suite_oidc_conformance.go

@@ -0,0 +1,249 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"time"
+
+	"go.yaml.in/yaml/v4"
+
+	"github.com/authelia/authelia/v4/internal/configuration/schema"
+	"github.com/authelia/authelia/v4/internal/oidc/conformance"
+	"github.com/authelia/authelia/v4/internal/random"
+	"github.com/authelia/authelia/v4/internal/utils"
+)
+
+const (
+	oidcConformanceSuiteName          = "OIDCConformance"
+	oidcConformanceBaseURL            = "https://conformance.example.com:8443"
+	oidcConformanceAutheliaURL        = "https://login.example.com:8080"
+	oidcConformancePlansFile          = "conformance-plans.json"
+	oidcConformanceClientsFile        = "conformance-clients.yml"
+	oidcConformanceMongoDBHostEnv     = "SUITE_OIDC_CONFORMANCE_MONGODB_HOST"
+	oidcConformanceMongoDBComposeFile = "OIDCConformance/compose.mongodb.yml"
+)
+
+// OIDCConformancePlanFile is one entry of the plans file exchanged between the setup and test processes. The API plan
+// name and the variant are carried as their own fields rather than being read back off the embedded
+// [conformance.Plan]: that type tags both of them json:"-" because authelia-gen holds them in memory and sends them as
+// query parameters while writing only the plan body to disk, so a round trip through this file would otherwise decode
+// them as their zero values and the suite would create every plan nameless and unvarianted.
+type OIDCConformancePlanFile struct {
+	// Name is the builder name of the profile, which is what the suite's per profile test methods look plans up by.
+	Name string `json:"name"`
+
+	// PlanName is the conformance suite's own name for the plan, sent as the planName query parameter.
+	PlanName string `json:"plan_name"`
+
+	// Variant is the plan's variant, sent as the variant query parameter.
+	Variant *conformance.PlanVariant `json:"variant,omitempty"`
+
+	// Plan is the plan body, sent as the request body.
+	Plan conformance.Plan `json:"plan"`
+}
+
+type oidcConformanceClients struct {
+	IdentityProviders struct {
+		OIDC struct {
+			Clients []schema.IdentityProvidersOpenIDConnectClient `yaml:"clients"`
+		} `yaml:"oidc"`
+	} `yaml:"identity_providers"`
+}
+
+func oidcConformanceGenerate() (err error) {
+	suiteURL, err := url.ParseRequestURI(oidcConformanceBaseURL)
+	if err != nil {
+		return err
+	}
+
+	autheliaURL, err := url.ParseRequestURI(oidcConformanceAutheliaURL)
+	if err != nil {
+		return err
+	}
+
+	version := random.New().StringCustom(8, random.CharSetAlphabeticLower+random.CharSetNumeric)
+
+	clients := &oidcConformanceClients{}
+	clients.IdentityProviders.OIDC.Clients = []schema.IdentityProvidersOpenIDConnectClient{}
+
+	var plans []OIDCConformancePlanFile
+
+	release := oidcConformanceRelease(oidcConformanceRevision())
+
+	for _, builder := range conformance.Builders(version, "implicit", "one_factor", "authelia", suiteURL, autheliaURL) {
+		suite := builder.Build()
+
+		if release != "" {
+			suite.Plan.Description = builder.Description(release)
+		}
+
+		clients.IdentityProviders.OIDC.Clients = append(clients.IdentityProviders.OIDC.Clients, suite.Clients...)
+		plans = append(plans, OIDCConformancePlanFile{Name: builder.Name, PlanName: suite.Plan.Name, Variant: suite.Plan.Variant, Plan: suite.Plan})
+	}
+
+	if err = oidcConformanceWriteYAML(SuiteTmpPath(oidcConformanceClientsFile), clients); err != nil {
+		return err
+	}
+
+	return oidcConformanceWriteJSON(SuiteTmpPath(oidcConformancePlansFile), plans)
+}
+
+func oidcConformanceRevision() (commit, tag string) {
+	commit, _, _ = utils.RunCommandAndReturnOutput("git rev-parse HEAD")
+	tag, _, _ = utils.RunCommandAndReturnOutput("git describe --tags --exact-match HEAD")
+
+	return commit, tag
+}
+
+func oidcConformanceRelease(commit, tag string) string {
+	switch {
+	case commit == "":
+		return ""
+	case tag == "":
+		return "commit " + commit
+	default:
+		return fmt.Sprintf("%s (commit %s)", tag, commit)
+	}
+}
+
+func oidcConformanceComposeFiles(mongodbHost string) (files []string) {
+	files = []string{
+		"internal/suites/compose.yml",
+		"internal/suites/OIDCConformance/compose.yml",
+		"internal/suites/example/compose/authelia/compose.backend.{}.yml",
+		"internal/suites/example/compose/authelia/compose.frontend.{}.yml",
+		"internal/suites/example/compose/nginx/backend/compose.yml",
+		"internal/suites/example/compose/nginx/portal/compose.yml",
+		"internal/suites/example/compose/smtp/compose.yml",
+		"internal/suites/example/compose/redis/compose.yml",
+		"internal/suites/example/compose/postgres/compose.yml",
+	}
+
+	if mongodbHost == "" {
+		files = append(files, "internal/suites/"+oidcConformanceMongoDBComposeFile)
+	}
+
+	return files
+}
+
+func oidcConformanceLogServices(mongodbHost string) (services []string) {
+	services = []string{"authelia-backend", "authelia-frontend", "postgres"}
+
+	if mongodbHost == "" {
+		services = append(services, "conformance-mongodb")
+	}
+
+	return append(services, "conformance-server")
+}
+
+func oidcConformanceWriteYAML(path string, value any) (err error) {
+	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if cerr := f.Close(); cerr != nil && err == nil {
+			err = fmt.Errorf("error closing '%s': %w", path, cerr)
+		}
+	}()
+
+	encoder := yaml.NewEncoder(f)
+	encoder.SetIndent(2)
+
+	defer func() {
+		if cerr := encoder.Close(); cerr != nil && err == nil {
+			err = fmt.Errorf("error flushing '%s': %w", path, cerr)
+		}
+	}()
+
+	return encoder.Encode(value)
+}
+
+func oidcConformanceWriteJSON(path string, value any) (err error) {
+	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if cerr := f.Close(); cerr != nil && err == nil {
+			err = fmt.Errorf("error closing '%s': %w", path, cerr)
+		}
+	}()
+
+	return json.NewEncoder(f).Encode(value)
+}
+
+func oidcConformanceReadPlans() (plans []OIDCConformancePlanFile, err error) {
+	data, err := os.ReadFile(SuiteTmpPath(oidcConformancePlansFile))
+	if err != nil {
+		return nil, fmt.Errorf("error reading the generated conformance plans, was the suite set up?: %w", err)
+	}
+
+	if err = json.Unmarshal(data, &plans); err != nil {
+		return nil, err
+	}
+
+	return plans, nil
+}
+
+func init() {
+	mongodbHost := os.Getenv(oidcConformanceMongoDBHostEnv)
+
+	dockerEnvironment := NewDockerEnvironment(oidcConformanceComposeFiles(mongodbHost))
+
+	setup := func(suitePath string) (err error) {
+		if err = oidcConformanceGenerate(); err != nil {
+			return err
+		}
+
+		if err = dockerEnvironment.Up(); err != nil {
+			return err
+		}
+
+		if err = waitUntilAutheliaIsReady(dockerEnvironment, oidcConformanceSuiteName); err != nil {
+			return err
+		}
+
+		client, err := NewConformanceClient(oidcConformanceBaseURL)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+		defer cancel()
+
+		if err = client.WaitReady(ctx); err != nil {
+			return err
+		}
+
+		return updateDevEnvFileForDomain(BaseDomain, dockerEnvironment)
+	}
+
+	displayLogs := func() error {
+		return dockerEnvironment.PrintLogs(oidcConformanceLogServices(mongodbHost)...)
+	}
+
+	teardown := func(suitePath string) error {
+		return dockerEnvironment.Down()
+	}
+
+	GlobalRegistry.Register(oidcConformanceSuiteName, Suite{
+		SetUp:           setup,
+		SetUpTimeout:    2 * time.Minute,
+		OnSetupTimeout:  displayLogs,
+		OnError:         displayLogs,
+		TestTimeout:     8 * time.Minute,
+		TearDown:        teardown,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite runs the OpenID Foundation conformance suite against Authelia for every profile Authelia is OpenID Certified for.",
+	})
+}

internal/suites/suite_oidc_conformance_test.go

@@ -0,0 +1,448 @@
+// SPDX-FileCopyrightText: 2026 Authelia
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suites
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.yaml.in/yaml/v4"
+
+	"github.com/authelia/authelia/v4/internal/oidc/conformance"
+	"github.com/authelia/authelia/v4/internal/utils"
+)
+
+var (
+	conformanceAcceptedResults  = []string{"PASSED", "REVIEW", "SKIPPED"}
+	conformanceAcceptedWarnings = map[string]string{
+		// ValidateIdTokenACRClaimAgainstAcrValuesRequest (OIDCC-3.1.2.1, OIDCC-15.1).
+		"oidcc-ensure-request-with-acr-values-succeeds": "Authelia does not select an acr from acr_values, which is a voluntary request, so the ID Token carries the fallback acr '0' rather than one of the requested values",
+	}
+)
+
+func conformanceResultAccepted(module, result string) bool {
+	if utils.IsStringInSlice(result, conformanceAcceptedResults) {
+		return true
+	}
+
+	_, ok := conformanceAcceptedWarnings[module]
+
+	return ok && result == "WARNING"
+}
+
+// conformanceOutcomeAccepted reports whether a module passes: the suite drove it without error, it was not interrupted,
+// and its result is accepted. A module is interrupted when it stops before running to completion, so its result says
+// nothing about the conformance of what it did not get to check.
+func conformanceOutcomeAccepted(outcome ConformanceOutcome) bool {
+	return outcome.Err == nil && outcome.Status != conformanceStatusInterrupted && conformanceResultAccepted(outcome.Module, outcome.Result)
+}
+
+const (
+	conformanceDiagnosticTimeout = time.Second * 30
+	conformancePlanTimeout       = time.Minute * 75
+	conformanceDrainTimeout      = time.Second * 45
+	conformanceArchiveDir        = "oidc-conformance-plans"
+)
+
+// OIDCConformanceSuite runs the OpenID Foundation conformance suite against Authelia, one test per certified profile.
+type OIDCConformanceSuite struct {
+	*RodSuite
+
+	client   *ConformanceClient
+	cancel   context.CancelFunc
+	browsers []*ConformanceBrowser
+	outcomes map[string]<-chan ConformanceOutcome
+	planIDs  map[string]string
+}
+
+// NewOIDCConformanceSuite returns a new *OIDCConformanceSuite.
+func NewOIDCConformanceSuite() *OIDCConformanceSuite {
+	return &OIDCConformanceSuite{
+		RodSuite: NewRodSuite(oidcConformanceSuiteName),
+		outcomes: map[string]<-chan ConformanceOutcome{},
+		planIDs:  map[string]string{},
+	}
+}
+
+// SetupSuite creates every plan and starts a runner per plan.
+func (s *OIDCConformanceSuite) SetupSuite() {
+	s.BaseSuite.SetupSuite()
+
+	complete := false
+
+	defer func() {
+		if !complete {
+			s.release()
+		}
+	}()
+
+	plans, err := oidcConformanceReadPlans()
+	s.Require().NoError(err)
+
+	s.client, err = NewConformanceClient(oidcConformanceBaseURL)
+	s.Require().NoError(err)
+
+	session, err := NewRodSession()
+	s.Require().NoError(err)
+
+	s.RodSession = session
+
+	ctx, cancel := context.WithTimeout(context.Background(), conformancePlanTimeout)
+
+	s.cancel = cancel
+
+	for _, plan := range plans {
+		plan := plan
+
+		created, err := s.client.CreatePlan(ctx, plan.PlanName, plan.Variant, &plan.Plan)
+		s.Require().NoErrorf(err, "error creating the '%s' plan", plan.Name)
+
+		s.planIDs[plan.Name] = created.ID
+
+		browser, err := NewConformanceBrowser(s.RodSession)
+		s.Require().NoError(err)
+
+		s.browsers = append(s.browsers, browser)
+
+		runner := NewConformanceRunner(s.client, browser, created.ID, created.Modules)
+		runner.debug = suiteDebug()
+
+		s.outcomes[plan.Name] = runner.Run(ctx)
+	}
+
+	complete = true
+}
+
+func (s *OIDCConformanceSuite) release() {
+	if s.cancel != nil {
+		s.cancel()
+	}
+
+	s.drainRunners()
+
+	for _, browser := range s.browsers {
+		browser.Close()
+	}
+
+	if s.RodSession != nil {
+		if err := s.Stop(); err != nil {
+			s.T().Logf("error stopping rod session: %v", err)
+		}
+	}
+}
+
+func (s *OIDCConformanceSuite) drainRunners() {
+	for name, outcomes := range s.outcomes {
+		timer := time.NewTimer(conformanceDrainTimeout)
+
+		for drained := false; !drained; {
+			select {
+			case _, ok := <-outcomes:
+				if !ok {
+					drained = true
+				}
+			case <-timer.C:
+				s.T().Logf("timed out after %s waiting for the '%s' plan's runner to finish", conformanceDrainTimeout, name)
+
+				drained = true
+			}
+		}
+
+		timer.Stop()
+	}
+}
+
+// TearDownSuite exports each plan's log for the CI artifacts, then releases the runners, browser contexts and rod
+// session via release.
+func (s *OIDCConformanceSuite) TearDownSuite() {
+	emitBuildkiteMarker(buildkiteGroup("OIDC Conformance Finalization: teardown"))
+
+	if s.client != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
+		defer cancel()
+
+		for name, id := range s.planIDs {
+			path := conformanceArchivePath(name)
+
+			if err := s.client.ExportPlanHTML(ctx, id, path); err != nil {
+				s.T().Logf("Error exporting the '%s' plan log: %v", name, err)
+
+				continue
+			}
+
+			s.T().Logf("Exported the '%s' plan log to '%s'", name, path)
+		}
+	}
+
+	s.release()
+}
+
+func (s *OIDCConformanceSuite) assertPlan(name string) {
+	outcomes, ok := s.outcomes[name]
+	s.Require().Truef(ok, "the '%s' plan was not created", name)
+
+	emitBuildkiteMarker(buildkiteGroup("OIDC Conformance Plan: " + name))
+
+	count, accepted, failed, skipped, expanded := 0, 0, 0, 0, false
+
+	for outcome := range outcomes {
+		outcome := outcome
+		count++
+
+		s.Run(outcome.Name, func() {
+			t := s.T()
+
+			if len(outcome.Trace) != 0 {
+				t.Log("Trace:\n" + strings.Join(outcome.Trace, "\n"))
+			}
+
+			if outcome.Skipped {
+				skipped++
+
+				t.Skipf("%s cannot run unattended: %s", outcome.Module, outcome.Reason)
+
+				return
+			}
+
+			if !conformanceOutcomeAccepted(outcome) {
+				failed++
+
+				s.reportFailure(t, name, outcome)
+
+				if !expanded {
+					emitBuildkiteMarker(buildkiteExpandGroup())
+
+					expanded = true
+				}
+			}
+
+			require.NoError(t, outcome.Err)
+			require.NotEqualf(t, conformanceStatusInterrupted, outcome.Status,
+				"module '%s' was interrupted with the result '%s' and the status '%s'", outcome.Module, outcome.Result, outcome.Status)
+			require.Truef(t, conformanceResultAccepted(outcome.Module, outcome.Result),
+				"module '%s' finished with the result '%s' and the status '%s'", outcome.Module, outcome.Result, outcome.Status)
+
+			accepted++
+
+			if outcome.Result == "WARNING" {
+				s.reportAcceptedWarning(outcome)
+			}
+		})
+	}
+
+	s.T().Log(conformancePlanSummary(name, accepted, failed, skipped))
+
+	s.Require().NotZerof(count, "the '%s' plan produced no modules", name)
+}
+
+func (s *OIDCConformanceSuite) reportAcceptedWarning(outcome ConformanceOutcome) {
+	reason := conformanceAcceptedWarnings[outcome.Module]
+
+	if s.client == nil || outcome.ID == "" {
+		s.T().Logf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s.", outcome.Module, outcome.Status, reason)
+
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), conformanceDiagnosticTimeout)
+	defer cancel()
+
+	entries, err := s.client.Log(ctx, outcome.ID)
+	if err != nil {
+		s.T().Logf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s. Its log could not be read: %v", outcome.Module, outcome.Status, reason, err)
+
+		return
+	}
+
+	s.T().Log(ConformanceAcceptedWarningReport(outcome.Module, outcome.Status, reason, entries))
+}
+
+func (s *OIDCConformanceSuite) reportFailure(t *testing.T, plan string, outcome ConformanceOutcome) {
+	t.Logf("Module '%s' of the '%s' plan finished with the result '%s' and the status '%s'.", outcome.Module, plan, outcome.Result, outcome.Status)
+
+	if outcome.Err != nil {
+		t.Logf("Suite error: %v", outcome.Err)
+	}
+
+	if outcome.ID == "" {
+		t.Log("The module was never created, so the conformance server has no log for it.")
+
+		return
+	}
+
+	if s.client == nil {
+		return
+	}
+
+	t.Logf("Module id %s, in %s", outcome.ID, conformanceArchiveReference(plan, outcome.LogURL))
+
+	ctx, cancel := context.WithTimeout(context.Background(), conformanceDiagnosticTimeout)
+	defer cancel()
+
+	entries, err := s.client.Log(ctx, outcome.ID)
+	if err != nil {
+		t.Logf("The module's log could not be read for this report: %v", err)
+
+		return
+	}
+
+	var stall *ConformanceErrorPageStallError
+
+	switch detail := ConformanceDiagnostics(entries); {
+	case detail != "":
+		t.Logf("Conformance log entries that did not pass:%s", detail)
+	case errors.As(outcome.Err, &stall):
+		t.Logf("The module logged no failing entries because it is still waiting for the callback Authelia's error page never sent.")
+	default:
+		t.Logf("The module logged no failing entries, which points at the suite driving it rather than at the provider.")
+	}
+}
+
+// TestConfig runs the Config OP certification profile.
+func (s *OIDCConformanceSuite) TestConfig() {
+	s.assertPlan(conformance.NameConfig)
+}
+
+// TestBasic runs the Basic OP certification profile.
+func (s *OIDCConformanceSuite) TestBasic() {
+	s.assertPlan(conformance.NameBasic)
+}
+
+// TestBasicFormPost runs the Basic OP certification profile with the form post response mode.
+func (s *OIDCConformanceSuite) TestBasicFormPost() {
+	s.assertPlan(conformance.NameBasicFormPost)
+}
+
+// TestHybrid runs the Hybrid OP certification profile.
+func (s *OIDCConformanceSuite) TestHybrid() {
+	s.assertPlan(conformance.NameHybrid)
+}
+
+// TestHybridFormPost runs the Hybrid OP certification profile with the form post response mode.
+func (s *OIDCConformanceSuite) TestHybridFormPost() {
+	s.assertPlan(conformance.NameHybridFormPost)
+}
+
+// TestImplicit runs the Implicit OP certification profile.
+func (s *OIDCConformanceSuite) TestImplicit() {
+	s.assertPlan(conformance.NameImplicit)
+}
+
+// TestImplicitFormPost runs the Implicit OP certification profile with the form post response mode.
+func (s *OIDCConformanceSuite) TestImplicitFormPost() {
+	s.assertPlan(conformance.NameImplicitFormPost)
+}
+
+func TestOIDCConformanceSuite(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping suite test in short mode")
+	}
+
+	suite.Run(t, NewOIDCConformanceSuite())
+}
+
+func TestOIDCConformanceGenerateRoundTrip(t *testing.T) {
+	t.Setenv("SUITE_TMP_PATH", t.TempDir())
+
+	require.NoError(t, oidcConformanceGenerate())
+
+	plans, err := oidcConformanceReadPlans()
+	require.NoError(t, err)
+
+	expected := map[string]struct {
+		plan    string
+		variant bool
+	}{
+		conformance.NameConfig:           {"oidcc-config-certification-test-plan", false},
+		conformance.NameBasic:            {"oidcc-basic-certification-test-plan", true},
+		conformance.NameBasicFormPost:    {"oidcc-formpost-basic-certification-test-plan", true},
+		conformance.NameHybrid:           {"oidcc-hybrid-certification-test-plan", true},
+		conformance.NameHybridFormPost:   {"oidcc-formpost-hybrid-certification-test-plan", true},
+		conformance.NameImplicit:         {"oidcc-implicit-certification-test-plan", true},
+		conformance.NameImplicitFormPost: {"oidcc-formpost-implicit-certification-test-plan", true},
+	}
+
+	require.Len(t, plans, len(expected))
+
+	data, err := os.ReadFile(SuiteTmpPath(oidcConformanceClientsFile))
+	require.NoError(t, err)
+
+	clients := &oidcConformanceClients{}
+	require.NoError(t, yaml.Unmarshal(data, clients))
+
+	ids := map[string]bool{}
+
+	for _, client := range clients.IdentityProviders.OIDC.Clients {
+		ids[client.ID] = true
+	}
+
+	require.NotEmpty(t, ids)
+
+	for _, plan := range plans {
+		want, ok := expected[plan.Name]
+
+		require.Truef(t, ok, "unexpected profile '%s'", plan.Name)
+
+		assert.Equalf(t, want.plan, plan.PlanName, "the '%s' profile lost its conformance plan name", plan.Name)
+		assert.NotEmptyf(t, plan.Plan.Alias, "the '%s' profile lost its alias", plan.Name)
+		assert.NotEmptyf(t, plan.Plan.Server.DiscoveryURL, "the '%s' profile lost its discovery url", plan.Name)
+
+		if want.variant {
+			require.NotNilf(t, plan.Variant, "the '%s' profile lost its variant", plan.Name)
+			assert.Equal(t, "discovery", plan.Variant.ServerMetadata)
+			assert.Equal(t, "static_client", plan.Variant.ClientRegistration)
+		} else {
+			assert.Nilf(t, plan.Variant, "the '%s' profile gained a variant", plan.Name)
+		}
+
+		if plan.Name == conformance.NameConfig {
+			assert.Nil(t, plan.Plan.Client)
+
+			continue
+		}
+
+		require.NotNilf(t, plan.Plan.Client, "the '%s' profile lost its client", plan.Name)
+		assert.NotEmpty(t, plan.Plan.Client.Secret)
+		assert.Truef(t, ids[plan.Plan.Client.ID], "the '%s' profile's client '%s' is not in the generated configuration", plan.Name, plan.Plan.Client.ID)
+		require.NotNilf(t, plan.Plan.ClientAlternate, "the '%s' profile lost its alternate client", plan.Name)
+		require.NotNilf(t, plan.Plan.ClientSecretPost, "the '%s' profile lost its secret post client", plan.Name)
+
+		assert.Truef(t, ids[plan.Plan.ClientAlternate.ID], "the '%s' profile's alternate client '%s' is not in the generated configuration", plan.Name, plan.Plan.ClientAlternate.ID)
+		assert.Truef(t, ids[plan.Plan.ClientSecretPost.ID], "the '%s' profile's secret post client '%s' is not in the generated configuration", plan.Name, plan.Plan.ClientSecretPost.ID)
+	}
+}
+
+func conformanceArchiveReference(plan, liveURL string) string {
+	if build, job := os.Getenv("BUILDKITE_BUILD_URL"), os.Getenv("BUILDKITE_JOB_ID"); os.Getenv("CI") == t && build != "" && job != "" {
+		return fmt.Sprintf("%s/%s in the job's artifacts: %s/waterfall?jid=%s&tab=artifacts", conformanceArchiveDir, conformanceArchiveName(plan), build, job)
+	}
+
+	return fmt.Sprintf("%s, or the live log while the suite is up: %s", conformanceArchivePath(plan), liveURL)
+}
+
+func conformancePlanSummary(plan string, accepted, failed, skipped int) string {
+	return fmt.Sprintf("Plan '%s': %d accepted, %d failed, %d skipped", plan, accepted, failed, skipped)
+}
+
+func conformanceArchiveName(plan string) string {
+	return fmt.Sprintf("conformance-%s.zip", plan)
+}
+
+func conformanceArchivePath(plan string) string {
+	if os.Getenv("CI") == t {
+		return filepath.Join("../..", conformanceArchiveDir, conformanceArchiveName(plan))
+	}
+
+	return filepath.Join(os.TempDir(), "authelia-suites-"+conformanceArchiveDir, composeProjectName(), conformanceArchiveName(plan))
+}

internal/suites/suite_oidc_traefik.go

@@ -72,11 +72,13 @@ func init() {
 
 	GlobalRegistry.Register(oidcTraefikSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description: `This suite has been created to test Authelia as an OpenID Connect 1.0 Provider behind the
+Traefik reverse proxy, with Redis backed sessions.`,
 	})
 }

internal/suites/suite_one_factor_only.go

@@ -42,12 +42,12 @@ func init() {
 
 	GlobalRegistry.Register(oneFactorOnlySuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     1 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description:     "This suite has been created to test Authelia in a one-factor only configuration",
 	})
 }

internal/suites/suite_pam.go

@@ -48,12 +48,12 @@ func init() {
 
 	GlobalRegistry.Register(pamSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     3 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description:     "PAM module integration tests for SSH authentication via pam_authelia",
 	})
 }

internal/suites/suite_pathprefix.go

@@ -64,11 +64,12 @@ func init() {
 
 	GlobalRegistry.Register(pathPrefixSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     150 * time.Second,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia served from a path prefix rather than the root of a domain.",
 	})
 }

internal/suites/suite_postgres.go

@@ -46,10 +46,12 @@ func init() {
 
 	GlobalRegistry.Register(postgresSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
+		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia with PostgreSQL as the storage backend and LDAP as the user backend.",
 	})
 }

internal/suites/suite_short_timeouts.go

@@ -43,12 +43,12 @@ func init() {
 
 	GlobalRegistry.Register(shortTimeoutsSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     3 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite has been created to configure Authelia with short timeouts for sessions expiration
 in order to test the inactivity feature and the remember me feature.`,
 	})

internal/suites/suite_standalone.go

@@ -51,12 +51,12 @@ func init() {
 
 	GlobalRegistry.Register(standaloneSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
-		TearDown:        teardown,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     5 * time.Minute,
-		TearDownTimeout: 2 * time.Minute,
+		TearDown:        teardown,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite is used to test Authelia in a standalone
 configuration with in-memory sessions and a local sqlite db stored on disk`,
 	})

internal/suites/suite_traefik2.go

@@ -66,11 +66,12 @@ func init() {
 
 	GlobalRegistry.Register(traefik2SuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia as an authorization server behind version 2 of the Traefik reverse proxy.",
 	})
 }

internal/suites/suite_traefik3.go

@@ -66,11 +66,12 @@ func init() {
 
 	GlobalRegistry.Register(traefik3SuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
 		OnError:         displayAutheliaLogs,
 		TestTimeout:     2 * time.Minute,
 		TearDown:        teardown,
-		TearDownTimeout: 2 * time.Minute,
+		TearDownTimeout: 1 * time.Minute,
+		Description:     "This suite has been created to test Authelia as an authorization server behind version 3 of the Traefik reverse proxy.",
 	})
 }

internal/suites/suite_two_factor.go

@@ -52,12 +52,12 @@ func init() {
 
 	GlobalRegistry.Register(twoFactorSuiteName, Suite{
 		SetUp:           setup,
-		SetUpTimeout:    5 * time.Minute,
-		OnError:         displayAutheliaLogs,
+		SetUpTimeout:    2 * time.Minute,
 		OnSetupTimeout:  displayAutheliaLogs,
-		TearDown:        teardown,
+		OnError:         displayAutheliaLogs,
 		TestTimeout:     4 * time.Minute,
-		TearDownTimeout: 2 * time.Minute,
+		TearDown:        teardown,
+		TearDownTimeout: 1 * time.Minute,
 		Description: `This suite is used to test Authelia in a two factor
 configuration with in-memory sessions and a local sqlite db stored on disk`,
 	})

internal/suites/utils.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/input"
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -95,21 +94,10 @@ const diagnosticsConsole = `() => JSON.stringify({
 	entries: window.__diagnostics__ || [],
 }, null, 2)`
 
+const coverageDir = "../../web/.nyc_output"
+
 var errPageNotCreated = errors.New("the page was not created")
 
-// StringToKeys returns the input.Key values which represent the given string.
-func StringToKeys(value string) []input.Key {
-	n := len(value)
-
-	keys := make([]input.Key, n)
-
-	for i := 0; i < n; i++ {
-		keys[i] = input.Key(value[i])
-	}
-
-	return keys
-}
-
 // ValidateBrowserPath validates the appropriate chromium browser path.
 func ValidateBrowserPath(path string) (browserPath string, err error) {
 	var info os.FileInfo
@@ -161,8 +149,6 @@ func (rs *RodSession) collectCoverage(page *rod.Page) {
 		return
 	}
 
-	coverageDir := "../../web/.nyc_output"
-
 	resp, err := page.Eval("() => JSON.stringify(window.__coverage__)")
 	if err != nil {
 		log.Errorf("Error collecting coverage: %v", err)
@@ -175,15 +161,54 @@ func (rs *RodSession) collectCoverage(page *rod.Page) {
 	_ = os.MkdirAll(coverageDir, 0775)
 
 	if coverageData != "<nil>" {
-		if err = os.WriteFile(fmt.Sprintf("%s/coverage-%s.json", coverageDir, uuid.New().String()), []byte(coverageData), 0664); err != nil { //nolint:gosec
+		if err = writeCoverage(coverageDir, coverageData); err != nil {
 			log.Errorf("Error writing coverage: %v", err)
-
-			return
 		}
+	}
+}
 
-		if err = filepath.Walk("../../web/.nyc_output", fixCoveragePath); err != nil {
-			log.Errorf("Error rewriting coverage paths: %v", err)
-		}
+func writeCoverage(dir, data string) (err error) {
+	if err = os.MkdirAll(dir, 0775); err != nil {
+		return err
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	data = strings.ReplaceAll(data, "/node/src/app/", strings.TrimSuffix(wd, "internal/suites")+"web/")
+
+	return os.WriteFile(filepath.Join(dir, fmt.Sprintf("coverage-%s.json", uuid.New().String())), []byte(data), 0664) //nolint:gosec
+}
+
+func suiteDebug() bool {
+	return os.Getenv("SUITE_DEBUG") == t
+}
+
+func buildkiteGroups() bool {
+	return os.Getenv("BUILDKITE") == t && suiteDebug()
+}
+
+func buildkiteGroup(title string) string {
+	if !buildkiteGroups() {
+		return ""
+	}
+
+	return "--- " + title
+}
+
+func buildkiteExpandGroup() string {
+	if !buildkiteGroups() {
+		return ""
+	}
+
+	return "^^^ +++"
+}
+
+func emitBuildkiteMarker(marker string) {
+	if marker != "" {
+		fmt.Println(marker) //nolint:forbidigo
 	}
 }
 
@@ -559,39 +584,6 @@ func (s *RodSuite) VerifyPageElementAttributeValue(t *testing.T, page *rod.Page,
 	assert.Equal(t, value, *attr)
 }
 
-func fixCoveragePath(path string, file os.FileInfo, err error) error {
-	if err != nil {
-		return err
-	}
-
-	if file.IsDir() {
-		return nil
-	}
-
-	coverage, err := filepath.Match("*.json", file.Name())
-	if err != nil {
-		return err
-	}
-
-	if coverage {
-		var data []byte
-
-		if data, err = os.ReadFile(path); err != nil { //nolint:gosec
-			return err
-		}
-
-		wd, _ := os.Getwd()
-		ciPath := strings.TrimSuffix(wd, "internal/suites")
-		content := strings.ReplaceAll(string(data), "/node/src/app/", ciPath+"web/")
-
-		if err = os.WriteFile(path, []byte(content), 0); err != nil { //nolint:gosec
-			return err
-		}
-	}
-
-	return nil
-}
-
 func getDomainEnvInfo(domain string) (info map[string]string, err error) {
 	info = make(map[string]string)