diff --git a/.buildkite/steps/e2etests.sh b/.buildkite/steps/e2etests.sh index b0c9c0491..790e30122 100755 --- a/.buildkite/steps/e2etests.sh +++ b/.buildkite/steps/e2etests.sh @@ -15,15 +15,33 @@ declare -A SUITE_AGENTS=( declare -A SUITE_TIMEOUTS=( [Kubernetes]="30" + [OIDCConformance]="120" ) +declare -A SUITE_NO_FAILFAST=( + [OIDCConformance]="true" +) + +DEBUG_REGEX='\[(debug test|test debug)\]' +SUITE_DEBUG="false" + +if [[ "${BUILDKITE_MESSAGE:-}" =~ ${DEBUG_REGEX} ]]; then + SUITE_DEBUG="true" +fi + for SUITE_NAME in $(authelia-scripts suites list); do AGENT="${SUITE_AGENTS[${SUITE_NAME}]:-all}" TIMEOUT="${SUITE_TIMEOUTS[${SUITE_NAME}]:-20}" + FAILFAST="--failfast" + + if [[ "${SUITE_NO_FAILFAST[${SUITE_NAME}]:-false}" == "true" ]]; then + FAILFAST="" + fi cat << EOF - label: ":selenium: ${SUITE_NAME} Suite" - command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --failfast --headless" + command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} ${FAILFAST} --headless" artifact_paths: + - "oidc-conformance-plans/*.zip" - "screenshots/**/*.access.log" - "screenshots/**/*.console.json" - "screenshots/**/*.containers.log" @@ -43,5 +61,6 @@ cat << EOF suite: "${AGENT}" env: SUITE: "${SUITE_NAME}" + SUITE_DEBUG: "${SUITE_DEBUG}" EOF done diff --git a/.gitignore b/.gitignore index 219128ab9..49fb6cf27 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ coverage.txt test-results-*.json test-results-*.xml screenshots/ +oidc-conformance-plans/ # Generated by TypeScript compiler dist/ diff --git a/cmd/authelia-gen/cmd_misc.go b/cmd/authelia-gen/cmd_misc.go index 9ab637158..d795bb193 100644 --- a/cmd/authelia-gen/cmd_misc.go +++ b/cmd/authelia-gen/cmd_misc.go @@ -18,6 +18,7 @@ import ( "go.yaml.in/yaml/v4" "github.com/authelia/authelia/v4/internal/configuration/schema" + "github.com/authelia/authelia/v4/internal/oidc/conformance" "github.com/authelia/authelia/v4/internal/utils" ) @@ -188,19 +189,11 @@ func miscOIDCConformance(version, token, consent, policy, brand string, authelia return nil } -func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL, suiteNames ...string) (suites []OpenIDConnectConformanceSuite) { - builders := []*OpenIDConnectConformanceSuiteBuilder{ - {brand, "config", "Config", true, version, consent, policy, nil, autheliaURL}, - {brand, "basic", "Basic", true, version, consent, policy, suiteURL, autheliaURL}, - {brand, suiteNameBasicFormPost, "Basic (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, - {brand, "hybrid", "Hybrid", true, version, consent, policy, suiteURL, autheliaURL}, - {brand, suiteNameHybridFormPost, "Hybrid (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, - {brand, "implicit", "Implicit", true, version, consent, policy, suiteURL, autheliaURL}, - {brand, suiteNameImplicitFormPost, "Implicit (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, - } +func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL, suiteNames ...string) (suites []conformance.Suite) { + builders := conformance.Builders(version, consent, policy, brand, suiteURL, autheliaURL) for _, builder := range builders { - if len(suiteNames) != 0 && !utils.IsStringInSlice(builder.name, suiteNames) { + if len(suiteNames) != 0 && !utils.IsStringInSlice(builder.Name, suiteNames) { continue } @@ -210,7 +203,7 @@ func miscOIDCConformanceBuildSuites(version, consent, policy, brand string, suit return suites } -func doOIDCConformanceSuitePostPlan(client *http.Client, base *url.URL, plan string, variant *OpenIDConnectConformanceSuitePlanVariant, body *bytes.Buffer) (err error) { +func doOIDCConformanceSuitePostPlan(client *http.Client, base *url.URL, plan string, variant *conformance.PlanVariant, body *bytes.Buffer) (err error) { if client == nil { return nil } diff --git a/cmd/authelia-gen/const.go b/cmd/authelia-gen/const.go index 597e394a0..dc2c16a24 100644 --- a/cmd/authelia-gen/const.go +++ b/cmd/authelia-gen/const.go @@ -51,18 +51,6 @@ const ( fileGitHubIssueTemplateBR = ".github/ISSUE_TEMPLATE/bug-report.yml" ) -const ( - suiteNameBasicFormPost = "basic-form-post" - suiteNameHybridFormPost = "hybrid-form-post" - suiteNameImplicitFormPost = "implicit-form-post" - suiteConformanceBasic = "conformance-basic" - suiteConformanceBasicFormPost = "conformance-basic-form-post" - suiteConformanceImplicit = "conformance-implicit" - suiteConformanceImplicitFormPost = "conformance-implicit-form-post" - suiteConformanceHybrid = "conformance-hybrid" - suiteConformanceHybridFormPost = "conformance-hybrid-form-post" -) - const ( pathJSONSchema = "json-schema" extJSON = ".json" diff --git a/cmd/authelia-gen/types.go b/cmd/authelia-gen/types.go index eaf05df79..c971a195b 100644 --- a/cmd/authelia-gen/types.go +++ b/cmd/authelia-gen/types.go @@ -8,8 +8,6 @@ import ( "fmt" "strings" "time" - - "github.com/authelia/authelia/v4/internal/configuration/schema" ) type tmplIssueTemplateData struct { @@ -230,90 +228,6 @@ type PackageJSON struct { Version string `json:"version"` } -// OpenIDConnectConformanceSuite represents a single OpenID Connect conformance suite. -type OpenIDConnectConformanceSuite struct { - Name string - Plan OpenIDConnectConformanceSuitePlan - Clients []schema.IdentityProvidersOpenIDConnectClient -} - -// OpenIDConnectConformanceSuitePlan represents the test plan for an OpenIDConnectConformanceSuite. -type OpenIDConnectConformanceSuitePlan struct { - Name string `json:"-"` - Alias string `json:"alias"` - Description string `json:"description"` - Publish string `json:"publish"` - - Variant *OpenIDConnectConformanceSuitePlanVariant `json:"-"` - Server OpenIDConnectConformanceSuitePlanServer `json:"server"` - Client *OpenIDConnectConformanceSuitePlanClient `json:"client,omitempty"` - ClientAlternate *OpenIDConnectConformanceSuitePlanClient `json:"client2,omitempty"` - ClientSecretPost *OpenIDConnectConformanceSuitePlanClient `json:"client_secret_post,omitempty"` - MutualTLS *OpenIDConnectConformanceSuitePlanMutualTLS `json:"mtls,omitempty"` - Resource *OpenIDConnectConformanceSuitePlanResource `json:"resource,omitempty"` - - EKYCVerifiedClaimsRequest string `json:"ekyc_verified_claims_request,omitempty"` - EKYCUserinfo string `json:"ekyc_userinfo,omitempty"` -} - -// OpenIDConnectConformanceSuitePlanVariant represents the variant of an OpenIDConnectConformanceSuitePlan. -type OpenIDConnectConformanceSuitePlanVariant struct { - ServerMetadata string `json:"server_metadata,omitempty"` - ClientRegistration string `json:"client_registration,omitempty"` - ClientAuthType string `json:"client_auth_type,omitempty"` - ResponseType string `json:"response_type,omitempty"` - ResponseMode string `json:"response_mode,omitempty"` - CIBAMode string `json:"ciba_mode,omitempty"` - FAPIProfile string `json:"fapi_profile,omitempty"` - FAPIResponseMode string `json:"fapi_response_mode,omitempty"` - FAPIAuthRequestMethod string `json:"fapi_auth_request_method,omitempty"` - SenderConstrain string `json:"sender_constrain,omitempty"` - AuthorizationRequestType string `json:"authorization_request_type,omitempty"` - OpenID string `json:"openid,omitempty"` -} - -// OpenIDConnectConformanceSuitePlanServer represents the server details of an OpenIDConnectConformanceSuitePlan. -type OpenIDConnectConformanceSuitePlanServer struct { - DiscoveryURL string `json:"discoveryUrl,omitempty"` - Issuer string `json:"issuer,omitempty"` - JWKSURI string `json:"jwks_uri,omitempty"` - AuthorizationEndpoint string `json:"authorization_endpoint,omitempty"` - TokenEndpoint string `json:"token_endpoint,omitempty"` - UserinfoEndpoint string `json:"userinfo_endpoint,omitempty"` - ACRValues string `json:"acr_values,omitempty"` - LoginHint string `json:"login_hint,omitempty"` -} - -// OpenIDConnectConformanceSuitePlanClient represents a client registered with an OpenIDConnectConformanceSuitePlan. -type OpenIDConnectConformanceSuitePlanClient struct { - ID string `json:"client_id,omitempty"` - Secret string `json:"client_secret,omitempty"` - Name string `json:"client_name,omitempty"` - Scope string `json:"scope,omitempty"` - SecretJWTAlgorithm string `json:"client_secret_jwt_alg,omitempty"` - DPOPSigningAlgorithm string `json:"dpop_signing_alg,omitempty"` - InitialAccessToken string `json:"initial_access_token,omitempty"` - HintType string `json:"hint_type,omitempty"` - HintValue string `json:"hint_value,omitempty"` - JWKS string `json:"jwks,omitempty"` -} - -// OpenIDConnectConformanceSuitePlanMutualTLS represents the Mutual TLS details of an OpenIDConnectConformanceSuitePlan. -type OpenIDConnectConformanceSuitePlanMutualTLS struct { - Certificate string `json:"certificate,omitempty"` - Key string `json:"key,omitempty"` - CertificateAuthority string `json:"ca,omitempty"` -} - -// OpenIDConnectConformanceSuitePlanResource represents the protected resource details of an OpenIDConnectConformanceSuitePlan. -type OpenIDConnectConformanceSuitePlanResource struct { - ResourceURL string `json:"resourceUrl"` - ResourceURLAccountRequests string `json:"resourceUrlAccountRequests,omitempty"` - ResourceURLAccountsResource string `json:"resourceUrlAccountsResource,omitempty"` - InstitutionID string `json:"institution_id,omitempty"` - RichAuthorizationRequest string `json:"richAuthorizationRequest,omitempty"` -} - // AllContributors represents the .all-contributorsrc configuration file. type AllContributors struct { Contributors []Contributor `json:"contributors"` diff --git a/cmd/authelia-scripts/cmd/helpers_results.go b/cmd/authelia-scripts/cmd/helpers_results.go index 71fc6dee7..184f71cfd 100644 --- a/cmd/authelia-scripts/cmd/helpers_results.go +++ b/cmd/authelia-scripts/cmd/helpers_results.go @@ -8,16 +8,28 @@ import ( "bytes" "encoding/json" "io" + "strings" ) type testEvent struct { Action string `json:"Action"` + Test string `json:"Test"` Output string `json:"Output"` } type testOutputWriter struct { - out io.Writer - buf bytes.Buffer + out io.Writer + buf bytes.Buffer + grouped bool + pending []testFraming + buildkite bool + deferring bool + deferred bytes.Buffer +} + +type testFraming struct { + test string + line string } func (w *testOutputWriter) Write(p []byte) (int, error) { @@ -34,17 +46,136 @@ func (w *testOutputWriter) Write(p []byte) (int, error) { var event testEvent if json.Unmarshal(line, &event) != nil { - if _, err = w.out.Write(line); err != nil { + if err = w.write(string(line)); err != nil { return len(p), err } continue } - if event.Action == "output" { - if _, err = io.WriteString(w.out, event.Output); err != nil { - return len(p), err - } + if err = w.handle(event); err != nil { + return len(p), err } } } + +func (w *testOutputWriter) handle(event testEvent) (err error) { + if event.Action != "output" { + return nil + } + + if !w.grouped { + return w.write(event.Output) + } + + switch output := event.Output; { + case isTestFramingLine(output): + w.pending = append(w.pending, testFraming{test: event.Test, line: output}) + + return nil + case isBuildkiteMarkerLine(output) && !isTestSummaryLine(output): + if err = w.release(func(framing testFraming) bool { return !isTestOrParent(framing.test, event.Test) }); err != nil { + return err + } + + if err = w.write(output); err != nil { + return err + } + + return w.release(func(testFraming) bool { return true }) + default: + if err = w.release(func(testFraming) bool { return true }); err != nil { + return err + } + + return w.write(output) + } +} + +func (w *testOutputWriter) write(output string) (err error) { + if w.buildkite && !w.deferring && isTopLevelTestSummaryLine(output) { + w.deferring = true + } + + if w.deferring { + w.deferred.WriteString(output) + + return nil + } + + _, err = io.WriteString(w.out, output) + + return err +} + +func (w *testOutputWriter) Flush() (err error) { + if err = w.release(func(testFraming) bool { return true }); err != nil { + return err + } + + _, err = w.out.Write(w.deferred.Bytes()) + + w.deferred.Reset() + + return err +} + +func (w *testOutputWriter) release(match func(framing testFraming) bool) (err error) { + kept := w.pending[:0] + + for _, framing := range w.pending { + if !match(framing) { + kept = append(kept, framing) + + continue + } + + if err = w.write(framing.line); err != nil { + return err + } + } + + w.pending = kept + + return nil +} + +func isTestOrParent(test, child string) bool { + return test == child || strings.HasPrefix(child, test+"/") +} + +func isTestFramingLine(output string) bool { + for _, prefix := range []string{"=== RUN", "=== PAUSE", "=== CONT", "=== NAME"} { + if strings.HasPrefix(output, prefix) { + return true + } + } + + return false +} + +func isTestSummaryLine(output string) bool { + trimmed := strings.TrimLeft(output, " ") + + for _, prefix := range []string{"--- PASS: ", "--- FAIL: ", "--- SKIP: "} { + if strings.HasPrefix(trimmed, prefix) { + return true + } + } + + return false +} + +func isTopLevelTestSummaryLine(output string) bool { + return isTestSummaryLine(output) && !strings.HasPrefix(output, " ") +} + +func isBuildkiteMarkerLine(output string) bool { + for _, prefix := range []string{"--- ", "+++ ", "~~~ ", "^^^ +++"} { + if strings.HasPrefix(output, prefix) { + return true + } + } + + return false +} diff --git a/cmd/authelia-scripts/cmd/helpers_results_test.go b/cmd/authelia-scripts/cmd/helpers_results_test.go index fb521da4c..fd5b9c5ad 100644 --- a/cmd/authelia-scripts/cmd/helpers_results_test.go +++ b/cmd/authelia-scripts/cmd/helpers_results_test.go @@ -6,9 +6,12 @@ package cmd import ( "bytes" + "errors" + "fmt" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTestOutputWriterShouldReconstructConsoleOutput(t *testing.T) { @@ -63,3 +66,188 @@ func TestTestOutputWriterShouldReconstructConsoleOutput(t *testing.T) { }) } } + +func TestTestOutputWriterShouldPlaceFramingInsideBuildkiteGroups(t *testing.T) { + event := func(test, output string) string { + return fmt.Sprintf(`{"Action":"output","Test":%q,"Output":%q}`, test, output) + "\n" + } + + testCases := []struct { + name string + events []string + expected string + }{ + { + "ShouldPlaceTheFramingOfTheTestWhichStartedAGroupInsideIt", + []string{ + event("TestSuite", "=== RUN TestSuite\n"), + event("TestSuite/TestBasic", "=== RUN TestSuite/TestBasic\n"), + event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"), + }, + "--- OIDC Conformance Plan: basic\n=== RUN TestSuite\n=== RUN TestSuite/TestBasic\n", + }, + { + "ShouldLeaveTheFramingOfTheLastGroupBeforeTheNextOne", + []string{ + event("TestSuite/TestBasic/Codereuse", "=== RUN TestSuite/TestBasic/Codereuse\n"), + event("TestSuite/TestBasicFormPost", "=== RUN TestSuite/TestBasicFormPost\n"), + event("TestSuite/TestBasicFormPost", "--- OIDC Conformance Plan: basic-form-post\n"), + }, + "=== RUN TestSuite/TestBasic/Codereuse\n--- OIDC Conformance Plan: basic-form-post\n=== RUN TestSuite/TestBasicFormPost\n", + }, + { + "ShouldPrintTheFramingOfEveryTestBeforeTheNextOutput", + []string{ + event("TestSuite/TestBasic/Server", "=== RUN TestSuite/TestBasic/Server\n"), + event("TestSuite/TestBasic/MaxAge1", "=== RUN TestSuite/TestBasic/MaxAge1\n"), + event("TestSuite/TestBasic/MaxAge1", " suite_test.go:1: Module finished with the result 'FAILED'\n"), + event("TestSuite/TestBasic/MaxAge1", "^^^ +++\n"), + }, + "=== RUN TestSuite/TestBasic/Server\n=== RUN TestSuite/TestBasic/MaxAge1\n suite_test.go:1: Module finished with the result 'FAILED'\n^^^ +++\n", + }, + { + "ShouldPrintHeldFramingBeforeTheSummary", + []string{ + event("TestSuite/TestBasic/Server", "=== RUN TestSuite/TestBasic/Server\n"), + event("TestSuite/TestBasic/Server", "--- PASS: TestSuite/TestBasic/Server (0.00s)\n"), + }, + "=== RUN TestSuite/TestBasic/Server\n--- PASS: TestSuite/TestBasic/Server (0.00s)\n", + }, + { + "ShouldPrintOutputWhichBelongsToNoTest", + []string{ + event("TestSuite", "=== RUN TestSuite\n"), + event("", "PASS\n"), + }, + "=== RUN TestSuite\nPASS\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := &bytes.Buffer{} + writer := &testOutputWriter{out: out, grouped: true} + + for _, write := range tc.events { + _, err := writer.Write([]byte(write)) + + assert.NoError(t, err) + } + + assert.Equal(t, tc.expected, out.String()) + }) + } + + t.Run("ShouldPrintEverythingInPlaceWhenNotGrouped", func(t *testing.T) { + out := &bytes.Buffer{} + writer := &testOutputWriter{out: out} + + for _, write := range []string{ + event("TestSuite/TestBasic", "=== RUN TestSuite/TestBasic\n"), + event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"), + } { + _, err := writer.Write([]byte(write)) + + assert.NoError(t, err) + } + + assert.Equal(t, "=== RUN TestSuite/TestBasic\n--- OIDC Conformance Plan: basic\n", out.String()) + }) +} + +func TestTestOutputWriterShouldDeferTheSummaryInBuildkite(t *testing.T) { + events := `{"Action":"output","Test":"TestSuite/TestBasic","Output":" suite_test.go:1: Exported the plan log\n"}` + "\n" + + `{"Action":"output","Test":"TestSuite","Output":"--- PASS: TestSuite (227.08s)\n"}` + "\n" + + `{"Action":"output","Test":"TestSuite/TestBasic","Output":" --- PASS: TestSuite/TestBasic (116.14s)\n"}` + "\n" + + `{"Action":"output","Output":"PASS\n"}` + "\n" + + summary := "--- PASS: TestSuite (227.08s)\n --- PASS: TestSuite/TestBasic (116.14s)\nPASS\n" + + testCases := []struct { + name string + buildkite bool + grouped bool + before string + }{ + {"ShouldDeferTheSummaryInBuildkite", true, false, " suite_test.go:1: Exported the plan log\n"}, + {"ShouldDeferTheSummaryInBuildkiteWhenGrouped", true, true, " suite_test.go:1: Exported the plan log\n"}, + {"ShouldPrintTheSummaryInPlaceElsewhere", false, false, " suite_test.go:1: Exported the plan log\n" + summary}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := &bytes.Buffer{} + writer := &testOutputWriter{out: out, buildkite: tc.buildkite, grouped: tc.grouped} + + _, err := writer.Write([]byte(events)) + assert.NoError(t, err) + + assert.Equal(t, tc.before, out.String()) + + out.WriteString("teardown\n") + + assert.NoError(t, writer.Flush()) + + if tc.buildkite { + assert.Equal(t, tc.before+"teardown\n"+summary, out.String()) + } else { + assert.Equal(t, tc.before+"teardown\n", out.String()) + } + }) + } +} + +func TestTestOutputWriterShouldReturnWriteErrors(t *testing.T) { + event := func(test, output string) string { + return fmt.Sprintf(`{"Action":"output","Test":%q,"Output":%q}`, test, output) + "\n" + } + + testCases := []struct { + name string + events []string + }{ + {"ShouldReturnTheErrorOfAPlainLine", []string{"not an event\n"}}, + {"ShouldReturnTheErrorOfHeldFramingBeforeAGroup", []string{ + event("TestSuite/TestBasic/Codereuse", "=== RUN TestSuite/TestBasic/Codereuse\n"), + event("TestSuite/TestBasicFormPost", "--- OIDC Conformance Plan: basic-form-post\n"), + }}, + {"ShouldReturnTheErrorOfAGroupHeader", []string{ + event("TestSuite/TestBasic", "--- OIDC Conformance Plan: basic\n"), + }}, + {"ShouldReturnTheErrorOfHeldFramingBeforeOutput", []string{ + event("TestSuite/TestBasic/Server", "=== RUN TestSuite/TestBasic/Server\n"), + event("TestSuite/TestBasic/Server", " suite_test.go:1: output\n"), + }}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + writer := &testOutputWriter{out: testFailingWriter{}, grouped: true} + + var err error + + for _, write := range tc.events { + if _, err = writer.Write([]byte(write)); err != nil { + break + } + } + + assert.EqualError(t, err, "write failed") + }) + } + + t.Run("ShouldReturnTheErrorOfHeldFramingWhenFlushing", func(t *testing.T) { + writer := &testOutputWriter{out: testFailingWriter{}, grouped: true} + + _, err := writer.Write([]byte(event("TestSuite/TestBasic/Server", "=== RUN TestSuite/TestBasic/Server\n"))) + require.NoError(t, err) + + assert.EqualError(t, writer.Flush(), "write failed") + }) +} + +type testFailingWriter struct{} + +func (testFailingWriter) Write(_ []byte) (int, error) { + return 0, errors.New("write failed") +} diff --git a/cmd/authelia-scripts/cmd/suites.go b/cmd/authelia-scripts/cmd/suites.go index 3fcf77426..ce79ee70f 100644 --- a/cmd/authelia-scripts/cmd/suites.go +++ b/cmd/authelia-scripts/cmd/suites.go @@ -351,8 +351,16 @@ func runSuiteTests(suiteName string, withEnv bool) error { defer results.Close() + output := &testOutputWriter{out: os.Stdout, buildkite: os.Getenv("BUILDKITE") == "true", grouped: os.Getenv("BUILDKITE") == "true" && os.Getenv("SUITE_DEBUG") == "true"} + + defer func() { + if err := output.Flush(); err != nil { + log.Errorf("Error writing the test summary: %v", err) + } + }() + cmd := utils.CommandWithStdout("bash", "-c", testCmdLine) - cmd.Stdout = io.MultiWriter(&testOutputWriter{out: os.Stdout}, results) + cmd.Stdout = io.MultiWriter(output, results) cmd.Stderr = os.Stderr cmd.Env = os.Environ() diff --git a/docs/content/contributing/development/integration-suites.md b/docs/content/contributing/development/integration-suites.md index e3bba0d5c..2332543a3 100644 --- a/docs/content/contributing/development/integration-suites.md +++ b/docs/content/contributing/development/integration-suites.md @@ -95,16 +95,17 @@ The suites run several concurrent copies of themselves when the Docker daemon is optional and falls back to the value used for a single local run, so leaving them all unset gives the behavior described above. -| Variable | Default | Purpose | -| :--------------------- | :-------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SUITE_SLOT` | _unset_ | Slot number owned by this agent or working tree. When set, `bootstrap.sh` derives `COMPOSE_PROJECT_NAME`, `SUITE_SUBNET`, `LDAP_ADMIN_PORT`, `ENVOY_ADMIN_PORT` and, outside CI, `SUITE_TMP` and `SUITE_TMP_PATH` from it. A slotted shell also leaves `/etc/hosts` alone. | -| `SUITE_SLOT_AUTO` | _unset_ | Set to `false` to stop `bootstrap.sh` allocating a slot for this working tree. Has no effect in CI, where the slot is supplied by the agent. | -| `COMPOSE_PROJECT_NAME` | `authelia` | Compose project name. Also scopes the Traefik Docker provider so it only discovers its own containers. | -| `SUITE_SUBNET` | `192.168.240` | First three octets of the suite network. | -| `SUITE_TMP` | `/tmp` | Host directory bound into the suite containers, which always see it at `/tmp`. Give each agent or working tree its own directory, because everything at its top level apart from the agent's own working files is removed when a job finishes. | -| `SUITE_TMP_PATH` | `/tmp` | Path the test process itself reads and writes that same content through. In CI this stays `/tmp` because `SUITE_TMP` is bound there inside the agent; locally it is set to `SUITE_TMP`, since the test process runs on the host. | -| `SUITE_IMAGE` | `authelia:dist` | Image the backend runs. | -| `AGENT_CONTAINER` | _unset_ | Name of the container the tests run in. When set, that container is attached to the suite network on setup and detached on teardown, so Chrome can reach the portal. | +| Variable | Default | Purpose | +| :------------------------------------ | :-------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SUITE_SLOT` | _unset_ | Slot number owned by this agent or working tree. When set, `bootstrap.sh` derives `COMPOSE_PROJECT_NAME`, `SUITE_SUBNET`, `LDAP_ADMIN_PORT`, `ENVOY_ADMIN_PORT` and, outside CI, `SUITE_TMP` and `SUITE_TMP_PATH` from it. A slotted shell also leaves `/etc/hosts` alone. | +| `SUITE_SLOT_AUTO` | _unset_ | Set to `false` to stop `bootstrap.sh` allocating a slot for this working tree. Has no effect in CI, where the slot is supplied by the agent. | +| `COMPOSE_PROJECT_NAME` | `authelia` | Compose project name. Also scopes the Traefik Docker provider so it only discovers its own containers. | +| `SUITE_SUBNET` | `192.168.240` | First three octets of the suite network. | +| `SUITE_TMP` | `/tmp` | Host directory bound into the suite containers, which always see it at `/tmp`. Give each agent or working tree its own directory, because everything at its top level apart from the agent's own working files is removed when a job finishes. | +| `SUITE_TMP_PATH` | `/tmp` | Path the test process itself reads and writes that same content through. In CI this stays `/tmp` because `SUITE_TMP` is bound there inside the agent; locally it is set to `SUITE_TMP`, since the test process runs on the host. | +| `SUITE_IMAGE` | `authelia:dist` | Image the backend runs. | +| `AGENT_CONTAINER` | _unset_ | Name of the container the tests run in. When set, that container is attached to the suite network on setup and detached on teardown, so Chrome can reach the portal. | +| `SUITE_OIDC_CONFORMANCE_MONGODB_HOST` | _unset_ | MongoDB host the OIDCConformance suite's conformance server uses. When set, the suite does not run its own MongoDB; the host has to be reachable from the suite network. | ## Running Suites Concurrently @@ -198,4 +199,20 @@ Creating a suite is as easy. Let's take the example of the **Standalone** suite: A suite can also be much more complex like setting up a complete Kubernetes ecosystem. You can check the Kubernetes suite as example. +## Commit Message Toggles + +Our continuous integration reads the following options from the commit message. It reads the message of the commit +being built, which for a pull request is its most recent commit, so an option has to be in that commit's message. An +option can appear anywhere in the message; placing it in the body or the footer keeps the header within its format. + +- `[skip test]` or `[test skip]`: skips linting, the external tests, the coverage image build, and the integration + tests. Every other step, including the unit tests, still runs. +- `[debug test]` or `[test debug]`: runs the integration suites with their opt-in debug output enabled. The OpenID + Connect 1.0 conformance suite groups its build log by plan, with each group holding its tests' `=== RUN` lines and + opening by itself when one of them fails, and logs a trace of what it did for each module, stamped with the time in + UTC, under that module's subtest. Without this option the build log is left as `go test` writes it. The trace is + available locally by setting the `SUITE_DEBUG` environment variable to `true` when running a suite. + +A commit whose header begins with `docs` also skips vulnerability scanning. + [delve]: https://github.com/go-delve/delve diff --git a/docs/content/integration/openid-connect/introduction.md b/docs/content/integration/openid-connect/introduction.md index 7717b0826..6ae85e338 100644 --- a/docs/content/integration/openid-connect/introduction.md +++ b/docs/content/integration/openid-connect/introduction.md @@ -46,6 +46,12 @@ actively perform the tests on each version of Authelia to maintain the latest co You can view our published conformance tests at [Certified OpenID Providers & Profiles] and [Certified OpenID Providers for Logout Profiles]. +In addition to the published conformance tests, all commits that change any part of the code run integration tests +against every module of the conformance plans that Authelia is certified for or is planning to get certified for. These +plans and modules are all of the tests required for an implementation to become certified. This is both done as a +quality assurance measure to ensure confidence in Authelia's implementation and also to thoroughly test the +implementation against normal use cases. + ### OpenID Connect Protocol Suite
diff --git a/internal/handlers/handler_oauth2_authorization_consent_core_test.go b/internal/handlers/handler_oauth2_authorization_consent_core_test.go index 9b7bf9e02..d13e657ea 100644 --- a/internal/handlers/handler_oauth2_authorization_consent_core_test.go +++ b/internal/handlers/handler_oauth2_authorization_consent_core_test.go @@ -1101,6 +1101,68 @@ func TestHandleOAuth2AuthorizationConsentModeImplicitWithID(t *testing.T) { }, expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) {}, }, + { + name: "ShouldUseTheResponseAlreadyGivenOnTheConsentForm", + issuer: mustParseURI(t, "https://auth.example.com"), + client: clientTest, + userSession: session.UserSession{Username: testValue, FirstFactorAuthnTimestamp: 1000000, SecondFactorAuthnTimestamp: 1000000}, + subject: sub, + requester: &oauthelia2.AuthorizeRequest{ + Request: oauthelia2.Request{ + Client: clientTest, + Form: url.Values{ + oidc.FormParameterMaximumAge: []string{"10"}, + }, + RequestedAt: time.Unix(1000000, 0), + }, + }, + expected: &model.OAuth2ConsentSession{ + ID: 40, + ClientID: "test", + Subject: uuid.NullUUID{UUID: sub, Valid: true}, + Authorized: true, + Form: "max_age=10", + RequestedAt: time.Unix(1000000, 0), + RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true}, + }, + handled: false, + setup: func(t *testing.T, mock *mocks.MockAutheliaCtx) { + gomock.InOrder( + mock.StorageMock.EXPECT(). + LoadOAuth2ConsentSessionByChallengeID(gomock.Eq(mock.Ctx), gomock.Eq(challenge)). + Return(&model.OAuth2ConsentSession{ID: 40, ChallengeID: challenge, ClientID: "test", Subject: uuid.NullUUID{UUID: sub, Valid: true}, Authorized: true, Form: "max_age=10", RequestedAt: time.Unix(1000000, 0), RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true}, ExpiresAt: mock.Ctx.Providers.Clock.Now().Add(time.Second * 10)}, nil), + ) + }, + expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) {}, + }, + { + name: "ShouldDenyWhenTheConsentFormWasRejected", + issuer: mustParseURI(t, "https://auth.example.com"), + client: clientTest, + userSession: session.UserSession{Username: testValue, FirstFactorAuthnTimestamp: 1000000, SecondFactorAuthnTimestamp: 1000000}, + subject: sub, + requester: &oauthelia2.AuthorizeRequest{ + Request: oauthelia2.Request{ + Client: clientTest, + Form: url.Values{ + oidc.FormParameterMaximumAge: []string{"10"}, + }, + RequestedAt: time.Unix(1000000, 0), + }, + }, + expected: nil, + handled: true, + setup: func(t *testing.T, mock *mocks.MockAutheliaCtx) { + gomock.InOrder( + mock.StorageMock.EXPECT(). + LoadOAuth2ConsentSessionByChallengeID(gomock.Eq(mock.Ctx), gomock.Eq(challenge)). + Return(&model.OAuth2ConsentSession{ID: 40, ChallengeID: challenge, ClientID: "test", Subject: uuid.NullUUID{UUID: sub, Valid: true}, Authorized: false, Form: "max_age=10", RequestedAt: time.Unix(1000000, 0), RespondedAt: sql.NullTime{Time: time.Unix(1000000, 0), Valid: true}, ExpiresAt: mock.Ctx.Providers.Clock.Now().Add(time.Second * 10)}, nil), + ) + }, + expect: func(t *testing.T, mock *mocks.MockAutheliaCtx) { + mock.AssertLastLogMessageRegexp(t, regexp.MustCompile(`^Authorization Request with id '[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}' on client with id 'test' using consent mode 'implicit' could not be processed: error occurred performing consent for consent session with id '11303e1f-f8af-436a-9a72-c7361bfc9f37': the user explicitly rejected this consent session$`), nil) + }, + }, } for _, tc := range testCases { diff --git a/internal/handlers/handler_oauth2_authorization_consent_implicit.go b/internal/handlers/handler_oauth2_authorization_consent_implicit.go index fce4a9a7c..78c1fc705 100644 --- a/internal/handlers/handler_oauth2_authorization_consent_implicit.go +++ b/internal/handlers/handler_oauth2_authorization_consent_implicit.go @@ -97,6 +97,18 @@ func handleOAuth2AuthorizationConsentModeImplicitWithID(ctx *middlewares.Autheli return nil, true } + if consent.Responded() { + if !consent.IsAuthorized() { + ctx.GetLogger().Errorf(logFmtErrConsentCantGrantRejected, requester.GetID(), client.GetID(), client.GetConsentPolicy(), consent.ChallengeID) + + ctx.Providers.OpenIDConnect.WriteDynamicAuthorizeError(ctx, rw, requester, oauthelia2.ErrAccessDenied) + + return nil, true + } + + return consent, false + } + var requests *oidc.ClaimsRequests if requests, err = oidc.NewClaimRequests(requester.GetRequestForm()); err == nil && requests != nil { diff --git a/cmd/authelia-gen/openid_conformance.go b/internal/oidc/conformance/builder.go similarity index 62% rename from cmd/authelia-gen/openid_conformance.go rename to internal/oidc/conformance/builder.go index 207ee208c..446d42baf 100644 --- a/cmd/authelia-gen/openid_conformance.go +++ b/internal/oidc/conformance/builder.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package main +package conformance import ( "fmt" @@ -17,80 +17,87 @@ import ( "github.com/authelia/authelia/v4/internal/random" ) -// OpenIDConnectConformanceSuiteBuilder builds an OpenIDConnectConformanceSuite. -type OpenIDConnectConformanceSuiteBuilder struct { - brand string - name string - friendly string - certification bool - version string - consent string - policy string - suiteURL *url.URL - autheliaURL *url.URL +// SuiteBuilder builds a Suite. +type SuiteBuilder struct { + Brand string + Name string + Friendly string + Certification bool + Version string + Consent string + Policy string + SuiteURL *url.URL + AutheliaURL *url.URL } -// Build returns the OpenIDConnectConformanceSuite for this builder. -func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceSuite { - var ( - apiname, namePrefix, clientIDPrefix, descriptionSuffix string - variant *OpenIDConnectConformanceSuitePlanVariant - ) - - if b.certification { - namePrefix = "conformance-" - clientIDPrefix = "conformance-certification" - descriptionSuffix = "Certification Profile" - } else { - clientIDPrefix = "conformance-test" - descriptionSuffix = "Test Profile" +// Description returns the plan description naming release as the version of Authelia under test. +func (b *SuiteBuilder) Description(release string) string { + if b.Certification { + return fmt.Sprintf("Authelia %s %s Certification Profile", release, b.Friendly) } - aliasSuffix := fmt.Sprintf("%s-%s", strings.ReplaceAll(strings.ToLower(b.name), ".", "-"), b.brand+strings.ReplaceAll(strings.ToLower(b.version), ".", "")) + return fmt.Sprintf("Authelia %s %s Test Profile", release, b.Friendly) +} - name := fmt.Sprintf("%s%s", namePrefix, b.name) - description := fmt.Sprintf("Authelia %s %s %s", b.version, b.friendly, descriptionSuffix) +// Build returns the Suite for this builder. +func (b *SuiteBuilder) Build() Suite { + var ( + apiname, namePrefix, clientIDPrefix string + variant *PlanVariant + ) - switch name { - case suiteConformanceBasic, suiteConformanceBasicFormPost, suiteConformanceHybrid, suiteConformanceHybridFormPost, suiteConformanceImplicit, suiteConformanceImplicitFormPost: - variant = &OpenIDConnectConformanceSuitePlanVariant{ + if b.Certification { + namePrefix = "conformance-" + clientIDPrefix = "conformance-certification" + } else { + clientIDPrefix = "conformance-test" + } + + aliasSuffix := fmt.Sprintf("%s-%s", strings.ReplaceAll(strings.ToLower(b.Name), ".", "-"), b.Brand+strings.ReplaceAll(strings.ToLower(b.Version), ".", "")) + + name := fmt.Sprintf("%s%s", namePrefix, b.Name) + description := b.Description(b.Version) + + switch b.Name { + case NameBasic, NameBasicFormPost, NameHybrid, NameHybridFormPost, NameImplicit, NameImplicitFormPost: + variant = &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", } } - switch name { - case "conformance-config": + switch b.Name { + case NameConfig: apiname = "oidcc-config-certification-test-plan" - case suiteConformanceBasic: + case NameBasic: apiname = "oidcc-basic-certification-test-plan" - case suiteConformanceBasicFormPost: + case NameBasicFormPost: apiname = "oidcc-formpost-basic-certification-test-plan" - case suiteConformanceHybrid: + case NameHybrid: apiname = "oidcc-hybrid-certification-test-plan" - case suiteConformanceHybridFormPost: + case NameHybridFormPost: apiname = "oidcc-formpost-hybrid-certification-test-plan" - case suiteConformanceImplicit: + case NameImplicit: apiname = "oidcc-implicit-certification-test-plan" - case suiteConformanceImplicitFormPost: + case NameImplicitFormPost: apiname = "oidcc-formpost-implicit-certification-test-plan" } - suite := OpenIDConnectConformanceSuite{ + suite := Suite{ Name: name, - Plan: OpenIDConnectConformanceSuitePlan{ + Plan: Plan{ Name: apiname, Variant: variant, Alias: fmt.Sprintf("%s%s", namePrefix, aliasSuffix), Publish: "summary", Description: description, - Server: OpenIDConnectConformanceSuitePlanServer{ - DiscoveryURL: b.autheliaURL.JoinPath(".well-known/openid-configuration").String(), + Server: PlanServer{ + DiscoveryURL: b.AutheliaURL.JoinPath(".well-known/openid-configuration").String(), }, }, } - if b.suiteURL == nil { + if b.SuiteURL == nil { return suite } @@ -100,17 +107,17 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS secretAlternate := r.StringCustom(80, random.CharSetAlphaNumeric) secretPost := r.StringCustom(80, random.CharSetAlphaNumeric) - suite.Plan.Client = &OpenIDConnectConformanceSuitePlanClient{ + suite.Plan.Client = &PlanClient{ ID: fmt.Sprintf("%s-%s", clientIDPrefix, aliasSuffix), Secret: secret, } - suite.Plan.ClientAlternate = &OpenIDConnectConformanceSuitePlanClient{ + suite.Plan.ClientAlternate = &PlanClient{ ID: fmt.Sprintf("%s-%s-alt", clientIDPrefix, aliasSuffix), Secret: secretAlternate, } - suite.Plan.ClientSecretPost = &OpenIDConnectConformanceSuitePlanClient{ + suite.Plan.ClientSecretPost = &PlanClient{ ID: fmt.Sprintf("%s-%s-post", clientIDPrefix, aliasSuffix), Secret: secretPost, } @@ -121,11 +128,11 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS responseModes []string ) - switch b.name { - case "implicit", suiteNameImplicitFormPost: + switch b.Name { + case NameImplicit, NameImplicitFormPost: grantTypes = []string{oidc.GrantTypeAuthorizationCode, oidc.GrantTypeImplicit, oidc.GrantTypeRefreshToken} responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow, oidc.ResponseTypeImplicitFlowIDToken, oidc.ResponseTypeImplicitFlowToken, oidc.ResponseTypeImplicitFlowBoth} - case "hybrid", suiteNameHybridFormPost: + case NameHybrid, NameHybridFormPost: grantTypes = []string{oidc.GrantTypeAuthorizationCode, oidc.GrantTypeImplicit, oidc.GrantTypeRefreshToken} responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow, oidc.ResponseTypeHybridFlowIDToken, oidc.ResponseTypeHybridFlowToken, oidc.ResponseTypeHybridFlowBoth} default: @@ -133,8 +140,8 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS responseTypes = []string{oidc.ResponseTypeAuthorizationCodeFlow} } - switch b.name { - case suiteNameBasicFormPost, suiteNameHybridFormPost, suiteNameImplicitFormPost: + switch b.Name { + case NameBasicFormPost, NameHybridFormPost, NameImplicitFormPost: responseModes = []string{oidc.ResponseModeFormPost, oidc.ResponseModeFormPostJWT} default: responseModes = []string{oidc.ResponseModeQuery, oidc.ResponseModeQueryJWT} @@ -145,9 +152,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS ID: suite.Plan.Client.ID, Name: description, Secret: MustHash(suite.Plan.Client.Secret), - RedirectURIs: []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, - AuthorizationPolicy: b.policy, - ConsentMode: b.consent, + RedirectURIs: []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, + AuthorizationPolicy: b.Policy, + ConsentMode: b.Consent, Public: false, Scopes: []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"}, ResponseTypes: responseTypes, @@ -160,9 +167,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS ID: suite.Plan.ClientAlternate.ID, Name: fmt.Sprintf("%s (Alternate)", description), Secret: MustHash(suite.Plan.ClientAlternate.Secret), - RedirectURIs: []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, - AuthorizationPolicy: b.policy, - ConsentMode: b.consent, + RedirectURIs: []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, + AuthorizationPolicy: b.Policy, + ConsentMode: b.Consent, Public: false, Scopes: []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"}, ResponseTypes: responseTypes, @@ -175,9 +182,9 @@ func (b *OpenIDConnectConformanceSuiteBuilder) Build() OpenIDConnectConformanceS ID: suite.Plan.ClientSecretPost.ID, Name: fmt.Sprintf("%s (Secret Post)", description), Secret: MustHash(suite.Plan.ClientSecretPost.Secret), - RedirectURIs: []string{b.suiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, - AuthorizationPolicy: b.policy, - ConsentMode: b.consent, + RedirectURIs: []string{b.SuiteURL.JoinPath("test", "a", suite.Plan.Alias, "callback").String()}, + AuthorizationPolicy: b.Policy, + ConsentMode: b.Consent, Public: false, Scopes: []string{oidc.ScopeOpenID, oidc.ScopeOfflineAccess, oidc.ScopeProfile, oidc.ScopeEmail, oidc.ScopePhone, oidc.ScopeAddress, "all"}, ResponseTypes: responseTypes, @@ -206,3 +213,16 @@ func MustHash(value string) *schema.PasswordDigest { return schema.NewPasswordDigest(digest) } + +// Builders returns the conformance suite builders for every profile Authelia is certified for, in a fixed order. +func Builders(version, consent, policy, brand string, suiteURL, autheliaURL *url.URL) []*SuiteBuilder { + return []*SuiteBuilder{ + {brand, NameConfig, "Config", true, version, consent, policy, nil, autheliaURL}, + {brand, NameBasic, "Basic", true, version, consent, policy, suiteURL, autheliaURL}, + {brand, NameBasicFormPost, "Basic (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, + {brand, NameHybrid, "Hybrid", true, version, consent, policy, suiteURL, autheliaURL}, + {brand, NameHybridFormPost, "Hybrid (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, + {brand, NameImplicit, "Implicit", true, version, consent, policy, suiteURL, autheliaURL}, + {brand, NameImplicitFormPost, "Implicit (Form Post)", true, version, consent, policy, suiteURL, autheliaURL}, + } +} diff --git a/cmd/authelia-gen/openid_conformance_test.go b/internal/oidc/conformance/builder_test.go similarity index 78% rename from cmd/authelia-gen/openid_conformance_test.go rename to internal/oidc/conformance/builder_test.go index f1b0dd4b4..77d881cf5 100644 --- a/cmd/authelia-gen/openid_conformance_test.go +++ b/internal/oidc/conformance/builder_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package main +package conformance import ( "net/url" @@ -20,7 +20,16 @@ func TestMustHash(t *testing.T) { }) } -func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { +func TestSuiteBuilder_Description(t *testing.T) { + certification := &SuiteBuilder{Friendly: "Basic", Certification: true, Version: "k3j2x9ab", AutheliaURL: &url.URL{Scheme: "https", Host: "auth.example.com"}} + test := &SuiteBuilder{Friendly: "Basic (Form Post)", Version: "k3j2x9ab"} + + assert.Equal(t, "Authelia v4.39.24 (commit 7498f635) Basic Certification Profile", certification.Description("v4.39.24 (commit 7498f635)")) + assert.Equal(t, "Authelia commit 7498f635 Basic (Form Post) Test Profile", test.Description("commit 7498f635")) + assert.Equal(t, certification.Build().Plan.Description, certification.Description(certification.Version), "the build describes the plan by its version") +} + +func TestSuiteBuilder_Build(t *testing.T) { suiteURL := &url.URL{ Scheme: "https", Host: "conformance.example.com", @@ -35,20 +44,40 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { testCases := []struct { name string - have *OpenIDConnectConformanceSuiteBuilder - expected OpenIDConnectConformanceSuite + have *SuiteBuilder + expected Suite }{ { "ShouldHandleConfig", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "config", "Config", true, "4.40", "implicit", "one_factor", nil, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "config", "Config", true, "4.40", "implicit", "one_factor", nil, autheliaURL}, + Suite{ Name: "conformance-config", - Plan: OpenIDConnectConformanceSuitePlan{ + Plan: Plan{ Name: "oidcc-config-certification-test-plan", Alias: "conformance-config-authelia440", Description: "Authelia 4.40 Config Certification Profile", Publish: "summary", - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ + DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", + }, + }, + }, + }, + { + "ShouldHandleBasicTestProfile", + &SuiteBuilder{"authelia", "basic", "Basic", false, "4.40", "implicit", "one_factor", nil, autheliaURL}, + Suite{ + Name: "basic", + Plan: Plan{ + Name: "oidcc-basic-certification-test-plan", + Alias: "basic-authelia440", + Description: "Authelia 4.40 Basic Test Profile", + Publish: "summary", + Variant: &PlanVariant{ + ServerMetadata: "discovery", + ClientRegistration: "static_client", + }, + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, }, @@ -56,30 +85,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleBasic", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "basic", "Basic", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "basic", "Basic", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-basic", - Plan: OpenIDConnectConformanceSuitePlan{ + Plan: Plan{ Name: "oidcc-basic-certification-test-plan", Alias: "conformance-basic-authelia440", Description: "Authelia 4.40 Basic Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-basic-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-basic-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-basic-authelia440-post", Secret: "present", }, @@ -126,30 +155,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleBasicFormPost", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "basic-form-post", "Basic (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "basic-form-post", "Basic (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-basic-form-post", - Plan: OpenIDConnectConformanceSuitePlan{ - Name: "oidcc-basic-form-post-certification-test-plan", + Plan: Plan{ + Name: "oidcc-formpost-basic-certification-test-plan", Alias: "conformance-basic-form-post-authelia440", Description: "Authelia 4.40 Basic (Form Post) Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-basic-form-post-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-basic-form-post-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-basic-form-post-authelia440-post", Secret: "present", }, @@ -196,30 +225,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleImplicit", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "implicit", "Implicit", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "implicit", "Implicit", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-implicit", - Plan: OpenIDConnectConformanceSuitePlan{ + Plan: Plan{ Name: "oidcc-implicit-certification-test-plan", Alias: "conformance-implicit-authelia440", Description: "Authelia 4.40 Implicit Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-implicit-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-implicit-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-implicit-authelia440-post", Secret: "present", }, @@ -266,30 +295,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleImplicitFormPost", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "implicit-form-post", "Implicit (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "implicit-form-post", "Implicit (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-implicit-form-post", - Plan: OpenIDConnectConformanceSuitePlan{ - Name: "oidcc-implicit-form-post-certification-test-plan", + Plan: Plan{ + Name: "oidcc-formpost-implicit-certification-test-plan", Alias: "conformance-implicit-form-post-authelia440", Description: "Authelia 4.40 Implicit (Form Post) Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-implicit-form-post-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-implicit-form-post-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-implicit-form-post-authelia440-post", Secret: "present", }, @@ -336,30 +365,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleHybrid", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "hybrid", "Hybrid", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "hybrid", "Hybrid", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-hybrid", - Plan: OpenIDConnectConformanceSuitePlan{ + Plan: Plan{ Name: "oidcc-hybrid-certification-test-plan", Alias: "conformance-hybrid-authelia440", Description: "Authelia 4.40 Hybrid Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-hybrid-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-hybrid-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-hybrid-authelia440-post", Secret: "present", }, @@ -406,30 +435,30 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }, { "ShouldHandleHybridFormPost", - &OpenIDConnectConformanceSuiteBuilder{"authelia", "hybrid-form-post", "Hybrid (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, - OpenIDConnectConformanceSuite{ + &SuiteBuilder{"authelia", "hybrid-form-post", "Hybrid (Form Post)", true, "4.40", "implicit", "one_factor", suiteURL, autheliaURL}, + Suite{ Name: "conformance-hybrid-form-post", - Plan: OpenIDConnectConformanceSuitePlan{ - Name: "oidcc-hybrid-form-post-certification-test-plan", + Plan: Plan{ + Name: "oidcc-formpost-hybrid-certification-test-plan", Alias: "conformance-hybrid-form-post-authelia440", Description: "Authelia 4.40 Hybrid (Form Post) Certification Profile", Publish: "summary", - Variant: &OpenIDConnectConformanceSuitePlanVariant{ + Variant: &PlanVariant{ ServerMetadata: "discovery", ClientRegistration: "static_client", }, - Server: OpenIDConnectConformanceSuitePlanServer{ + Server: PlanServer{ DiscoveryURL: "https://auth.example.com/.well-known/openid-configuration", }, - Client: &OpenIDConnectConformanceSuitePlanClient{ + Client: &PlanClient{ ID: "conformance-certification-hybrid-form-post-authelia440", Secret: "present", }, - ClientAlternate: &OpenIDConnectConformanceSuitePlanClient{ + ClientAlternate: &PlanClient{ ID: "conformance-certification-hybrid-form-post-authelia440-alt", Secret: "present", }, - ClientSecretPost: &OpenIDConnectConformanceSuitePlanClient{ + ClientSecretPost: &PlanClient{ ID: "conformance-certification-hybrid-form-post-authelia440-post", Secret: "present", }, @@ -481,6 +510,14 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { actual := tc.have.Build() assert.Equal(t, tc.expected.Name, actual.Name) + assert.Equal(t, tc.expected.Plan.Name, actual.Plan.Name) + assert.Equal(t, tc.expected.Plan.Variant, actual.Plan.Variant) + + assert.Equal(t, tc.expected.Plan.Alias, actual.Plan.Alias) + assert.Equal(t, tc.expected.Plan.Description, actual.Plan.Description) + assert.Equal(t, tc.expected.Plan.Publish, actual.Plan.Publish) + assert.Equal(t, tc.expected.Plan.Server, actual.Plan.Server) + require.Equal(t, len(tc.expected.Clients), len(actual.Clients)) for i, expected := range tc.expected.Clients { @@ -524,3 +561,31 @@ func TestOpenIDConnectConformanceSuiteBuilder_Build(t *testing.T) { }) } } + +func TestBuilders(t *testing.T) { + suiteURL := &url.URL{Scheme: "https", Host: "conformance.example.com"} + autheliaURL := &url.URL{Scheme: "https", Host: "auth.example.com"} + + builders := Builders("4.40", "implicit", "one_factor", "authelia", suiteURL, autheliaURL) + + names := make([]string, len(builders)) + + for i, builder := range builders { + names[i] = builder.Name + + assert.True(t, builder.Certification, builder.Name) + assert.Equal(t, "authelia", builder.Brand, builder.Name) + assert.Equal(t, "4.40", builder.Version, builder.Name) + assert.Equal(t, "implicit", builder.Consent, builder.Name) + assert.Equal(t, "one_factor", builder.Policy, builder.Name) + assert.Equal(t, autheliaURL, builder.AutheliaURL, builder.Name) + + if builder.Name == NameConfig { + assert.Nil(t, builder.SuiteURL, "the config profile registers no clients, so it has no suite to redirect to") + } else { + assert.Equal(t, suiteURL, builder.SuiteURL, builder.Name) + } + } + + assert.Equal(t, []string{NameConfig, NameBasic, NameBasicFormPost, NameHybrid, NameHybridFormPost, NameImplicit, NameImplicitFormPost}, names) +} diff --git a/internal/oidc/conformance/const.go b/internal/oidc/conformance/const.go new file mode 100644 index 000000000..3c27176c9 --- /dev/null +++ b/internal/oidc/conformance/const.go @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +// Package conformance provides the OpenID Connect 1.0 conformance suite plan and client definitions Authelia is +// certified against. It is shared by the authelia-gen conformance generator and the OIDCConformance integration suite +// so that a plan's client identifiers, secrets and redirect URIs cannot drift from the clients Authelia is configured +// with. +package conformance + +const ( + // NameConfig is the builder name of the Config OP profile. + NameConfig = "config" + + // NameBasic is the builder name of the Basic OP profile. + NameBasic = "basic" + + // NameBasicFormPost is the builder name of the Basic OP profile using the form post response mode. + NameBasicFormPost = "basic-form-post" + + // NameHybrid is the builder name of the Hybrid OP profile. + NameHybrid = "hybrid" + + // NameHybridFormPost is the builder name of the Hybrid OP profile using the form post response mode. + NameHybridFormPost = "hybrid-form-post" + + // NameImplicit is the builder name of the Implicit OP profile. + NameImplicit = "implicit" + + // NameImplicitFormPost is the builder name of the Implicit OP profile using the form post response mode. + NameImplicitFormPost = "implicit-form-post" +) diff --git a/internal/oidc/conformance/types.go b/internal/oidc/conformance/types.go new file mode 100644 index 000000000..b829c64a1 --- /dev/null +++ b/internal/oidc/conformance/types.go @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package conformance + +import ( + "github.com/authelia/authelia/v4/internal/configuration/schema" +) + +// Suite represents a single OpenID Connect conformance suite. +type Suite struct { + Name string + Plan Plan + Clients []schema.IdentityProvidersOpenIDConnectClient +} + +// Plan represents the test plan for a Suite. +type Plan struct { + Name string `json:"-"` + Alias string `json:"alias"` + Description string `json:"description"` + Publish string `json:"publish"` + + Variant *PlanVariant `json:"-"` + Server PlanServer `json:"server"` + Client *PlanClient `json:"client,omitempty"` + ClientAlternate *PlanClient `json:"client2,omitempty"` + ClientSecretPost *PlanClient `json:"client_secret_post,omitempty"` + MutualTLS *PlanMutualTLS `json:"mtls,omitempty"` + Resource *PlanResource `json:"resource,omitempty"` + + EKYCVerifiedClaimsRequest string `json:"ekyc_verified_claims_request,omitempty"` + EKYCUserinfo string `json:"ekyc_userinfo,omitempty"` +} + +// PlanVariant represents the variant of a Plan. +type PlanVariant struct { + ServerMetadata string `json:"server_metadata,omitempty"` + ClientRegistration string `json:"client_registration,omitempty"` + ClientAuthType string `json:"client_auth_type,omitempty"` + ResponseType string `json:"response_type,omitempty"` + ResponseMode string `json:"response_mode,omitempty"` + CIBAMode string `json:"ciba_mode,omitempty"` + FAPIProfile string `json:"fapi_profile,omitempty"` + FAPIResponseMode string `json:"fapi_response_mode,omitempty"` + FAPIAuthRequestMethod string `json:"fapi_auth_request_method,omitempty"` + SenderConstrain string `json:"sender_constrain,omitempty"` + AuthorizationRequestType string `json:"authorization_request_type,omitempty"` + OpenID string `json:"openid,omitempty"` +} + +// PlanServer represents the server details of a Plan. +type PlanServer struct { + DiscoveryURL string `json:"discoveryUrl,omitempty"` + Issuer string `json:"issuer,omitempty"` + JWKSURI string `json:"jwks_uri,omitempty"` + AuthorizationEndpoint string `json:"authorization_endpoint,omitempty"` + TokenEndpoint string `json:"token_endpoint,omitempty"` + UserinfoEndpoint string `json:"userinfo_endpoint,omitempty"` + ACRValues string `json:"acr_values,omitempty"` + LoginHint string `json:"login_hint,omitempty"` +} + +// PlanClient represents a client registered with a Plan. +type PlanClient struct { + ID string `json:"client_id,omitempty"` + Secret string `json:"client_secret,omitempty"` + Name string `json:"client_name,omitempty"` + Scope string `json:"scope,omitempty"` + SecretJWTAlgorithm string `json:"client_secret_jwt_alg,omitempty"` + DPOPSigningAlgorithm string `json:"dpop_signing_alg,omitempty"` + InitialAccessToken string `json:"initial_access_token,omitempty"` + HintType string `json:"hint_type,omitempty"` + HintValue string `json:"hint_value,omitempty"` + JWKS string `json:"jwks,omitempty"` +} + +// PlanMutualTLS represents the Mutual TLS details of a Plan. +type PlanMutualTLS struct { + Certificate string `json:"certificate,omitempty"` + Key string `json:"key,omitempty"` + CertificateAuthority string `json:"ca,omitempty"` +} + +// PlanResource represents the protected resource details of a Plan. +type PlanResource struct { + ResourceURL string `json:"resourceUrl"` + ResourceURLAccountRequests string `json:"resourceUrlAccountRequests,omitempty"` + ResourceURLAccountsResource string `json:"resourceUrlAccountsResource,omitempty"` + InstitutionID string `json:"institution_id,omitempty"` + RichAuthorizationRequest string `json:"richAuthorizationRequest,omitempty"` +} diff --git a/internal/suites/OIDCConformance/Dockerfile.server b/internal/suites/OIDCConformance/Dockerfile.server new file mode 100644 index 000000000..a6b067870 --- /dev/null +++ b/internal/suites/OIDCConformance/Dockerfile.server @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2026 Authelia +# +# SPDX-License-Identifier: Apache-2.0 + +FROM registry.gitlab.com/openid/conformance-suite:release-v5.2.4 + +# The conformance server fetches Authelia's discovery document, JWKS, token endpoint and userinfo endpoint over TLS +# signed by the suite development CA, which no public root vouches for. Importing into the JDK's own cacerts adds that +# CA while leaving the public roots in place, which a -Djavax.net.ssl.trustStore override would not. +COPY --link common/pki/ca.public.crt /tmp/authelia-ca.crt + +RUN keytool -importcert -noprompt -alias authelia-development-ca -file /tmp/authelia-ca.crt -cacerts -storepass changeit diff --git a/internal/suites/OIDCConformance/compose.mongodb.yml b/internal/suites/OIDCConformance/compose.mongodb.yml new file mode 100644 index 000000000..3fd50b6ad --- /dev/null +++ b/internal/suites/OIDCConformance/compose.mongodb.yml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Authelia +# +# SPDX-License-Identifier: Apache-2.0 + +--- +services: + conformance-mongodb: + image: 'mongo:6.0.13' + cpus: 2 + mem_limit: 2g + networks: + authelianet: {} + + conformance-server: + depends_on: + - 'conformance-mongodb' +... diff --git a/internal/suites/OIDCConformance/compose.yml b/internal/suites/OIDCConformance/compose.yml new file mode 100644 index 000000000..024191fe9 --- /dev/null +++ b/internal/suites/OIDCConformance/compose.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2026 Authelia +# +# SPDX-License-Identifier: Apache-2.0 + +--- +services: + authelia-backend: + environment: + X_AUTHELIA_CONFIG_FILTERS: 'template' + X_AUTHELIA_CONFIG: '/config/configuration.yml,/tmp/conformance-clients.yml' + volumes: + - './OIDCConformance/configuration.yml:/config/configuration.yml' + - './OIDCConformance/users.yml:/config/users.yml' + - './common/pki:/pki' + - './common/pki/public.crt:/certs/public.crt' + - '${SUITE_TMP:-/tmp}:/tmp' + + conformance-server: + build: + context: '.' + dockerfile: 'OIDCConformance/Dockerfile.server' + cpus: 4 + mem_limit: 4g + environment: + BASE_URL: 'https://conformance.example.com:8443' + MONGODB_HOST: '${SUITE_OIDC_CONFORMANCE_MONGODB_HOST:-conformance-mongodb}' + JAVA_EXTRA_ARGS: '-Dfintechlabs.devmode=true' + OIDC_GOOGLE_CLIENTID: 'google-client' + OIDC_GOOGLE_SECRET: 'google-secret' + OIDC_GITLAB_CLIENTID: 'gitlab-client' + OIDC_GITLAB_SECRET: 'gitlab-secret' + depends_on: + - 'authelia-backend' + expose: + - 8080 + networks: + authelianet: {} + +... diff --git a/internal/suites/OIDCConformance/configuration.yml b/internal/suites/OIDCConformance/configuration.yml new file mode 100644 index 000000000..da0f2fc06 --- /dev/null +++ b/internal/suites/OIDCConformance/configuration.yml @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 Authelia +# +# SPDX-License-Identifier: Apache-2.0 + +--- +certificates_directory: '/certs/' + +server: + address: 'tcp://:9091' + timeouts: + read: '30s' + tls: + certificate: '/pki/public.backend.crt' + key: '/pki/private.backend.pem' + +log: + level: 'debug' + +storage: + encryption_key: 'a_not_so_secure_encryption_key' + postgres: + address: 'tcp://postgres:5432' + database: 'authelia' + username: 'admin' + password: 'password' + +notifier: + smtp: + address: 'smtp://mail.example.com:1025' + sender: 'admin@example.com' + +identity_validation: + reset_password: + jwt_secret: 'unsecure_secret' + + +# Every plan in this suite authenticates as the same user against this one Authelia, concurrently, and a module which +# leaves the sign in form on screen has its credentials resubmitted. Three failures within the default two minute +# window would ban the account for the default five minutes, failing unrelated modules in the other six plans with +# errors which point nowhere near the cause. Regulation is not what this suite tests, so it is disabled outright. +regulation: + max_retries: 0 + +session: + secret: 'unsecure_session_secret' + cookies: + - domain: 'example.com' + authelia_url: 'https://login.example.com:8080' + expiration: '1 hour' + inactivity: '5 minutes' + remember_me: '1 year' + redis: + host: 'redis' + port: 6379 + +authentication_backend: + file: + path: '/config/users.yml' + +totp: + disable_reuse_security_policy: true + +webauthn: + disable: false + enable_passkey_login: true + display_name: 'Authelia' + attestation_conveyance_preference: 'indirect' + timeout: '60 seconds' + filtering: + permitted_aaguids: [] + prohibited_aaguids: [] + prohibit_backup_eligibility: false + selection_criteria: + attachment: '' + discoverability: 'required' + user_verification: 'preferred' + metadata: + enabled: false + validate_trust_anchor: true + validate_entry: false + validate_entry_permit_zero_aaguid: true + validate_status: true + validate_status_permitted: [] + validate_status_prohibited: [] + +identity_providers: + oidc: + enable_client_debug_messages: true + hmac_secret: 'IVPWBkAdJHje3uz7LtFTDU2pFUfh39Xm' + jwks: + - key: {{ secret "/pki/private.oidc.pem" | mindent 10 "|" | msquote }} + certificate_chain: {{ secret "/pki/public.oidc.chain.pem" | mindent 10 "|" | msquote }} +access_control: + default_policy: 'deny' + rules: + - domain: 'home.example.com' + policy: 'bypass' + - domain: 'public.example.com' + policy: 'bypass' + - domain: 'admin.example.com' + policy: 'two_factor' + - domain: 'secure.example.com' + policy: 'two_factor' + - domain: 'singlefactor.example.com' + policy: 'one_factor' + - domain: 'oidc.example.com' + policy: 'two_factor' + - domain: 'oidc-public.example.com' + policy: 'bypass' +... diff --git a/internal/suites/OIDCConformance/users.yml b/internal/suites/OIDCConformance/users.yml new file mode 100644 index 000000000..6b98a2819 --- /dev/null +++ b/internal/suites/OIDCConformance/users.yml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: 2026 Authelia +# +# SPDX-License-Identifier: Apache-2.0 + +--- +users: + john: + displayname: "John Doe" + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" # yamllint disable-line rule:line-length + given_name: "John" + middle_name: "Richard" + family_name: "Doe" + nickname: "Johnny" + gender: "male" + birthdate: "1985-07-31" + website: "https://john.example.com" + profile: "https://john.example.com/profile" + picture: "https://john.example.com/picture.png" + zoneinfo: "Australia/Melbourne" + locale: "en-AU" + phone_number: "+610000000000" + phone_extension: "1234" + email: john.doe@authelia.com + address: + street_address: "123 Example Street" + locality: "Melbourne" + region: "Victoria" + postal_code: "3000" + country: "Australia" + groups: + - admins + - dev + + harry: + displayname: "Harry Potter" + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" # yamllint disable-line rule:line-length + email: harry.potter@authelia.com + groups: [] + + bob: + displayname: "Bob Dylan" + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" # yamllint disable-line rule:line-length + email: bob.dylan@authelia.com + groups: + - dev + + james: + displayname: "James Dean" + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" # yamllint disable-line rule:line-length + email: james.dean@authelia.com +... diff --git a/internal/suites/conformance_api.go b/internal/suites/conformance_api.go new file mode 100644 index 000000000..e21206561 --- /dev/null +++ b/internal/suites/conformance_api.go @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/authelia/authelia/v4/internal/oidc/conformance" + "github.com/authelia/authelia/v4/internal/utils" +) + +const ( + conformanceWaitStateTimeout = time.Second * 30 + conformanceReadyInterval = time.Second * 2 + conformanceWaitStateRetryInterval = time.Millisecond * 500 +) + +// ConformancePlanModule is one entry in a conformance test plan's module list. +type ConformancePlanModule struct { + TestModule string `json:"testModule"` + Variant map[string]string `json:"variant"` +} + +// ConformanceCreatedPlan is the response to creating a conformance test plan. +type ConformanceCreatedPlan struct { + ID string `json:"id"` + Modules []ConformancePlanModule `json:"modules"` +} + +// ConformanceTestInfo is the persisted state of a single conformance test module instance. +type ConformanceTestInfo struct { + ID string `json:"_id"` + TestName string `json:"testName"` + Status string `json:"status"` + Result string `json:"result"` +} + +// ConformanceBrowserStatus describes the URLs a module is waiting for a browser to visit. +type ConformanceBrowserStatus struct { + URLs []string `json:"urls"` + Visited []string `json:"visited"` +} + +// ConformanceLogEntry is one entry of a module's log. An entry with a non-empty Upload is an image placeholder which +// has not been filled, and which is holding the module in WAITING. +type ConformanceLogEntry struct { + Msg string `json:"msg"` + Result string `json:"result"` + Src string `json:"src"` + Upload string `json:"upload"` + + Fields map[string]any `json:"-"` +} + +// UnmarshalJSON decodes the typed fields and retains the entry whole. +func (e *ConformanceLogEntry) UnmarshalJSON(data []byte) (err error) { + type entry ConformanceLogEntry + + if err = json.Unmarshal(data, (*entry)(e)); err != nil { + return err + } + + return json.Unmarshal(data, &e.Fields) +} + +// ConformanceClient talks to the OpenID Foundation conformance suite's HTTP API. The suite runs with +// fintechlabs.devmode enabled, which disables its OAuth login, so no credentials are sent. +type ConformanceClient struct { + base *url.URL + client *http.Client +} + +// NewConformanceClient returns a ConformanceClient for the conformance suite at base. +func NewConformanceClient(base string) (client *ConformanceClient, err error) { + var parsed *url.URL + + if parsed, err = url.ParseRequestURI(base); err != nil { + return nil, fmt.Errorf("error parsing conformance base url: %w", err) + } + + return &ConformanceClient{ + base: parsed, + client: &http.Client{ + Transport: NewHTTPTransport(), + Timeout: conformanceWaitStateTimeout + time.Second*10, + }, + }, nil +} + +func (c *ConformanceClient) uri(query url.Values, elem ...string) string { + uri := c.base.JoinPath(append([]string{"api"}, elem...)...) + + if query != nil { + uri.RawQuery = query.Encode() + } + + return uri.String() +} + +// LogDetailURL returns the human readable log page for a module, for inclusion in failure output. +func (c *ConformanceClient) LogDetailURL(id string) string { + uri := c.base.JoinPath("log-detail.html") + + query := uri.Query() + query.Set("log", id) + uri.RawQuery = query.Encode() + + return uri.String() +} + +type conformanceUnexpectedStatusError struct { + Method string + URI string + Body []byte + Expected int + Status int +} + +// Error implements the error interface. +func (e *conformanceUnexpectedStatusError) Error() string { + return fmt.Sprintf("%s %s: expected status %d but got %d: %s", e.Method, e.URI, e.Expected, e.Status, e.Body) +} + +func (c *ConformanceClient) do(ctx context.Context, method, uri string, body io.Reader, contentType string, expected int, out any) (err error) { + data, err := c.doRaw(ctx, method, uri, body, contentType, expected) + if err != nil { + return err + } + + if out == nil { + return nil + } + + return json.Unmarshal(data, out) +} + +func (c *ConformanceClient) doRaw(ctx context.Context, method, uri string, body io.Reader, contentType string, expected int) (data []byte, err error) { + var req *http.Request + + if req, err = http.NewRequestWithContext(ctx, method, uri, body); err != nil { + return nil, err + } + + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + + var resp *http.Response + + if resp, err = c.client.Do(req); err != nil { + return nil, err + } + + defer resp.Body.Close() + + if data, err = io.ReadAll(resp.Body); err != nil { + return nil, err + } + + if resp.StatusCode != expected { + return nil, &conformanceUnexpectedStatusError{Method: method, URI: uri, Body: data, Expected: expected, Status: resp.StatusCode} + } + + return data, nil +} + +// WaitReady blocks until the conformance server answers an API request or ctx expires. The JVM binds its port well +// before Spring has finished starting, so a connection check is not sufficient. +func (c *ConformanceClient) WaitReady(ctx context.Context) (err error) { + query := url.Values{} + query.Set("length", "1") + + for { + if err = c.do(ctx, http.MethodGet, c.uri(query, "plan"), nil, "", http.StatusOK, nil); err == nil { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("conformance suite was not ready: %w (last error: %v)", ctx.Err(), err) + case <-time.After(conformanceReadyInterval): + } + } +} + +// CreatePlan creates a conformance test plan and returns its id and module list. The plan name and the variant are +// parameters rather than being read off plan because [conformance.Plan] deliberately excludes both from its JSON +// representation: they belong in the query string, not the body, and the caller is the one which knows them. +func (c *ConformanceClient) CreatePlan(ctx context.Context, name string, variant *conformance.PlanVariant, plan *conformance.Plan) (created *ConformanceCreatedPlan, err error) { + if name == "" { + return nil, fmt.Errorf("error creating plan '%s': the conformance plan name is empty", plan.Alias) + } + + body, err := json.Marshal(plan) + if err != nil { + return nil, err + } + + query := url.Values{} + query.Set("planName", name) + + if variant != nil { + var data []byte + + if data, err = json.Marshal(variant); err != nil { + return nil, err + } + + query.Set("variant", string(data)) + } + + created = &ConformanceCreatedPlan{} + + if err = c.do(ctx, http.MethodPost, c.uri(query, "plan"), bytes.NewReader(body), "application/json", http.StatusCreated, created); err != nil { + return nil, err + } + + if created.ID == "" { + return nil, fmt.Errorf("error creating plan '%s': the conformance suite returned no plan identifier", plan.Alias) + } + + return created, nil +} + +// CreateTest creates an instance of a plan's module and returns its id. The variant is sent alongside the plan even +// though the endpoint documents it as applying to standalone tests: the upstream scripts/conformance.py sends both +// together for plan modules, and it is what makes plans which repeat a module across variants work. +func (c *ConformanceClient) CreateTest(ctx context.Context, planID, module string, variant map[string]string) (id string, err error) { + query := url.Values{} + query.Set("test", module) + query.Set("plan", planID) + + if len(variant) != 0 { + var data []byte + + if data, err = json.Marshal(variant); err != nil { + return "", err + } + + query.Set("variant", string(data)) + } + + out := struct { + ID string `json:"id"` + }{} + + if err = c.do(ctx, http.MethodPost, c.uri(query, "runner"), nil, "application/json", http.StatusCreated, &out); err != nil { + return "", err + } + + if out.ID == "" { + return "", fmt.Errorf("error creating module '%s': the conformance suite returned no test identifier", module) + } + + return out.ID, nil +} + +// StartTest starts a module which is waiting in the CONFIGURED state. +func (c *ConformanceClient) StartTest(ctx context.Context, id string) error { + return c.do(ctx, http.MethodPost, c.uri(nil, "runner", id), nil, "", http.StatusOK, nil) +} + +// WaitState long polls until the module reaches one of states, or ctx expires. The server clamps each poll to 30 +// seconds and answers {"timeout":true} when it expires, so this loops. Once a module finishes it leaves the running +// registry and the endpoint answers 404; the persisted status from Info is authoritative at that point. A transport +// level failure - a 5xx response, a network error - is treated as transient and retried until ctx expires, so a +// single blip during a long unattended run does not fail the poll outright. Any other status will not change on retry, +// so it fails immediately. +// +// A 200 whose body is not the documented shape is not retried. A body which does not decode at all, and a body which +// decodes but carries neither a state nor the timeout flag - an empty body, a proxy's own 200 page, a schema change +// on a suite version bump - are both permanent for as long as the run lasts, so retrying either would only spin +// against the server for the rest of the module budget from every plan at once. Both fail immediately with the body +// echoed, which is also the only way the reason for a schema change ever reaches the run's output. +func (c *ConformanceClient) WaitState(ctx context.Context, id string, states ...string) (state string, err error) { + query := url.Values{} + query.Set("states", strings.Join(states, ",")) + query.Set("timeoutMs", strconv.FormatInt(conformanceWaitStateTimeout.Milliseconds(), 10)) + + for { + data, err := c.doRaw(ctx, http.MethodGet, c.uri(query, "runner", id, "wait-state"), nil, "", http.StatusOK) + if err != nil { + if ctx.Err() != nil { + return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err()) + } + + var statusErr *conformanceUnexpectedStatusError + + if errors.As(err, &statusErr) { + switch { + case statusErr.Status == http.StatusNotFound: + return c.waitStateFromInfo(ctx, id, states) + case statusErr.Status < http.StatusInternalServerError: + return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, err) + } + } + + select { + case <-ctx.Done(): + return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err()) + case <-time.After(conformanceWaitStateRetryInterval): + } + + continue + } + + out := struct { + State string `json:"state"` + Timeout bool `json:"timeout"` + }{} + + if err = json.Unmarshal(data, &out); err != nil { + return "", fmt.Errorf("module '%s' wait-state poll answered with a body which is not JSON: %w: %s", id, err, data) + } + + switch { + case out.State != "": + return out.State, nil + case out.Timeout: + // The poll expired without a transition. Fall through to the context check and poll again. + default: + return "", fmt.Errorf("module '%s' wait-state poll answered with neither a state nor a timeout: %s", id, data) + } + + if ctx.Err() != nil { + return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err()) + } + } +} + +func (c *ConformanceClient) waitStateFromInfo(ctx context.Context, id string, states []string) (state string, err error) { + var info *ConformanceTestInfo + + if info, err = c.Info(ctx, id); err != nil { + if ctx.Err() != nil { + return "", fmt.Errorf("module '%s' did not reach one of %v: %w", id, states, ctx.Err()) + } + + return "", err + } + + if utils.IsStringInSlice(info.Status, states) { + return info.Status, nil + } + + return "", fmt.Errorf("module '%s' is no longer running and its persisted status is '%s', which is not one of %v", id, info.Status, states) +} + +// Info returns the persisted status and result of a module. +func (c *ConformanceClient) Info(ctx context.Context, id string) (info *ConformanceTestInfo, err error) { + info = &ConformanceTestInfo{} + + if err = c.do(ctx, http.MethodGet, c.uri(nil, "info", id), nil, "", http.StatusOK, info); err != nil { + return nil, err + } + + return info, nil +} + +// BrowserStatus returns the URLs a module is waiting to have visited. +func (c *ConformanceClient) BrowserStatus(ctx context.Context, id string) (status *ConformanceBrowserStatus, err error) { + status = &ConformanceBrowserStatus{} + + if err = c.do(ctx, http.MethodGet, c.uri(nil, "runner", "browser", id), nil, "", http.StatusOK, status); err != nil { + return nil, err + } + + return status, nil +} + +// Visit marks a URL as visited. The server matches by exact string equality, so uri must be the URL as it was given. +func (c *ConformanceClient) Visit(ctx context.Context, id, uri string) error { + query := url.Values{} + query.Set("url", uri) + + return c.do(ctx, http.MethodPost, c.uri(query, "runner", "browser", id, "visit"), nil, "", http.StatusNoContent, nil) +} + +// Log returns a module's log entries. +func (c *ConformanceClient) Log(ctx context.Context, id string) (entries []ConformanceLogEntry, err error) { + if err = c.do(ctx, http.MethodGet, c.uri(nil, "log", id), nil, "", http.StatusOK, &entries); err != nil { + return nil, err + } + + return entries, nil +} + +// UploadPlaceholder fills an image placeholder, releasing a module which is blocked in waitForPlaceholders(). +func (c *ConformanceClient) UploadPlaceholder(ctx context.Context, id, placeholder, dataURI string) error { + return c.do(ctx, http.MethodPost, c.uri(nil, "log", id, "images", placeholder), bytes.NewReader([]byte(dataURI)), "text/plain", http.StatusOK, nil) +} + +// ExportPlanHTML writes a plan's HTML export to path, for collection as a CI artifact. +func (c *ConformanceClient) ExportPlanHTML(ctx context.Context, planID, path string) (err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.uri(nil, "plan", "exporthtml", planID), nil) + if err != nil { + return err + } + + resp, err := c.client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status %d exporting plan '%s'", resp.StatusCode, planID) + } + + if err = os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + + defer f.Close() + + _, err = io.Copy(f, resp.Body) + + return err +} diff --git a/internal/suites/conformance_api_test.go b/internal/suites/conformance_api_test.go new file mode 100644 index 000000000..6542f5744 --- /dev/null +++ b/internal/suites/conformance_api_test.go @@ -0,0 +1,651 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authelia/authelia/v4/internal/oidc/conformance" +) + +const conformanceTestScreenshot = "data:image/png;base64,iVBORw0KGgo=" + +func TestConformanceClient_CreatePlan(t *testing.T) { + var query string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/plan", r.URL.Path) + + query = r.URL.RawQuery + + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"name":"oidcc-basic-certification-test-plan","id":"plan1","modules":[{"testModule":"oidcc-server","variant":{"response_type":"code"}}]}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + plan := &conformance.Plan{Alias: "alias"} + + created, err := client.CreatePlan(context.Background(), "oidcc-basic-certification-test-plan", &conformance.PlanVariant{ServerMetadata: "discovery"}, plan) + require.NoError(t, err) + + assert.Equal(t, "plan1", created.ID) + require.Len(t, created.Modules, 1) + assert.Equal(t, "oidcc-server", created.Modules[0].TestModule) + assert.Equal(t, "code", created.Modules[0].Variant["response_type"]) + assert.Contains(t, query, "planName=oidcc-basic-certification-test-plan") + assert.Contains(t, query, "variant=") +} + +func TestConformanceClient_CreatePlanRejectsAResponseWithoutAnIdentifier(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"name":"oidcc-basic-certification-test-plan","modules":[{"testModule":"oidcc-server"}]}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + _, err = client.CreatePlan(context.Background(), "oidcc-basic-certification-test-plan", nil, &conformance.Plan{Alias: "the-alias"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "the-alias") + assert.Contains(t, err.Error(), "returned no plan identifier") +} + +func TestConformanceClient_WaitStateReturnsPersistedStatusWhenTestNoLongerRunning(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner/abc/wait-state": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"test not found"}`)) + case "/api/info/abc": + _, _ = w.Write([]byte(`{"_id":"abc","testName":"oidcc-server","status":"FINISHED","result":"PASSED"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + state, err := client.WaitState(ctx, "abc", "FINISHED") + require.NoError(t, err) + assert.Equal(t, "FINISHED", state) +} + +func TestConformanceClient_WaitStateRetriesOnTimeout(t *testing.T) { + var calls int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path) + + calls++ + + if calls == 1 { + _, _ = w.Write([]byte(`{"timeout":true}`)) + + return + } + + _, _ = w.Write([]byte(`{"state":"WAITING"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + state, err := client.WaitState(ctx, "abc", "WAITING") + require.NoError(t, err) + assert.Equal(t, "WAITING", state) + assert.Equal(t, 2, calls) +} + +func TestConformanceClient_WaitStateRetriesOnTransientError(t *testing.T) { + var calls int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner/abc/wait-state": + calls++ + + if calls == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"internal error"}`)) + + return + } + + _, _ = w.Write([]byte(`{"state":"FINISHED"}`)) + case "/api/info/abc": + t.Errorf("unexpected call to %s: a transient 5xx must not fall back to Info", r.URL.Path) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + state, err := client.WaitState(ctx, "abc", "FINISHED") + require.NoError(t, err) + assert.Equal(t, "FINISHED", state) + assert.Equal(t, 2, calls) +} + +func TestConformanceClient_WaitStateReturnsFramedErrorOnPersistentTransientFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path) + + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"internal error"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*750) + defer cancel() + + start := time.Now() + + _, err = client.WaitState(ctx, "abc", "FINISHED") + + elapsed := time.Since(start) + + require.Error(t, err) + assert.Contains(t, err.Error(), "abc") + assert.Contains(t, err.Error(), "FINISHED") + assert.Less(t, elapsed, time.Second*5, "WaitState should return promptly once ctx expires rather than hang") +} + +func TestConformanceClient_CreateTestReportsAliasCollision(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":"alias already in use"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + _, err = client.CreateTest(context.Background(), "plan1", "oidcc-server", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "alias already in use") +} + +func TestConformanceClient_CreateTestRejectsAResponseWithoutAnIdentifier(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"name":"oidcc-server"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + id, err := client.CreateTest(context.Background(), "plan1", "oidcc-server", nil) + require.Error(t, err) + assert.Empty(t, id) + assert.Contains(t, err.Error(), "oidcc-server") + assert.Contains(t, err.Error(), "no test identifier") +} + +func TestConformanceClient_WaitStateFailsFastOnAClientError(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden} { + t.Run(http.StatusText(status), func(t *testing.T) { + var calls atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(status) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + _, err = client.WaitState(ctx, "m1", "FINISHED") + require.Error(t, err) + require.NoError(t, ctx.Err(), "a client error is not going to change on retry, so it must not wait out the context") + + var statusErr *conformanceUnexpectedStatusError + + require.ErrorAs(t, err, &statusErr) + assert.Equal(t, status, statusErr.Status) + assert.Contains(t, err.Error(), "module 'm1'") + assert.Equal(t, int32(1), calls.Load()) + }) + } +} + +func TestConformanceClient_LogExposesUnfilledPlaceholders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/log/abc", r.URL.Path) + + _, _ = w.Write([]byte(`[{"msg":"a"},{"msg":"upload me","upload":"ph1","result":"REVIEW"}]`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + entries, err := client.Log(context.Background(), "abc") + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, "", entries[0].Upload) + assert.Equal(t, "ph1", entries[1].Upload) +} + +func TestConformanceClient_UploadPlaceholder(t *testing.T) { + var body []byte + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/log/abc/images/ph1", r.URL.Path) + + body, _ = io.ReadAll(r.Body) + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + require.NoError(t, client.UploadPlaceholder(context.Background(), "abc", "ph1", conformanceTestScreenshot)) + assert.Equal(t, conformanceTestScreenshot, string(body)) +} + +func TestConformanceClient_CreatePlanRejectsAnEmptyPlanName(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("the client must not post a nameless plan, got %s", r.URL.String()) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + _, err = client.CreatePlan(context.Background(), "", nil, &conformance.Plan{Alias: "alias"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "alias") + assert.Contains(t, err.Error(), "plan name is empty") +} + +func TestConformanceClient_WaitStateFailsFastOnAnUnrecognizedBody(t *testing.T) { + testCases := []struct { + name string + body string + expected string + }{ + {"ShouldFailOnAnEmptyJSONObject", `{}`, "neither a state nor a timeout"}, + {"ShouldFailOnAFalseTimeout", `{"timeout":false}`, "neither a state nor a timeout"}, + {"ShouldFailOnASchemaChange", `{"status":"FINISHED"}`, "neither a state nor a timeout"}, + {"ShouldFailOnAMalformedBody", `gateway`, "is not JSON"}, + {"ShouldFailOnAnEmptyBody", ``, "is not JSON"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var calls int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/runner/abc/wait-state", r.URL.Path) + + calls++ + + _, _ = w.Write([]byte(tc.body)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + start := time.Now() + + _, err = client.WaitState(ctx, "abc", "FINISHED") + + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expected) + assert.Contains(t, err.Error(), "abc") + assert.Equal(t, 1, calls, "an unrecognized body must not be retried") + assert.Less(t, time.Since(start), time.Second, "an unrecognized body must fail fast rather than spin") + }) + } +} + +func TestNewConformanceClientRejectsAnInvalidURL(t *testing.T) { + _, err := NewConformanceClient("not a url") + + assert.ErrorContains(t, err, "error parsing conformance base url") +} + +func TestConformanceClient_WaitReady(t *testing.T) { + t.Run("ShouldReturnOnceTheServerAnswers", func(t *testing.T) { + var calls atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/plan", r.URL.Path) + assert.Equal(t, "1", r.URL.Query().Get("length")) + + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + + return + } + + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + require.NoError(t, client.WaitReady(ctx)) + assert.Equal(t, int32(2), calls.Load(), "the server is asked again after it could not answer") + }) + + t.Run("ShouldGiveUpWhenTheContextExpires", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + + err = client.WaitReady(ctx) + + assert.ErrorContains(t, err, "conformance suite was not ready") + assert.ErrorContains(t, err, "expected status 200 but got 503") + }) +} + +func TestConformanceClient_Requests(t *testing.T) { + var ( + mu sync.Mutex + requests []string + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + + requests = append(requests, r.Method+" "+r.URL.RequestURI()) + mu.Unlock() + + switch r.URL.Path { + case "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"m1"}`)) + case "/api/runner/m1": + _, _ = w.Write([]byte(`{}`)) + case "/api/runner/browser/m1/visit": + w.WriteHeader(http.StatusNoContent) + case "/api/plan/exporthtml/p1": + _, _ = w.Write([]byte("PK archive")) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx := context.Background() + + id, err := client.CreateTest(ctx, "p1", "oidcc-server", map[string]string{"response_type": "code"}) + require.NoError(t, err) + assert.Equal(t, "m1", id) + + require.NoError(t, client.StartTest(ctx, "m1")) + require.NoError(t, client.Visit(ctx, "m1", "https://login.example.com:8080/api/oidc/authorization?client_id=a&state=b")) + + archive := filepath.Join(t.TempDir(), "plans", "conformance-basic.zip") + + require.NoError(t, client.ExportPlanHTML(ctx, "p1", archive)) + + data, err := os.ReadFile(archive) + require.NoError(t, err) + assert.Equal(t, "PK archive", string(data)) + + mu.Lock() + defer mu.Unlock() + + assert.Equal(t, []string{ + "POST /api/runner?plan=p1&test=oidcc-server&variant=%7B%22response_type%22%3A%22code%22%7D", + "POST /api/runner/m1", + "POST /api/runner/browser/m1/visit?url=https%3A%2F%2Flogin.example.com%3A8080%2Fapi%2Foidc%2Fauthorization%3Fclient_id%3Da%26state%3Db", + "GET /api/plan/exporthtml/p1", + }, requests) +} + +func TestConformanceClient_ReportsServerErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"broken"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx := context.Background() + + _, err = client.CreatePlan(ctx, "oidcc-basic-certification-test-plan", nil, &conformance.Plan{Alias: "alias"}) + assert.ErrorContains(t, err, "broken") + + _, err = client.Info(ctx, "m1") + assert.ErrorContains(t, err, "broken") + + _, err = client.BrowserStatus(ctx, "m1") + assert.ErrorContains(t, err, "broken") + + _, err = client.Log(ctx, "m1") + assert.ErrorContains(t, err, "broken") + + err = client.ExportPlanHTML(ctx, "p1", filepath.Join(t.TempDir(), "conformance-basic.zip")) + assert.EqualError(t, err, "unexpected status 500 exporting plan 'p1'") +} + +func TestConformanceClient_ExportPlanHTMLReportsADirectoryItCannotCreate(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("PK archive")) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + file := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(file, nil, 0600)) + + assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", filepath.Join(file, "conformance-basic.zip"))) +} + +func TestConformanceClient_LogRejectsAMalformedEntry(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[{"msg":5}]`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + _, err = client.Log(context.Background(), "m1") + assert.Error(t, err) +} + +func TestConformanceClient_WaitStateFromInfo(t *testing.T) { + testCases := []struct { + name string + info func(w http.ResponseWriter) + expected string + }{ + { + "ShouldReportAPersistedStatusWhichIsNotWaitedFor", + func(w http.ResponseWriter) { _, _ = w.Write([]byte(`{"_id":"m1","status":"INTERRUPTED"}`)) }, + "module 'm1' is no longer running and its persisted status is 'INTERRUPTED', which is not one of [FINISHED]", + }, + { + "ShouldReportThatThePersistedStatusCouldNotBeRead", + func(w http.ResponseWriter) { w.WriteHeader(http.StatusInternalServerError) }, + "expected status 200 but got 500", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner/m1/wait-state": + w.WriteHeader(http.StatusNotFound) + case "/api/info/m1": + tc.info(w) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + _, err = client.WaitState(context.Background(), "m1", "FINISHED") + + assert.ErrorContains(t, err, tc.expected) + }) + } +} + +func TestConformanceClient_WaitStateStopsWhenTheContextExpiresBetweenPolls(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(time.Millisecond * 20) + + _, _ = w.Write([]byte(`{"timeout":true}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + + _, err = client.WaitState(ctx, "m1", "FINISHED") + + assert.ErrorContains(t, err, "module 'm1' did not reach one of [FINISHED]") +} + +func TestConformanceClient_ExportPlanHTMLReportsWhatFailed(t *testing.T) { + t.Run("ShouldReportAServerWhichCannotBeReached", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + server.Close() + + assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", filepath.Join(t.TempDir(), "conformance-basic.zip"))) + }) + + t.Run("ShouldReportAnArchiveWhichCannotBeCreated", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("PK archive")) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + assert.Error(t, client.ExportPlanHTML(context.Background(), "p1", t.TempDir()), "the path is a directory") + }) +} + +func TestConformanceClient_WaitStateFromInfoStopsWhenTheContextExpires(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/runner/m1/wait-state" { + w.WriteHeader(http.StatusNotFound) + + return + } + + time.Sleep(time.Millisecond * 200) + + _, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`)) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + + _, err = client.WaitState(ctx, "m1", "FINISHED") + + assert.ErrorContains(t, err, "module 'm1' did not reach one of [FINISHED]") +} diff --git a/internal/suites/conformance_browser.go b/internal/suites/conformance_browser.go new file mode 100644 index 000000000..48c4dce4a --- /dev/null +++ b/internal/suites/conformance_browser.go @@ -0,0 +1,556 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "strings" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" + log "github.com/sirupsen/logrus" +) + +// ConformancePageState is what the browser is looking at part way through a conformance module's authorization flow. +type ConformancePageState int + +const ( + // ConformancePageUnknown is a page the driver has no action for. + ConformancePageUnknown ConformancePageState = iota + + // ConformancePageFirstFactor is Authelia's first factor sign in form. + ConformancePageFirstFactor + + // ConformancePageConsent is Authelia's OpenID Connect 1.0 consent decision stage. + ConformancePageConsent + + // ConformancePageAutheliaError is Authelia reporting an error rather than presenting a stage. + ConformancePageAutheliaError + + // ConformancePageCallback is the conformance suite's own callback, which ends a leg of the flow. + ConformancePageCallback +) + +const ( + conformanceSelectorFirstFactor = "#first-factor-stage" + conformanceSelectorConsent = "#openid-consent-decision-stage" + conformanceSelectorConsentAccept = "#openid-consent-accept" + conformanceSelectorConsentReauthentication = "#openid-consent-prompt-login" + conformanceSelectorConsentPassword = "#openid-consent-prompt-login #password-textfield" + conformanceSelectorConsentAuthenticate = "#openid-consent-authenticate" + conformanceSelectorUsername = "#username-textfield" + conformanceSelectorPassword = "#password-textfield" + conformanceSelectorSignIn = "#sign-in-button" + conformanceCallbackPathPrefix = "/test/a/" + conformanceSelectorAutheliaError = `.notification[data-type="error"]` + conformanceSelectorCompletionError = `[data-testid="openid-completion-outcome"][data-outcome="error"]` + conformanceElementTimeout = time.Second * 2 + conformanceDriveInterval = time.Millisecond * 250 + conformanceSettleTimeout = time.Second * 30 + conformanceLegPatience = time.Second * 30 + conformancePageTimeout = time.Second * 30 + conformanceSignInAttempts = 3 + conformanceConsentAttempts = 3 + conformanceScreenshotLimit = 500 * 1024 + conformanceCoverageBinding = "__autheliaConformanceCoverage" + conformanceCoverageScript = `addEventListener('beforeunload', () => { if (window.__coverage__) window.` + conformanceCoverageBinding + `(JSON.stringify(window.__coverage__)); });` +) + +// ConformanceClassifyPage decides what the browser is looking at. It takes the observations rather than the page so +// that the decision is testable without a browser. The callback wins over every stage selector, because a stage left +// in the DOM of the document being navigated away from would otherwise be read as the current page. +func ConformanceClassifyPage(uri string, hasFirstFactor, hasConsent, hasError bool) ConformancePageState { + switch { + case conformanceIsCallback(uri): + return ConformancePageCallback + case hasConsent: + return ConformancePageConsent + case hasFirstFactor: + return ConformancePageFirstFactor + case hasError: + return ConformancePageAutheliaError + default: + return ConformancePageUnknown + } +} + +func conformanceIsCallback(uri string) bool { + u, err := url.Parse(uri) + if err != nil { + return false + } + + return strings.HasPrefix(u.Path, conformanceCallbackPathPrefix) +} + +// ConformanceLeg is what the driver saw over the course of one browser leg of a module's authorization flow. It is +// recorded as the leg is driven rather than probed afterwards: [ConformanceBrowser.Drive] only returns once the flow +// has left Authelia for the conformance suite's callback, so anything asked of the live DOM at that point is a +// question about the callback page and not about what Authelia did. Being a plain value also makes the module +// overrides which consume it testable without a browser. +type ConformanceLeg struct { + // Index is the zero based position of this leg among the module's browser legs. Modules such as + // oidcc-prompt-login perform two authorization round trips, and only the second one carries the parameter under + // test, so an expectation about re-authentication belongs to Index 1 rather than to every leg. + Index int + + // FirstFactor is whether Authelia presented its first factor sign in form at any point during the leg. + FirstFactor bool + + // Consent is whether Authelia presented its OpenID Connect 1.0 consent decision stage at any point during the leg. + Consent bool + + // Reauthentication is whether Authelia asked for the password again during the leg. This is the signal a + // prompt=login or max_age expectation turns on, not FirstFactor: Authelia asks on the consent decision form + // rather than by returning to the sign in page. + Reauthentication bool + + // AutheliaError is whether the leg ended on Authelia reporting an error rather than reaching the callback. + AutheliaError bool + + // ErrorURL is the page the leg ended on when AutheliaError is set. Authelia's completion view carries the error in + // its query, so this is what describes the error to a report. + ErrorURL string + + // LoginScreenshot is the last page during the leg which asked for the password, captured with it entered and + // before it was submitted: the consent form's re-authentication step, or the sign in form. + LoginScreenshot string + + // ErrorScreenshot is the Authelia error page the leg ended on. + ErrorScreenshot string +} + +// ConformanceBrowser is one conformance plan's isolated browser context. Every method returns an error rather than +// failing a test, because the driver runs on a goroutine which is not the test's. +type ConformanceBrowser struct { + session *RodSession + incognito *rod.Browser + page *rod.Page + username string + password string + + trace *ConformanceTrace + stopCoverage func() +} + +// NewConformanceBrowser returns a ConformanceBrowser with its own cookie jar. +func NewConformanceBrowser(session *RodSession) (browser *ConformanceBrowser, err error) { + incognito, err := session.WebDriver.Incognito() + if err != nil { + return nil, fmt.Errorf("error creating an isolated browser context: %w", err) + } + + page, err := incognito.Page(proto.TargetCreateTarget{}) + if err != nil { + _ = incognito.Close() + + return nil, fmt.Errorf("error creating a tab: %w", err) + } + + browser = &ConformanceBrowser{ + session: session, + incognito: incognito, + page: page, + username: testUsername, + password: testPassword, + } + + if err = browser.collectCoverage(); err != nil { + _ = incognito.Close() + + return nil, fmt.Errorf("error arranging for frontend coverage to be collected: %w", err) + } + + return browser, nil +} + +func (b *ConformanceBrowser) collectCoverage() (err error) { + if err = (proto.RuntimeAddBinding{Name: conformanceCoverageBinding}).Call(b.page); err != nil { + return err + } + + if _, err = b.page.EvalOnNewDocument(conformanceCoverageScript); err != nil { + return err + } + + page, cancel := b.page.WithCancel() + + wait := page.EachEvent(func(e *proto.RuntimeBindingCalled) { + if e.Name != conformanceCoverageBinding { + return + } + + if err := writeCoverage(coverageDir, e.Payload); err != nil { + log.Errorf("Error writing coverage: %v", err) + } + }) + + done := make(chan struct{}) + + go func() { + wait() + close(done) + }() + + b.stopCoverage = func() { + cancel() + <-done + } + + return nil +} + +// Close disposes of the browser context. The document the browser ends on is never left, so its coverage is collected +// directly, and any coverage still being written is finished before the context goes. +func (b *ConformanceBrowser) Close() { + if b.incognito == nil { + return + } + + b.session.collectCoverage(b.page.Timeout(conformancePageTimeout)) + + if b.stopCoverage != nil { + b.stopCoverage() + } + + _ = b.incognito.Close() + + b.incognito = nil +} + +// ClearCookies discards every cookie in this context, for the modules whose instructions are to remove any cookies +// received from the provider before proceeding. +func (b *ConformanceBrowser) ClearCookies() error { + return proto.NetworkClearBrowserCookies{}.Call(b.page.Timeout(conformancePageTimeout)) +} + +func (b *ConformanceBrowser) has(selector string) bool { + has, _, err := b.page.Timeout(conformanceElementTimeout).Has(selector) + + return err == nil && has +} + +func (b *ConformanceBrowser) hasAutheliaError() bool { + return b.has(conformanceSelectorAutheliaError) || b.has(conformanceSelectorCompletionError) +} + +func (b *ConformanceBrowser) url() string { + info, err := b.page.Timeout(conformancePageTimeout).Info() + if err != nil { + return "" + } + + return info.URL +} + +func (b *ConformanceBrowser) title() string { + info, err := b.page.Timeout(conformancePageTimeout).Info() + if err != nil { + return "" + } + + return info.Title +} + +func (b *ConformanceBrowser) settle(ctx context.Context, selector, startURL string) error { + deadline := time.Now().Add(conformanceSettleTimeout) + + for time.Now().Before(deadline) { + if ctx.Err() != nil { + return ctx.Err() + } + + if !b.has(selector) || b.url() != startURL { + return nil + } + + time.Sleep(conformanceDriveInterval) + } + + return fmt.Errorf("the form at '%s' was submitted but '%s' was still on screen %s later", startURL, selector, conformanceSettleTimeout) +} + +// Drive navigates to uri and works the flow through to the conformance suite's callback. It is deliberately reactive: +// it acts on whatever page appears rather than on a script per module, so that a module added by a future conformance +// suite release runs instead of failing. +func (b *ConformanceBrowser) Drive(ctx context.Context, index int, uri string) (leg ConformanceLeg, err error) { + leg = ConformanceLeg{Index: index} + + loaded := b.page.Timeout(conformancePageTimeout).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded) + + if err = b.session.doNavigate(b.page.Timeout(conformancePageTimeout), uri); err != nil { + return leg, fmt.Errorf("error navigating to '%s': %w", uri, err) + } + + loaded() + + attempts, consents := 0, 0 + + progress := newConformanceProgress(conformanceLegPatience, time.Now()) + + for { + if ctx.Err() != nil { + return leg, fmt.Errorf("the flow beginning at '%s' did not reach the conformance callback, the last page was '%s': %w", uri, b.url(), ctx.Err()) + } + + pageURL := b.url() + + switch ConformanceClassifyPage(pageURL, b.has(conformanceSelectorFirstFactor), b.has(conformanceSelectorConsent), b.hasAutheliaError()) { + case ConformancePageCallback: + return leg, nil + case ConformancePageFirstFactor: + leg.FirstFactor = true + + if attempts++; attempts > conformanceSignInAttempts { + return leg, fmt.Errorf("the sign in form at '%s' was still present after %d attempts", pageURL, conformanceSignInAttempts) + } + + var screenshot string + + if screenshot, err = b.submitSignIn(ctx, pageURL, attempts); err != nil { + return leg, err + } + + leg.LoginScreenshot = conformanceLatest(leg.LoginScreenshot, screenshot) + + progress.observe(time.Now(), pageURL, true) + case ConformancePageConsent: + leg.Consent = true + + if consents++; consents > conformanceConsentAttempts { + return leg, fmt.Errorf("the consent decision form at '%s' was still present after %d attempts", pageURL, conformanceConsentAttempts) + } + + var ( + reauthentication bool + screenshot string + ) + + if reauthentication, screenshot, err = b.submitConsent(ctx, pageURL, consents); err != nil { + return leg, err + } + + leg.Reauthentication = leg.Reauthentication || reauthentication + leg.LoginScreenshot = conformanceLatest(leg.LoginScreenshot, screenshot) + + progress.observe(time.Now(), pageURL, true) + case ConformancePageAutheliaError: + leg.AutheliaError, leg.ErrorURL, leg.ErrorScreenshot = true, pageURL, b.screenshot() + + return leg, nil + case ConformancePageUnknown: + if progress.observe(time.Now(), pageURL, false) { + return leg, fmt.Errorf("the flow beginning at '%s' stopped at '%s' (%q), a page the driver has no action for, and it did not change for %s", + uri, pageURL, b.title(), conformanceLegPatience) + } + + time.Sleep(conformanceDriveInterval) + } + } +} + +func (b *ConformanceBrowser) signIn() (screenshot string, err error) { + if err = b.input(conformanceSelectorUsername, b.username); err != nil { + return "", err + } + + if err = b.input(conformanceSelectorPassword, b.password); err != nil { + return "", err + } + + screenshot = b.screenshot() + + return screenshot, b.click(conformanceSelectorSignIn) +} + +func (b *ConformanceBrowser) screenshot() string { + uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) { + req := &proto.PageCaptureScreenshot{Format: format} + + if quality != 0 { + req.Quality = &quality + } + + return b.page.Timeout(conformancePageTimeout).Screenshot(false, req) + }) + if err != nil { + log.Warnf("Conformance driver could not capture a screenshot of '%s': %v", b.url(), err) + + return "" + } + + return uri +} + +func conformanceScreenshotDataURI(capture func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error)) (uri string, err error) { + for _, attempt := range []struct { + format proto.PageCaptureScreenshotFormat + quality int + mime string + }{ + {proto.PageCaptureScreenshotFormatPng, 0, "image/png"}, + {proto.PageCaptureScreenshotFormatJpeg, 80, "image/jpeg"}, + {proto.PageCaptureScreenshotFormatJpeg, 50, "image/jpeg"}, + } { + var data []byte + + if data, err = capture(attempt.format, attempt.quality); err != nil { + return "", err + } + + if len(data) <= conformanceScreenshotLimit { + return "data:" + attempt.mime + ";base64," + base64.StdEncoding.EncodeToString(data), nil + } + } + + return "", fmt.Errorf("the screenshot does not fit the conformance suite's %dKB upload limit in any format", conformanceScreenshotLimit/1024) +} + +func conformanceLatest(current, next string) string { + if next == "" { + return current + } + + return next +} + +func (b *ConformanceBrowser) input(selector, value string) (err error) { + element, err := b.page.Timeout(elementLocateTimeout).Element(selector) + if err != nil { + return fmt.Errorf("error locating '%s': %w", selector, err) + } + + if err = element.SelectAllText(); err != nil { + return fmt.Errorf("error selecting the text of '%s': %w", selector, err) + } + + if err = element.Input(value); err != nil { + return fmt.Errorf("error typing into '%s': %w", selector, err) + } + + return nil +} + +func (b *ConformanceBrowser) click(selector string) (err error) { + element, err := b.page.Timeout(elementLocateTimeout).Element(selector) + if err != nil { + return fmt.Errorf("error locating '%s': %w", selector, err) + } + + if err = element.Click(proto.InputMouseButtonLeft, 1); err != nil { + return fmt.Errorf("error clicking '%s': %w", selector, err) + } + + return nil +} + +type conformanceProgress struct { + patience time.Duration + url string + deadline time.Time +} + +func newConformanceProgress(patience time.Duration, now time.Time) *conformanceProgress { + return &conformanceProgress{patience: patience, deadline: now.Add(patience)} +} + +func (p *conformanceProgress) observe(now time.Time, url string, recognized bool) (stalled bool) { + if url != p.url { + p.url, p.deadline = url, now.Add(p.patience) + + return false + } + + if recognized { + p.deadline = now.Add(p.patience) + + return false + } + + return now.After(p.deadline) +} + +func (b *ConformanceBrowser) submitSignIn(ctx context.Context, pageURL string, attempt int) (screenshot string, err error) { + b.trace.Logf("Signing in at '%s' (attempt %d)", pageURL, attempt) + + if screenshot, err = b.signIn(); err != nil { + return "", fmt.Errorf("error signing in at '%s': %w", pageURL, err) + } + + if err = b.settle(ctx, conformanceSelectorFirstFactor, pageURL); err != nil { + return "", fmt.Errorf("error signing in at '%s': %w", pageURL, err) + } + + return screenshot, nil +} + +func (b *ConformanceBrowser) submitConsent(ctx context.Context, pageURL string, attempt int) (reauthentication bool, screenshot string, err error) { + b.trace.Logf("Accepting the consent decision form at '%s' (attempt %d)", pageURL, attempt) + + if err = b.click(conformanceSelectorConsentAccept); err != nil { + return false, "", fmt.Errorf("error accepting consent at '%s': %w", pageURL, err) + } + + if reauthentication, err = b.awaitConsentAccepted(ctx, pageURL); err != nil { + return false, "", err + } + + if !reauthentication { + return false, "", nil + } + + b.trace.Logf("Answering the consent form's re-authentication step at '%s'", pageURL) + + if err = b.input(conformanceSelectorConsentPassword, b.password); err != nil { + return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err) + } + + screenshot = b.screenshot() + + if err = b.click(conformanceSelectorConsentAuthenticate); err != nil { + return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err) + } + + if err = b.settle(ctx, conformanceSelectorConsentReauthentication, pageURL); err != nil { + return true, "", fmt.Errorf("error answering the re-authentication step at '%s': %w", pageURL, err) + } + + return true, screenshot, nil +} + +func (b *ConformanceBrowser) awaitConsentAccepted(ctx context.Context, pageURL string) (reauthentication bool, err error) { + deadline := time.Now().Add(conformanceSettleTimeout) + + for time.Now().Before(deadline) { + if ctx.Err() != nil { + return false, ctx.Err() + } + + reauthentication, done := ConformanceConsentProgress(b.has(conformanceSelectorConsentReauthentication), b.has(conformanceSelectorConsent), b.url() != pageURL) + if done { + return reauthentication, nil + } + + time.Sleep(conformanceDriveInterval) + } + + return false, fmt.Errorf("the consent decision form at '%s' was accepted but neither left nor asked for the password %s later", pageURL, conformanceSettleTimeout) +} + +// ConformanceConsentProgress reports what accepting the consent decision achieved. +func ConformanceConsentProgress(hasReauthentication, hasStage, urlChanged bool) (reauthentication, done bool) { + switch { + case hasReauthentication: + return true, true + case !hasStage || urlChanged: + return false, true + default: + return false, false + } +} diff --git a/internal/suites/conformance_browser_driver_test.go b/internal/suites/conformance_browser_driver_test.go new file mode 100644 index 000000000..3e0bcfc8d --- /dev/null +++ b/internal/suites/conformance_browser_driver_test.go @@ -0,0 +1,372 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceBrowser_Drive(t *testing.T) { + driver := newConformanceTestBrowser(t) + portal := conformanceFakePortal(t) + + drive := func(t *testing.T, index int, path string) ConformanceLeg { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + leg, err := driver.Drive(ctx, index, portal.URL+path) + require.NoError(t, err) + + return leg + } + + t.Run("ShouldSignInThenAnswerTheConsentFormsReauthentication", func(t *testing.T) { + leg := drive(t, 1, "/authorize-reauthenticate") + + assert.Equal(t, 1, leg.Index) + assert.True(t, leg.FirstFactor) + assert.True(t, leg.Consent) + assert.True(t, leg.Reauthentication) + assert.False(t, leg.AutheliaError) + assert.True(t, strings.HasPrefix(leg.LoginScreenshot, "data:image/png;base64,"), "the password step is captured") + assert.Empty(t, leg.ErrorScreenshot) + }) + + t.Run("ShouldAcceptConsentWhichAsksForNoPassword", func(t *testing.T) { + leg := drive(t, 0, "/consent") + + assert.True(t, leg.Consent) + assert.False(t, leg.Reauthentication) + assert.False(t, leg.FirstFactor) + assert.Empty(t, leg.LoginScreenshot) + }) + + t.Run("ShouldDriveALegWhichStartsOnTheLastLegsCallback", func(t *testing.T) { + drive(t, 0, "/consent") + + leg := drive(t, 1, "/error") + + assert.True(t, leg.AutheliaError) + }) + + t.Run("ShouldEndTheLegOnAnAutheliaErrorPage", func(t *testing.T) { + leg := drive(t, 0, "/error") + + assert.True(t, leg.AutheliaError) + assert.Equal(t, portal.URL+"/error", leg.ErrorURL) + assert.True(t, strings.HasPrefix(leg.ErrorScreenshot, "data:image/png;base64,"), "the error page is captured") + }) + + t.Run("ShouldHandCoverageOverAsThePageIsLeft", func(t *testing.T) { + before, _ := os.ReadDir(coverageDir) + + drive(t, 0, "/coverage") + + assert.Eventually(t, func() bool { + after, _ := os.ReadDir(coverageDir) + + return len(after) > len(before) + }, time.Second*5, time.Millisecond*50) + }) + + t.Run("ShouldClearTheContextsCookies", func(t *testing.T) { + drive(t, 0, "/authorize") + + cookies, err := driver.page.Cookies(nil) + require.NoError(t, err) + require.NotEmpty(t, cookies) + + require.NoError(t, driver.ClearCookies()) + + cookies, err = driver.page.Cookies(nil) + require.NoError(t, err) + assert.Empty(t, cookies) + }) + + t.Run("ShouldReportANavigationWhichFailed", func(t *testing.T) { + _, err := driver.Drive(context.Background(), 0, "http://127.0.0.1:1/") + + assert.ErrorContains(t, err, "error navigating to 'http://127.0.0.1:1/'") + }) + + t.Run("ShouldStopWhenTheContextHasExpired", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := driver.Drive(ctx, 0, portal.URL+"/error") + + assert.ErrorContains(t, err, "did not reach the conformance callback") + }) +} + +func TestConformanceRunner_VisitURLsDrivesEachLeg(t *testing.T) { + driver := newConformanceTestBrowser(t) + portal := conformanceFakePortal(t) + + var ( + mu sync.Mutex + visited []string + ) + + visit := func(status int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + + visited = append(visited, r.URL.Query().Get("url")) + mu.Unlock() + + w.WriteHeader(status) + } + } + + t.Run("ShouldVisitALegOnceItIsDriven", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{"/api/runner/browser/m1/visit": visit(http.StatusNoContent)}) + + runner := NewConformanceRunner(client, driver, "plan1", nil) + runner.trace = NewConformanceTrace(true) + + legs := &conformanceLegs{visited: map[string]bool{}} + + progressed, err := runner.visitURLs(context.Background(), "m1", ConformanceOverride{Screenshot: ConformanceScreenshotErrorPage}, legs, []string{portal.URL + "/error"}) + + require.NoError(t, err) + assert.True(t, progressed) + assert.Equal(t, portal.URL+"/error", legs.errorURL) + assert.True(t, strings.HasPrefix(legs.screenshot, "data:image/png;base64,")) + assert.Equal(t, []string{portal.URL + "/error"}, visited) + assert.Len(t, runner.trace.Lines(), 2, "the leg is traced as it starts and as it ends") + }) + + t.Run("ShouldNotVisitALegWhoseAssertionFailed", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{}) + + legs := &conformanceLegs{visited: map[string]bool{}} + + _, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{Assert: conformanceAssertErrorPage}, legs, []string{portal.URL + "/consent"}) + + assert.EqualError(t, err, "expected Authelia to show an error page rather than redirect to the client but it did not") + }) + + t.Run("ShouldReportAVisitWhichFailed", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{"/api/runner/browser/m1/visit": visit(http.StatusInternalServerError)}) + + legs := &conformanceLegs{visited: map[string]bool{}} + + _, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{}, legs, []string{portal.URL + "/consent-reauthenticate"}) + + assert.ErrorContains(t, err, "expected status 204 but got 500") + }) + + t.Run("ShouldReportALegWhichCouldNotBeDriven", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{}) + + legs := &conformanceLegs{visited: map[string]bool{}} + + _, err := NewConformanceRunner(client, driver, "plan1", nil).visitURLs(context.Background(), "m1", ConformanceOverride{}, legs, []string{"http://127.0.0.1:1/"}) + + assert.ErrorContains(t, err, "error navigating") + }) +} + +func TestConformanceBrowser_AfterClose(t *testing.T) { + driver := newConformanceTestBrowser(t) + + driver.Close() + + assert.Empty(t, driver.url()) + assert.Empty(t, driver.title()) + assert.False(t, driver.has(conformanceSelectorFirstFactor)) + assert.Empty(t, driver.screenshot()) + + assert.NotPanics(t, driver.Close, "closing twice is harmless") +} + +func TestConformanceRunner_WithABrowser(t *testing.T) { + driver := newConformanceTestBrowser(t) + + t.Run("ShouldClearCookiesForAModuleWhichAsksForIt", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/runner": conformanceReply(http.StatusCreated, `{"id":"m1"}`), + "/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`), + "/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`), + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, driver, "plan1", []ConformancePlanModule{{TestModule: "oidcc-display-page"}}).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + require.NoError(t, outcomes[0].Err) + assert.Equal(t, "PASSED", outcomes[0].Result) + }) + + t.Run("ShouldReportALegWhichCouldNotBeDriven", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/runner/browser/m1": conformanceReply(http.StatusOK, `{"urls":["http://127.0.0.1:1/"]}`), + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + err := NewConformanceRunner(client, driver, "plan1", nil).interact(ctx, "m1", ConformanceOverride{}) + + assert.ErrorContains(t, err, "error navigating to 'http://127.0.0.1:1/'") + }) +} + +func conformanceFakePortal(t *testing.T) *httptest.Server { + t.Helper() + + const callback = "/test/a/alias/callback?code=abc" + + page := func(w http.ResponseWriter, body string) { + w.Header().Set("Content-Type", "text/html") + _, _ = fmt.Fprintf(w, "Login - Authelia%s", body) + } + + mux := http.NewServeMux() + + for _, variant := range []struct { + suffix string + reauthentication bool + }{ + {"", false}, + {"-reauthenticate", true}, + } { + mux.HandleFunc("/authorize"+variant.suffix, func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/login"+variant.suffix, http.StatusFound) + }) + + mux.HandleFunc("/login"+variant.suffix, func(w http.ResponseWriter, _ *http.Request) { + page(w, fmt.Sprintf(`
+ +
+`, testUsername, testPassword, "/consent"+variant.suffix)) + }) + + mux.HandleFunc("/consent"+variant.suffix, func(w http.ResponseWriter, _ *http.Request) { + page(w, fmt.Sprintf(` +`, variant.reauthentication, callback, testPassword, callback)) + }) + } + + mux.HandleFunc("/error", func(w http.ResponseWriter, _ *http.Request) { + page(w, `
An error occurred processing the request
`) + }) + + mux.HandleFunc("/coverage", func(w http.ResponseWriter, _ *http.Request) { + page(w, fmt.Sprintf(``, callback)) + }) + + mux.HandleFunc("/test/a/alias/callback", func(w http.ResponseWriter, _ *http.Request) { + page(w, "callback") + }) + + server := httptest.NewServer(mux) + + t.Cleanup(server.Close) + + return server +} + +func newConformanceTestBrowser(t *testing.T) *ConformanceBrowser { + t.Helper() + + if testing.Short() { + t.Skip("skipping browser test in short mode") + } + + preserveCoverageDir(t) + + path, err := GetBrowserPath() + require.NoError(t, err) + + l := launcher.New().Bin(path).Headless(true) + t.Cleanup(l.Cleanup) + + browser := rod.New().ControlURL(l.MustLaunch()).MustConnect() + + t.Cleanup(func() { _ = browser.Close() }) + + driver, err := NewConformanceBrowser(&RodSession{WebDriver: browser}) + require.NoError(t, err) + + t.Cleanup(driver.Close) + + return driver +} + +func preserveCoverageDir(t *testing.T) { + t.Helper() + + existed := map[string]bool{} + + entries, err := os.ReadDir(coverageDir) + dirExisted := err == nil + + for _, entry := range entries { + existed[entry.Name()] = true + } + + t.Cleanup(func() { + entries, _ := os.ReadDir(coverageDir) + + for _, entry := range entries { + if !existed[entry.Name()] { + _ = os.Remove(filepath.Join(coverageDir, entry.Name())) + } + } + + if !dirExisted { + _ = os.Remove(coverageDir) + } + }) +} diff --git a/internal/suites/conformance_browser_test.go b/internal/suites/conformance_browser_test.go new file mode 100644 index 000000000..475850cf8 --- /dev/null +++ b/internal/suites/conformance_browser_test.go @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "encoding/base64" + "errors" + "fmt" + "testing" + + "github.com/go-rod/rod/lib/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceClassifyPage(t *testing.T) { + testCases := []struct { + name string + url string + firstFactor, consent, autheliaError bool + expected ConformancePageState + }{ + { + "ShouldDetectTheCallbackByPath", + "https://conformance.example.com:8443/test/a/conformance-basic-authelia1a2b3c4d/callback?code=abc", + false, false, false, + ConformancePageCallback, + }, + { + "ShouldPreferTheCallbackOverAnyStageStillInTheDOM", + "https://conformance.example.com:8443/test/a/alias/callback", + true, false, false, + ConformancePageCallback, + }, + { + "ShouldNotMistakeAnAuthorizationRequestCarryingTheCallbackForTheCallback", + "https://login.example.com:8080/api/oidc/authorization?client_id=abc&redirect_uri=https://conformance.example.com:8443/test/a/alias/callback", + false, false, false, + ConformancePageUnknown, + }, + { + "ShouldDetectTheFirstFactorPage", + "https://login.example.com:8080/", + true, false, false, + ConformancePageFirstFactor, + }, + { + "ShouldDetectTheConsentPage", + "https://login.example.com:8080/consent/openid/decision", + false, true, false, + ConformancePageConsent, + }, + { + "ShouldDetectAnAutheliaError", + "https://login.example.com:8080/", + false, false, true, + ConformancePageAutheliaError, + }, + { + "ShouldPreferTheFirstFactorPageOverACoexistingErrorToast", + "https://login.example.com:8080/", + true, false, true, + ConformancePageFirstFactor, + }, + { + "ShouldPreferTheConsentPageOverACoexistingErrorToast", + "https://login.example.com:8080/consent/openid/decision", + false, true, true, + ConformancePageConsent, + }, + { + "ShouldReportUnknownWhenNothingMatches", + "https://login.example.com:8080/", + false, false, false, + ConformancePageUnknown, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, ConformanceClassifyPage(tc.url, tc.firstFactor, tc.consent, tc.autheliaError)) + }) + } +} + +func TestConformanceConsentProgress(t *testing.T) { + t.Run("ShouldTreatTheRevealedPasswordFieldAsTheReauthenticationStep", func(t *testing.T) { + reauthentication, done := ConformanceConsentProgress(true, true, false) + assert.True(t, reauthentication) + assert.True(t, done) + }) + + t.Run("ShouldTreatTheStageClearingAsTheFlowLeaving", func(t *testing.T) { + reauthentication, done := ConformanceConsentProgress(false, false, false) + assert.False(t, reauthentication) + assert.True(t, done) + }) + + t.Run("ShouldTreatAChangedURLAsTheFlowLeaving", func(t *testing.T) { + reauthentication, done := ConformanceConsentProgress(false, true, true) + assert.False(t, reauthentication) + assert.True(t, done) + }) + + t.Run("ShouldKeepWaitingWhileTheFormIsStillOnItsDecisionStep", func(t *testing.T) { + reauthentication, done := ConformanceConsentProgress(false, true, false) + assert.False(t, reauthentication) + assert.False(t, done) + }) + + t.Run("ShouldPreferTheReauthenticationStepOverTheFormLeaving", func(t *testing.T) { + reauthentication, done := ConformanceConsentProgress(true, false, true) + assert.True(t, reauthentication) + assert.True(t, done) + }) +} + +func TestConformanceScreenshotDataURI(t *testing.T) { + small, large := []byte("small"), make([]byte, conformanceScreenshotLimit+1) + + t.Run("ShouldPreferAPNGWhichFits", func(t *testing.T) { + uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) { + assert.Equal(t, proto.PageCaptureScreenshotFormatPng, format) + + return small, nil + }) + + require.NoError(t, err) + assert.Equal(t, "data:image/png;base64,"+base64.StdEncoding.EncodeToString(small), uri) + }) + + t.Run("ShouldFallBackToAJPEGWhenThePNGIsTooLarge", func(t *testing.T) { + var asked []string + + uri, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) { + asked = append(asked, fmt.Sprintf("%s:%d", format, quality)) + + if format == proto.PageCaptureScreenshotFormatPng { + return large, nil + } + + return small, nil + }) + + require.NoError(t, err) + assert.Equal(t, "data:image/jpeg;base64,"+base64.StdEncoding.EncodeToString(small), uri) + assert.Equal(t, []string{"png:0", "jpeg:80"}, asked) + }) + + t.Run("ShouldFailWhenNothingFits", func(t *testing.T) { + _, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) { + return large, nil + }) + + assert.ErrorContains(t, err, "500KB") + }) + + t.Run("ShouldFailWhenTheCaptureFails", func(t *testing.T) { + _, err := conformanceScreenshotDataURI(func(format proto.PageCaptureScreenshotFormat, quality int) ([]byte, error) { + return nil, errors.New("target closed") + }) + + assert.ErrorContains(t, err, "target closed") + }) +} + +func TestConformanceIsCallbackRejectsAnUnparsableURL(t *testing.T) { + assert.False(t, conformanceIsCallback("%zz/test/a/alias/callback")) +} diff --git a/internal/suites/conformance_compose_test.go b/internal/suites/conformance_compose_test.go new file mode 100644 index 000000000..4b99941bd --- /dev/null +++ b/internal/suites/conformance_compose_test.go @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "net/url" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v4" +) + +func TestOIDCConformanceComposeSuppliesOIDCLoginPlaceholders(t *testing.T) { + data, err := os.ReadFile("OIDCConformance/compose.yml") + require.NoError(t, err) + + compose := struct { + Services map[string]struct { + Environment map[string]string `yaml:"environment"` + } `yaml:"services"` + }{} + + require.NoError(t, yaml.Unmarshal(data, &compose)) + + server, ok := compose.Services["conformance-server"] + require.True(t, ok, "the compose file must define a conformance-server service") + + for _, name := range []string{"OIDC_GOOGLE_CLIENTID", "OIDC_GOOGLE_SECRET", "OIDC_GITLAB_CLIENTID", "OIDC_GITLAB_SECRET"} { + value, ok := server.Environment[name] + + assert.Truef(t, ok, "%s must be set: the image's entrypoint interpolates it unconditionally, so leaving it unset overrides the suite's own default with an empty system property", name) + assert.NotEmptyf(t, value, "%s must not be empty: Spring rejects an empty client id while building OAuth2ClientProperties, which fails the application context before devmode is consulted", name) + } +} + +func TestOIDCConformanceComposeMongoDB(t *testing.T) { + type service struct { + Environment map[string]string `yaml:"environment"` + DependsOn []string `yaml:"depends_on"` + } + + read := func(t *testing.T, path string) map[string]service { + t.Helper() + + data, err := os.ReadFile(path) + require.NoError(t, err) + + compose := struct { + Services map[string]service `yaml:"services"` + }{} + + require.NoError(t, yaml.Unmarshal(data, &compose)) + + return compose.Services + } + + t.Run("ShouldLeaveTheDatabaseToItsOwnComposeFile", func(t *testing.T) { + services := read(t, "OIDCConformance/compose.yml") + + assert.NotContains(t, services, "conformance-mongodb") + assert.NotContains(t, services["conformance-server"].DependsOn, "conformance-mongodb") + assert.Equal(t, "${"+oidcConformanceMongoDBHostEnv+":-conformance-mongodb}", services["conformance-server"].Environment["MONGODB_HOST"], + "the server uses the database the suite was pointed at, and its own otherwise") + }) + + t.Run("ShouldRunTheDatabaseFromItsOwnComposeFile", func(t *testing.T) { + services := read(t, oidcConformanceMongoDBComposeFile) + + assert.Contains(t, services, "conformance-mongodb") + assert.Contains(t, services["conformance-server"].DependsOn, "conformance-mongodb") + }) +} + +func TestOIDCConformanceComposeFiles(t *testing.T) { + assert.Contains(t, oidcConformanceComposeFiles(""), "internal/suites/"+oidcConformanceMongoDBComposeFile) + assert.Contains(t, oidcConformanceLogServices(""), "conformance-mongodb") + + assert.NotContains(t, oidcConformanceComposeFiles("mongodb.example.com"), "internal/suites/"+oidcConformanceMongoDBComposeFile, + "a database the suite is pointed at replaces its own") + assert.NotContains(t, oidcConformanceLogServices("mongodb.example.com"), "conformance-mongodb", "there is no container to print the logs of") + + for _, host := range []string{"", "mongodb.example.com"} { + assert.Contains(t, oidcConformanceComposeFiles(host), "internal/suites/OIDCConformance/compose.yml") + assert.Contains(t, oidcConformanceLogServices(host), "conformance-server") + } +} + +func TestOIDCConformanceServedByThePortal(t *testing.T) { + base, err := url.ParseRequestURI(oidcConformanceBaseURL) + require.NoError(t, err) + + conf, err := os.ReadFile("example/compose/nginx/portal/nginx.conf") + require.NoError(t, err) + + nginx := string(conf) + + t.Run("ShouldListenOnThePortTheBaseURLUses", func(t *testing.T) { + assert.Containsf(t, nginx, "listen "+base.Port()+" ssl;", + "the portal must terminate TLS on %s, which is the port %s resolves to", base.Port(), oidcConformanceBaseURL) + }) + + t.Run("ShouldServeTheConformanceHost", func(t *testing.T) { + assert.Equal(t, 3, strings.Count(nginx, `server_name ~^conformance\.example([0-9])*\.com$;`), + "one server block per listener the published nginx image exposed") + assert.Equal(t, 3, strings.Count(nginx, "set $upstream_endpoint http://conformance-server:8080;"), + "the portal proxies straight to the server now that the published nginx image is gone") + }) + + t.Run("ShouldExposeTheMutualTLSAndFAPIListeners", func(t *testing.T) { + // 8444 and 8445 carry no traffic yet, but a profile that reaches for them should find them rather than + // fail part way through a plan, and they are the reason the published image existed at all. + assert.Contains(t, nginx, "listen 8444 ssl;") + assert.Contains(t, nginx, "listen 8445 ssl;") + + assert.Equal(t, 2, strings.Count(nginx, "ssl_verify_client optional_no_ca;"), + "the mutual TLS listeners ask for a client certificate but leave verifying it to the suite") + assert.Contains(t, nginx, "ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384;", + "8445 narrows the ciphers to the FAPI 2.0 Final set") + }) + + t.Run("ShouldReproduceWhatThePublishedImageSet", func(t *testing.T) { + // The suite depends on each of these: large headers carry its request objects, the long timeouts outlast + // requests that block up to 60s on a callback, and it rebuilds its own base URL from the forwarded headers. + for _, directive := range []string{ + "client_header_buffer_size 32k;", + "large_client_header_buffers 4 32k;", + "proxy_read_timeout 120s;", + "proxy_send_timeout 120s;", + "proxy_set_header X-Forwarded-Host $host;", + "proxy_set_header X-Forwarded-Port $server_port;", + "proxy_set_header X-Ssl-Cert $ssl_client_cert;", + } { + assert.Equalf(t, 3, strings.Count(nginx, directive), + "the published nginx image set %s on every listener and the suite relies on it", directive) + } + + // Only the listener that does not ask for a client certificate tells the suite so. + assert.Equal(t, 1, strings.Count(nginx, "proxy_set_header X-Test-Mtls-Called-On-Wrong-Host $mtls_wrong_host;")) + assert.Equal(t, 1, strings.Count(nginx, "ssl_verify_client off;")) + }) + + t.Run("ShouldResolveTheConformanceHostToThePortal", func(t *testing.T) { + entries := map[string]string{} + + for _, entry := range HostEntries() { + entries[entry.Domain] = entry.IP + } + + require.Contains(t, entries, base.Hostname()) + assert.Equal(t, entries["login.example.com"], entries[base.Hostname()], + "the conformance host is served by the portal, so it resolves where every other portal domain does") + }) + + t.Run("ShouldRunThePortal", func(t *testing.T) { + assert.Contains(t, oidcConformanceComposeFiles(""), "internal/suites/example/compose/nginx/portal/compose.yml") + }) +} diff --git a/internal/suites/conformance_diagnostics.go b/internal/suites/conformance_diagnostics.go new file mode 100644 index 000000000..9eb774dbe --- /dev/null +++ b/internal/suites/conformance_diagnostics.go @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +const ( + conformanceDiagnosticEntries = 20 + conformanceDiagnosticValue = 2000 +) + +var ( + conformanceDiagnosticResults = map[string]bool{"FAILURE": true, "WARNING": true, "REVIEW": true} + conformanceDiagnosticSkip = map[string]bool{ + "msg": true, "result": true, "src": true, "requirements": true, + "_id": true, "testId": true, "testOwner": true, "time": true, + "blockId": true, "updatedAt": true, "img": true, "upload": true, + } +) + +// ConformanceDiagnostics renders the graded entries of a module's log for a failure message. Each entry keeps its +// condition's own arguments, which is where a conformance failure actually explains itself: the message says a claim +// was missing, the arguments say which one. +func ConformanceDiagnostics(entries []ConformanceLogEntry) string { + builder := &strings.Builder{} + + reported := 0 + + for _, entry := range entries { + if !conformanceDiagnosticResults[entry.Result] { + continue + } + + if reported == conformanceDiagnosticEntries { + fmt.Fprintf(builder, "\n ... further entries omitted, see the exported plan archive for the complete log") + + break + } + + reported++ + + fmt.Fprintf(builder, "\n [%s] %s", entry.Result, entry.Msg) + + if entry.Src != "" { + fmt.Fprintf(builder, "\n condition: %s", entry.Src) + } + + if requirements := conformanceDiagnosticField(entry.Fields["requirements"]); requirements != "" { + fmt.Fprintf(builder, "\n requirements: %s", requirements) + } + + for _, name := range conformanceDiagnosticNames(entry.Fields) { + fmt.Fprintf(builder, "\n %s: %s", name, conformanceDiagnosticField(entry.Fields[name])) + } + } + + if reported == 0 { + return "" + } + + return builder.String() +} + +func conformanceDiagnosticNames(fields map[string]any) (names []string) { + for name := range fields { + if conformanceDiagnosticSkip[name] { + continue + } + + names = append(names, name) + } + + sort.Strings(names) + + return names +} + +func conformanceDiagnosticField(value any) string { + if value == nil { + return "" + } + + rendered, ok := value.(string) + + if !ok { + data, err := json.Marshal(value) + if err != nil { + return fmt.Sprintf("%v", value) + } + + rendered = string(data) + } + + if len(rendered) > conformanceDiagnosticValue { + return rendered[:conformanceDiagnosticValue] + fmt.Sprintf("... (%d bytes truncated)", len(rendered)-conformanceDiagnosticValue) + } + + return rendered +} + +// ConformanceAcceptedWarningReport renders a module's accepted WARNING for its subtest's log: why it is accepted, +// followed by the graded entries of the module's log, so the warning which was accepted is visible rather than only +// the fact that one was. +func ConformanceAcceptedWarningReport(module, status, reason string, entries []ConformanceLogEntry) string { + report := fmt.Sprintf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s.", module, status, reason) + + if detail := ConformanceDiagnostics(entries); detail != "" { + return report + " Conformance log entries:" + detail + } + + return report + " The module logged no warning entries." +} diff --git a/internal/suites/conformance_diagnostics_test.go b/internal/suites/conformance_diagnostics_test.go new file mode 100644 index 000000000..53df2fd75 --- /dev/null +++ b/internal/suites/conformance_diagnostics_test.go @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceDiagnostics(t *testing.T) { + scopeFailure := `[ + {"result":"SUCCESS","src":"CallUserInfoEndpoint","msg":"Userinfo endpoint response was 200"}, + {"result":"INFO","src":"ExtractUserInfo","msg":"Extracted userinfo"}, + {"result":"WARNING","src":"VerifyScopesReturnedInUserInfoClaims","msg":"'claims' in userinfo doesn't contain all scope items of scope in authorization request","requirements":["OIDCC-5.4"], + "expected_scope_items":["name","given_name","birthdate"],"actual_scope_items":["name"],"missing_items":["given_name","birthdate"], + "_id":"abc","testId":"t1","testOwner":"anon","time":1789021616378} + ]` + + var entries []ConformanceLogEntry + + require.NoError(t, json.Unmarshal([]byte(scopeFailure), &entries)) + + out := ConformanceDiagnostics(entries) + + t.Run("ShouldReportTheGradedEntry", func(t *testing.T) { + assert.Contains(t, out, "[WARNING]") + assert.Contains(t, out, "doesn't contain all scope items") + assert.Contains(t, out, "condition: VerifyScopesReturnedInUserInfoClaims") + assert.Contains(t, out, `requirements: ["OIDCC-5.4"]`) + }) + + t.Run("ShouldReportTheConditionArgumentsThatExplainIt", func(t *testing.T) { + assert.Contains(t, out, `missing_items: ["given_name","birthdate"]`) + assert.Contains(t, out, `expected_scope_items: ["name","given_name","birthdate"]`) + assert.Contains(t, out, `actual_scope_items: ["name"]`) + }) + + t.Run("ShouldOmitUngradedNarrativeAndBookkeeping", func(t *testing.T) { + assert.NotContains(t, out, "Userinfo endpoint response was 200") + assert.NotContains(t, out, "Extracted userinfo") + assert.NotContains(t, out, "testOwner") + assert.NotContains(t, out, "1789021616378") + }) + + t.Run("ShouldOrderFieldsStablyAcrossRuns", func(t *testing.T) { + for i := 0; i < 8; i++ { + var repeat []ConformanceLogEntry + + require.NoError(t, json.Unmarshal([]byte(scopeFailure), &repeat)) + assert.Equal(t, out, ConformanceDiagnostics(repeat)) + } + }) +} + +func TestConformanceDiagnosticsReportsNothingWhenEveryEntryPassed(t *testing.T) { + entries := []ConformanceLogEntry{ + {Result: "SUCCESS", Msg: "fine"}, + {Result: "INFO", Msg: "also fine"}, + } + + assert.Empty(t, ConformanceDiagnostics(entries)) +} + +func TestConformanceDiagnosticsTruncatesAValueLargeEnoughToBuryTheRest(t *testing.T) { + entries := []ConformanceLogEntry{{ + Result: "FAILURE", + Msg: "token endpoint returned an error", + Fields: map[string]any{"result": "FAILURE", "msg": "token endpoint returned an error", "http_response_body": strings.Repeat("x", conformanceDiagnosticValue+500)}, + }} + + out := ConformanceDiagnostics(entries) + + assert.Contains(t, out, "500 bytes truncated") + assert.Less(t, len(out), conformanceDiagnosticValue+400) +} + +func TestConformanceDiagnosticsCapsTheNumberOfEntries(t *testing.T) { + entries := make([]ConformanceLogEntry, 0, conformanceDiagnosticEntries+5) + + for i := 0; i < conformanceDiagnosticEntries+5; i++ { + entries = append(entries, ConformanceLogEntry{Result: "FAILURE", Msg: "failed"}) + } + + out := ConformanceDiagnostics(entries) + + assert.Equal(t, conformanceDiagnosticEntries, strings.Count(out, "[FAILURE]")) + assert.Contains(t, out, "further entries omitted") +} + +func TestConformanceAcceptedWarningReport(t *testing.T) { + entries := []ConformanceLogEntry{ + {Msg: "Checked the id_token", Result: "SUCCESS", Src: "ValidateIdToken"}, + {Msg: "acr value in id_token is not (one of the) requested values", Result: "WARNING", Src: "ValidateIdTokenACRClaimAgainstAcrValuesRequest", + Fields: map[string]any{"requirements": []any{"OIDCC-3.1.2.1", "OIDCC-15.1"}, "id_token_acr": "0"}}, + } + + report := ConformanceAcceptedWarningReport("oidcc-ensure-request-with-acr-values-succeeds", "FINISHED", "the acr is the fallback", entries) + + assert.Contains(t, report, "Module 'oidcc-ensure-request-with-acr-values-succeeds' finished with the accepted result 'WARNING' and the status 'FINISHED': the acr is the fallback.") + assert.Contains(t, report, "[WARNING] acr value in id_token is not (one of the) requested values") + assert.Contains(t, report, "condition: ValidateIdTokenACRClaimAgainstAcrValuesRequest") + assert.Contains(t, report, "id_token_acr: 0") + assert.NotContains(t, report, "Checked the id_token", "only the graded entries are reported") + + assert.Equal(t, "Module 'oidcc-server' finished with the accepted result 'WARNING' and the status 'FINISHED': reason. The module logged no warning entries.", + ConformanceAcceptedWarningReport("oidcc-server", "FINISHED", "reason", nil)) +} + +func TestConformanceDiagnosticsOmitsTheConformanceUIsBookkeeping(t *testing.T) { + detail := ConformanceDiagnostics([]ConformanceLogEntry{ + {Msg: "acr value in id_token is not (one of the) requested values", Result: "WARNING", + Fields: map[string]any{"blockId": "a1c507", "updatedAt": float64(1789035070157), "id_token_acr": "0"}}, + }) + + assert.Contains(t, detail, "id_token_acr: 0") + assert.NotContains(t, detail, "blockId") + assert.NotContains(t, detail, "updatedAt") +} + +func TestConformanceDiagnosticFieldFallsBackForAValueJSONCannotEncode(t *testing.T) { + assert.Equal(t, "(1+2i)", conformanceDiagnosticField(complex(1, 2))) +} diff --git a/internal/suites/conformance_generate_test.go b/internal/suites/conformance_generate_test.go new file mode 100644 index 000000000..bb2e51d12 --- /dev/null +++ b/internal/suites/conformance_generate_test.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOIDCConformanceGenerateReportsAFileItCannotWrite(t *testing.T) { + t.Setenv("SUITE_TMP_PATH", filepath.Join(t.TempDir(), "missing")) + + assert.Error(t, oidcConformanceGenerate()) + assert.Error(t, oidcConformanceWriteJSON(SuiteTmpPath(oidcConformancePlansFile), []OIDCConformancePlanFile{})) +} + +func TestOIDCConformanceReadPlans(t *testing.T) { + t.Run("ShouldReportPlansWhichWereNeverGenerated", func(t *testing.T) { + t.Setenv("SUITE_TMP_PATH", t.TempDir()) + + _, err := oidcConformanceReadPlans() + + assert.ErrorContains(t, err, "error reading the generated conformance plans, was the suite set up?") + }) + + t.Run("ShouldReportPlansWhichCannotBeDecoded", func(t *testing.T) { + dir := t.TempDir() + + t.Setenv("SUITE_TMP_PATH", dir) + + require.NoError(t, os.WriteFile(filepath.Join(dir, oidcConformancePlansFile), []byte("{"), 0600)) + + _, err := oidcConformanceReadPlans() + + assert.Error(t, err) + }) +} diff --git a/internal/suites/conformance_names.go b/internal/suites/conformance_names.go new file mode 100644 index 000000000..339c89bbb --- /dev/null +++ b/internal/suites/conformance_names.go @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "sort" + "strings" + "unicode" +) + +const conformanceModulePrefix = "oidcc-" + +// ConformanceSubtestNames returns the PascalCase subtest name of each module, index aligned with modules. A module +// which a plan lists more than once has its variant appended so the names within a plan stay unique, which is what +// makes go test able to address a single one with -run. +func ConformanceSubtestNames(modules []ConformancePlanModule) (names []string) { + counts := make(map[string]int, len(modules)) + + for _, module := range modules { + counts[module.TestModule]++ + } + + names = make([]string, len(modules)) + + for i, module := range modules { + name := conformancePascalCase(strings.TrimPrefix(module.TestModule, conformanceModulePrefix)) + + if counts[module.TestModule] > 1 { + name += conformanceVariantSuffix(module.Variant) + } + + names[i] = name + } + + return names +} + +func conformanceVariantSuffix(variant map[string]string) (suffix string) { + keys := make([]string, 0, len(variant)) + + for key := range variant { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + suffix += conformancePascalCase(variant[key]) + } + + return suffix +} + +func conformancePascalCase(value string) string { + builder := &strings.Builder{} + + for _, part := range strings.FieldsFunc(value, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + runes := []rune(strings.ToLower(part)) + runes[0] = unicode.ToUpper(runes[0]) + + builder.WriteString(string(runes)) + } + + return builder.String() +} diff --git a/internal/suites/conformance_names_test.go b/internal/suites/conformance_names_test.go new file mode 100644 index 000000000..2c2dafc0d --- /dev/null +++ b/internal/suites/conformance_names_test.go @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConformanceSubtestNames(t *testing.T) { + testCases := []struct { + name string + have []ConformancePlanModule + expected []string + }{ + { + "ShouldStripPrefixAndPascalCase", + []ConformancePlanModule{{TestModule: "oidcc-server"}}, + []string{"Server"}, + }, + { + "ShouldPascalCaseEverySegment", + []ConformancePlanModule{{TestModule: "oidcc-ensure-request-without-nonce-succeeds-for-code-flow"}}, + []string{"EnsureRequestWithoutNonceSucceedsForCodeFlow"}, + }, + { + "ShouldKeepDigits", + []ConformancePlanModule{{TestModule: "oidcc-max-age-10000"}}, + []string{"MaxAge10000"}, + }, + { + "ShouldHandleModulesWithoutThePrefix", + []ConformancePlanModule{{TestModule: "oidcc-config-certification"}, {TestModule: "discovery-issuer-not-matching-config"}}, + []string{"ConfigCertification", "DiscoveryIssuerNotMatchingConfig"}, + }, + { + "ShouldNotSuffixUniqueModules", + []ConformancePlanModule{ + {TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code"}}, + {TestModule: "oidcc-scope-address", Variant: map[string]string{"response_type": "code"}}, + }, + []string{"Server", "ScopeAddress"}, + }, + { + "ShouldSuffixRepeatedModulesWithTheirVariant", + []ConformancePlanModule{ + {TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code id_token"}}, + {TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code id_token token"}}, + }, + []string{"ServerCodeIdToken", "ServerCodeIdTokenToken"}, + }, + { + "ShouldOrderVariantKeysDeterministically", + []ConformancePlanModule{ + {TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code", "response_mode": "form_post"}}, + {TestModule: "oidcc-server", Variant: map[string]string{"response_type": "code", "response_mode": "query"}}, + }, + []string{"ServerFormPostCode", "ServerQueryCode"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, ConformanceSubtestNames(tc.have)) + }) + } +} + +func TestConformanceResultAccepted(t *testing.T) { + testCases := []struct { + name string + module string + result string + expected bool + }{ + {"ShouldAcceptPassedFromAnyModule", "oidcc-server", "PASSED", true}, + {"ShouldAcceptReviewFromAnyModule", "oidcc-server", "REVIEW", true}, + {"ShouldAcceptSkippedFromAnyModule", "oidcc-server", "SKIPPED", true}, + {"ShouldAcceptAWarningFromAListedModule", "oidcc-ensure-request-with-acr-values-succeeds", "WARNING", true}, + {"ShouldRejectAWarningFromAnUnlistedModule", "oidcc-scope-profile", "WARNING", false}, + {"ShouldRejectAFailureFromAListedModule", "oidcc-ensure-request-with-acr-values-succeeds", "FAILED", false}, + {"ShouldRejectAFailure", "oidcc-server", "FAILED", false}, + {"ShouldRejectAnUnknownResult", "oidcc-server", "", false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, conformanceResultAccepted(tc.module, tc.result)) + }) + } + + for module, reason := range conformanceAcceptedWarnings { + assert.NotEmptyf(t, reason, "the WARNING accepted from '%s' must say why it is expected", module) + } +} + +func TestConformanceArchivePath(t *testing.T) { + t.Run("ShouldWriteWhereTheCIArtifactPathCollectsIt", func(t *testing.T) { + t.Setenv("CI", "true") + + assert.Equal(t, filepath.Join("..", "..", "oidc-conformance-plans", "conformance-basic.zip"), conformanceArchivePath("basic")) + }) + + t.Run("ShouldWriteToTheTemporaryDirectoryOutsideCI", func(t *testing.T) { + t.Setenv("CI", "") + + assert.Equal(t, filepath.Join(os.TempDir(), "authelia-suites-oidc-conformance-plans", composeProjectName(), "conformance-basic.zip"), conformanceArchivePath("basic")) + }) +} + +func TestBuildkiteLogMarkers(t *testing.T) { + t.Run("ShouldMarkGroupsInBuildkiteWhenDebugging", func(t *testing.T) { + t.Setenv("BUILDKITE", "true") + t.Setenv("SUITE_DEBUG", "true") + + assert.Equal(t, "--- OIDC Conformance Plan: basic", buildkiteGroup("OIDC Conformance Plan: basic")) + assert.Equal(t, "^^^ +++", buildkiteExpandGroup()) + }) + + t.Run("ShouldMarkNothingInBuildkiteWhenNotDebugging", func(t *testing.T) { + t.Setenv("BUILDKITE", "true") + t.Setenv("SUITE_DEBUG", "") + + assert.Empty(t, buildkiteGroup("OIDC Conformance Plan: basic")) + assert.Empty(t, buildkiteExpandGroup()) + }) + + t.Run("ShouldMarkNothingOutsideBuildkite", func(t *testing.T) { + t.Setenv("BUILDKITE", "") + t.Setenv("SUITE_DEBUG", "true") + + assert.Empty(t, buildkiteGroup("OIDC Conformance Plan: basic")) + assert.Empty(t, buildkiteExpandGroup()) + }) +} + +func TestConformancePlanSummary(t *testing.T) { + assert.Equal(t, "Plan 'basic': 45 accepted, 2 failed, 1 skipped", conformancePlanSummary("basic", 45, 2, 1)) +} + +func TestConformanceArchiveReference(t *testing.T) { + t.Run("ShouldLinkTheJobsArtifactsInBuildkite", func(t *testing.T) { + t.Setenv("CI", "true") + t.Setenv("BUILDKITE_BUILD_URL", "https://buildkite.com/authelia/authelia/builds/58611") + t.Setenv("BUILDKITE_JOB_ID", "0192") + + assert.Equal(t, "oidc-conformance-plans/conformance-basic.zip in the job's artifacts: https://buildkite.com/authelia/authelia/builds/58611/waterfall?jid=0192&tab=artifacts", + conformanceArchiveReference("basic", "https://conformance.example.com:8443/log-detail.html?log=m1")) + }) + + t.Run("ShouldPointAtTheLiveLogElsewhere", func(t *testing.T) { + t.Setenv("CI", "") + t.Setenv("BUILDKITE_BUILD_URL", "") + t.Setenv("BUILDKITE_JOB_ID", "") + + assert.Equal(t, conformanceArchivePath("basic")+", or the live log while the suite is up: https://conformance.example.com:8443/log-detail.html?log=m1", + conformanceArchiveReference("basic", "https://conformance.example.com:8443/log-detail.html?log=m1")) + }) +} + +func TestOIDCConformanceRelease(t *testing.T) { + assert.Equal(t, "v4.39.24 (commit 7498f635a5f3)", oidcConformanceRelease("7498f635a5f3", "v4.39.24")) + assert.Equal(t, "commit 7498f635a5f3", oidcConformanceRelease("7498f635a5f3", "")) + assert.Empty(t, oidcConformanceRelease("", ""), "without a commit the builder's own version is left in place") +} + +func TestConformanceOutcomeAccepted(t *testing.T) { + testCases := []struct { + name string + outcome ConformanceOutcome + expected bool + }{ + {"ShouldAcceptAFinishedPass", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "PASSED"}, true}, + {"ShouldAcceptAFinishedListedWarning", ConformanceOutcome{Module: "oidcc-ensure-request-with-acr-values-succeeds", Status: "FINISHED", Result: "WARNING"}, true}, + {"ShouldRejectAnInterruptedReview", ConformanceOutcome{Module: "oidcc-server", Status: "INTERRUPTED", Result: "REVIEW"}, false}, + {"ShouldRejectAnInterruptedPass", ConformanceOutcome{Module: "oidcc-server", Status: "INTERRUPTED", Result: "PASSED"}, false}, + {"ShouldRejectAFinishedFailure", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "FAILED"}, false}, + {"ShouldRejectASuiteError", ConformanceOutcome{Module: "oidcc-server", Status: "FINISHED", Result: "PASSED", Err: errors.New("stalled")}, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, conformanceOutcomeAccepted(tc.outcome)) + }) + } +} diff --git a/internal/suites/conformance_progress_test.go b/internal/suites/conformance_progress_test.go new file mode 100644 index 000000000..5a34d3ead --- /dev/null +++ b/internal/suites/conformance_progress_test.go @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestConformanceProgress(t *testing.T) { + patience := time.Second * 30 + start := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC) + + t.Run("ShouldStallOnAnUnrecognizedPageThatNeverMoves", func(t *testing.T) { + progress := newConformanceProgress(patience, start) + + assert.False(t, progress.observe(start, "https://stuck.example.com", false)) + assert.False(t, progress.observe(start.Add(time.Second*29), "https://stuck.example.com", false)) + assert.True(t, progress.observe(start.Add(time.Second*31), "https://stuck.example.com", false)) + }) + + t.Run("ShouldNotStallWhileTheURLKeepsChanging", func(t *testing.T) { + progress := newConformanceProgress(patience, start) + + for i, at := range []time.Duration{0, time.Second * 25, time.Second * 50, time.Second * 75} { + assert.Falsef(t, progress.observe(start.Add(at), "https://hop.example.com/"+string(rune('a'+i)), false), "hop %d", i) + } + }) + + t.Run("ShouldNotStallOnAPageTheDriverIsActingOn", func(t *testing.T) { + progress := newConformanceProgress(patience, start) + + assert.False(t, progress.observe(start, "https://login.example.com/consent", true)) + assert.False(t, progress.observe(start.Add(time.Second*45), "https://login.example.com/consent", true)) + assert.False(t, progress.observe(start.Add(time.Second*60), "https://login.example.com/consent", true)) + }) + + t.Run("ShouldStallOnceAPageStopsBeingActedOn", func(t *testing.T) { + progress := newConformanceProgress(patience, start) + + assert.False(t, progress.observe(start, "https://login.example.com/consent", true)) + assert.False(t, progress.observe(start.Add(time.Second*10), "https://login.example.com/consent", false)) + assert.True(t, progress.observe(start.Add(time.Second*41), "https://login.example.com/consent", false)) + }) +} diff --git a/internal/suites/conformance_runner.go b/internal/suites/conformance_runner.go new file mode 100644 index 000000000..761d34068 --- /dev/null +++ b/internal/suites/conformance_runner.go @@ -0,0 +1,568 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +const ( + conformanceReauthenticationLeg = 1 + conformanceStatusConfigured = "CONFIGURED" + conformanceStatusWaiting = "WAITING" + conformanceStatusFinished = "FINISHED" + conformanceStatusInterrupted = "INTERRUPTED" + conformanceModuleTimeout = time.Minute * 5 + conformancePlaceholderTimeout = time.Second * 30 + conformanceUploadSettleInterval = time.Second + conformanceUploadSettleTimeout = time.Second * 10 + conformancePlaceholderInterval = time.Second + conformanceOutcomeInfoTimeout = time.Second * 10 + conformanceErrorSettleTimeout = time.Second * 30 +) + +var ( + conformanceTerminalStates = []string{conformanceStatusFinished, conformanceStatusInterrupted} + + conformanceOverrides = map[string]ConformanceOverride{ + // The second authorization carries prompt=login, so Authelia must ask for credentials again despite the session. + "oidcc-prompt-login": {Assert: conformanceAssertReauthentication, Screenshot: ConformanceScreenshotReauthentication}, + + // The max age one module sets max_age=1 with a second of delay must force re-authentication and an auth_time claim. + "oidcc-max-age-1": {Assert: conformanceAssertReauthentication, Screenshot: ConformanceScreenshotReauthentication}, + + // The max age 1000 module sets max_age=10000 is longer than the session has existed, so the provider must not ask + // again. + "oidcc-max-age-10000": {Assert: conformanceAssertNoReauthentication}, + + // The redirect_uri is not registered, so Authelia must show its own error page rather than send the browser to it. + // This is the page the module's screenshot placeholder asks for. + "oidcc-ensure-registered-redirect-uri": {Assert: conformanceAssertErrorPage, Screenshot: ConformanceScreenshotErrorPage}, + + // The request object carries a registered redirect_uri and the query an unregistered one. The module accepts either + // Authelia using the request object's and completing the flow, or showing its error page, and only asks for a + // screenshot of the latter. + "oidcc-ensure-request-object-with-redirect-uri": {Screenshot: ConformanceScreenshotErrorPage}, + + // Every module whose upstream summary says to remove any cookies received from the provider, and no others. The + // plans also carry oidcc-registration-logo-uri, -policy-uri and -tos-uri under that instruction, but those are + // Dynamic OP modules and none of the profiles Authelia is certified for includes them. + // + // oidcc-prompt-none-not-logged-in is the one where a stale session is not merely untidy: prompt=none against a + // live session returns a code, and the module is asserting that the provider errors instead, so it fails. + "oidcc-display-page": {ClearCookies: true}, + "oidcc-display-popup": {ClearCookies: true}, + "oidcc-login-hint": {ClearCookies: true}, + "oidcc-prompt-none-not-logged-in": {ClearCookies: true}, + "oidcc-ui-locales": {ClearCookies: true}, + } + + conformanceUnattended = map[string]string{ + "oidcc-server-rotate-keys": "requires the provider's signing keys to be rotated by hand while the module waits", + } +) + +// ConformanceOutcome is the result of one module, produced off the test goroutine and asserted on it. +type ConformanceOutcome struct { + Name string + Module string + ID string + Result string + Status string + LogURL string + Skipped bool + Reason string + Err error + Trace []string +} + +// ConformanceOverride is the per module behavior which differs from the reactive default. +type ConformanceOverride struct { + // ClearCookies discards the context's cookies before the module's first browser leg, for the modules whose + // instructions are to remove any cookies received from the provider before proceeding. + ClearCookies bool + + // Assert stands in for the screenshot the conformance suite would otherwise want. It is handed what the driver + // observed over one browser leg, once that leg has been driven to its end, and its error becomes the module's + // verdict. It is given the record rather than the browser because by the time a leg ends the browser is on the + // conformance suite's callback, so nothing about Authelia's pages can be established from the live DOM any more. + // The record carries the leg's index, so an expectation which belongs to the second authorization round trip is + // not applied to the first. + Assert func(leg ConformanceLeg) error + + // Screenshot is the page this module's screenshot placeholder asks for. A placeholder is only filled with a capture + // of that page, so one from a module without it is left unfilled and the module stalls. + Screenshot ConformanceScreenshot +} + +// ConformanceScreenshot is a page a module's screenshot placeholder asks for. +type ConformanceScreenshot int + +const ( + // ConformanceScreenshotNone is a module which asks for no screenshot. + ConformanceScreenshotNone ConformanceScreenshot = iota + + // ConformanceScreenshotReauthentication is Authelia asking for the password again on the second authorization. + ConformanceScreenshotReauthentication + + // ConformanceScreenshotErrorPage is Authelia's error page. + ConformanceScreenshotErrorPage +) + +func (o ConformanceOverride) screenshotOf(leg ConformanceLeg) string { + switch o.Screenshot { + case ConformanceScreenshotReauthentication: + if leg.Index == conformanceReauthenticationLeg { + return leg.LoginScreenshot + } + case ConformanceScreenshotErrorPage: + return leg.ErrorScreenshot + } + + return "" +} + +func conformanceAssertReauthentication(leg ConformanceLeg) error { + if leg.Index != conformanceReauthenticationLeg { + return nil + } + + if !leg.Reauthentication && !leg.FirstFactor { + return errors.New("expected Authelia to ask for the password again on the second authorization but it did not") + } + + return nil +} + +func conformanceAssertNoReauthentication(leg ConformanceLeg) error { + if leg.Index != conformanceReauthenticationLeg { + return nil + } + + if leg.Reauthentication || leg.FirstFactor { + return errors.New("expected Authelia to reuse the existing session on the second authorization but it asked for the password again") + } + + return nil +} + +func conformanceAssertErrorPage(leg ConformanceLeg) error { + if !leg.AutheliaError { + return errors.New("expected Authelia to show an error page rather than redirect to the client but it did not") + } + + return nil +} + +// ConformanceRunner walks one plan's modules in order. +type ConformanceRunner struct { + client *ConformanceClient + browser *ConformanceBrowser + planID string + modules []ConformancePlanModule + names []string + + // debug is whether each module's trace is recorded. + debug bool + + // trace is the trace of the module being run. Modules run one at a time, so it is replaced as each begins. + trace *ConformanceTrace +} + +// NewConformanceRunner returns a ConformanceRunner for one plan. browser may be nil only when no module in modules +// reaches the WAITING state, which in practice means only in tests. +func NewConformanceRunner(client *ConformanceClient, browser *ConformanceBrowser, planID string, modules []ConformancePlanModule) *ConformanceRunner { + return &ConformanceRunner{ + client: client, + browser: browser, + planID: planID, + modules: modules, + names: ConformanceSubtestNames(modules), + } +} + +// Run walks the plan on its own goroutine and returns the channel its outcomes arrive on, one per module in plan +// order. The channel closes when the plan is done. Nothing here touches [testing.T]: an outcome carries its error so the +// test goroutine can be the one that fails. +func (r *ConformanceRunner) Run(ctx context.Context) <-chan ConformanceOutcome { + outcomes := make(chan ConformanceOutcome, len(r.modules)) + + go func() { + defer close(outcomes) + + for i, module := range r.modules { + outcomes <- r.run(ctx, r.names[i], module) + } + }() + + return outcomes +} + +func (r *ConformanceRunner) run(ctx context.Context, name string, module ConformancePlanModule) (outcome ConformanceOutcome) { + outcome = ConformanceOutcome{Name: name, Module: module.TestModule} + + r.trace = NewConformanceTrace(r.debug) + + if r.browser != nil { + r.browser.trace = r.trace + } + + defer func() { + outcome.Trace = r.trace.Lines() + }() + + if reason, ok := conformanceUnattended[module.TestModule]; ok { + outcome.Skipped, outcome.Reason = true, reason + + return outcome + } + + parent := ctx + + ctx, cancel := context.WithTimeout(ctx, conformanceModuleTimeout) + defer cancel() + + override := conformanceOverrides[module.TestModule] + + id, state, err := r.createAndAdvance(ctx, module, override) + + r.trace.Logf("Module '%s' created as '%s' in state '%s'", module.TestModule, id, state) + + if id != "" { + outcome.ID, outcome.LogURL = id, r.client.LogDetailURL(id) + } + + if err != nil { + outcome.Err = err + + if id != "" { + r.settleAfterError(parent, id) + } + + return outcome + } + + if state == conformanceStatusWaiting { + if err = r.interact(ctx, id, override); err != nil { + outcome.Err = err + } + } + + if outcome.Err == nil { + if _, err = r.client.WaitState(ctx, id, conformanceTerminalStates...); err != nil { + outcome.Err = err + } + } + + if outcome.Err != nil { + r.settleAfterError(parent, id) + } + + infoCtx, infoCancel := context.WithTimeout(context.WithoutCancel(ctx), conformanceOutcomeInfoTimeout) + defer infoCancel() + + info, err := r.client.Info(infoCtx, id) + if err != nil { + if outcome.Err == nil { + outcome.Err = err + } + + return outcome + } + + outcome.Status, outcome.Result = info.Status, info.Result + + return outcome +} + +func (r *ConformanceRunner) settleAfterError(parent context.Context, id string) { + ctx, cancel := context.WithTimeout(parent, conformanceErrorSettleTimeout) + defer cancel() + + state, err := r.client.WaitState(ctx, id, append([]string{conformanceStatusConfigured, conformanceStatusWaiting}, conformanceTerminalStates...)...) + + r.trace.Logf("Module '%s' settled in state '%s' after the suite failed it (%v)", id, state, err) +} + +func (r *ConformanceRunner) createAndAdvance(ctx context.Context, module ConformancePlanModule, override ConformanceOverride) (id, state string, err error) { + if id, err = r.client.CreateTest(ctx, r.planID, module.TestModule, module.Variant); err != nil { + return "", "", fmt.Errorf("error creating module '%s': %w", module.TestModule, err) + } + + if override.ClearCookies && r.browser != nil { + if err = r.browser.ClearCookies(); err != nil { + return id, "", err + } + } + + if state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusConfigured, conformanceStatusWaiting}, conformanceTerminalStates...)...); err != nil { + return id, "", err + } + + if state != conformanceStatusConfigured { + return id, state, nil + } + + if err = r.client.StartTest(ctx, id); err != nil { + return id, "", err + } + + state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusWaiting}, conformanceTerminalStates...)...) + + return id, state, err +} + +func (r *ConformanceRunner) interact(ctx context.Context, id string, override ConformanceOverride) (err error) { + if r.browser == nil { + return errors.New("the module needs a browser but the runner has none") + } + + legs := &conformanceLegs{visited: map[string]bool{}} + + var deadline time.Time + + for { + if ctx.Err() != nil { + return fmt.Errorf("module '%s' did not leave the WAITING state: %w", id, ctx.Err()) + } + + status, err := r.client.BrowserStatus(ctx, id) + if err != nil { + return err + } + + if deadline.IsZero() { + deadline = time.Now().Add(conformancePlaceholderTimeout) + } + + progressed, err := r.visitURLs(ctx, id, override, legs, status.URLs) + if err != nil { + return err + } + + filled, pending, err := r.fillPlaceholders(ctx, id, legs.screenshot) + if err != nil { + return err + } + + legs.pendingPlaceholder = pending + + done, settled, err := r.advance(ctx, id, filled) + if err != nil { + return err + } + + if done { + return nil + } + + if progressed || filled || settled { + deadline = time.Now().Add(conformancePlaceholderTimeout) + } else if time.Now().After(deadline) { + return legs.stalled(id) + } + + time.Sleep(conformancePlaceholderInterval) + } +} + +func (r *ConformanceRunner) advance(ctx context.Context, id string, filled bool) (done, settled bool, err error) { + var state string + + if state, err = r.moduleState(ctx, id, filled); err != nil { + return false, false, err + } + + switch state { + case conformanceStatusFinished, conformanceStatusInterrupted: + return true, false, nil + case conformanceStatusWaiting: + return false, false, nil + } + + r.trace.Logf("Module '%s' is in state '%s' between browser legs", id, state) + + if state, err = r.client.WaitState(ctx, id, append([]string{conformanceStatusWaiting}, conformanceTerminalStates...)...); err != nil { + return false, false, err + } + + return state != conformanceStatusWaiting, true, nil +} + +type conformanceLegs struct { + visited map[string]bool + count int + errorURL string + screenshot string + pendingPlaceholder bool +} + +func (l *conformanceLegs) stalled(id string) error { + if l.pendingPlaceholder { + return fmt.Errorf("module '%s' is waiting for an evidence image, but no leg showed the page it asks for", id) + } + + if l.errorURL != "" { + return &ConformanceErrorPageStallError{ID: id, URL: l.errorURL} + } + + return fmt.Errorf("module '%s' stayed in the WAITING state with nothing left to visit or fill", id) +} + +// ConformanceErrorPageStallError is a module left waiting for the flow to return to the client after Authelia ended +// it on an error page, which the module had no placeholder for. +type ConformanceErrorPageStallError struct { + ID string + URL string +} + +// Error names the module and what Authelia's error page reported. +func (e *ConformanceErrorPageStallError) Error() string { + return fmt.Sprintf("module '%s' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: %s", + e.ID, conformanceDescribeErrorPage(e.URL)) +} + +func conformanceDescribeErrorPage(uri string) string { + u, err := url.Parse(uri) + if err != nil { + return uri + } + + query := u.Query() + + var parts []string + + for _, field := range []struct{ label, key string }{ + {"error", "error"}, + {"description", "error_description"}, + {"hint", "error_hint"}, + {"debug", "error_debug"}, + } { + if value := query.Get(field.key); value != "" { + parts = append(parts, fmt.Sprintf("%s '%s'", field.label, value)) + } + } + + if len(parts) == 0 { + return uri + } + + return strings.Join(parts, ", ") +} + +func (r *ConformanceRunner) visitURLs(ctx context.Context, id string, override ConformanceOverride, legs *conformanceLegs, urls []string) (progressed bool, err error) { + for _, uri := range urls { + if legs.visited[uri] { + continue + } + + legs.visited[uri] = true + progressed = true + + index := legs.count + legs.count++ + + r.trace.Logf("Driving leg %d at '%s'", index, uri) + + leg, err := r.browser.Drive(ctx, index, uri) + if err != nil { + return progressed, err + } + + r.trace.Logf("Finished leg %d: first factor %t, consent %t, reauthentication %t, error page %t", + index, leg.FirstFactor, leg.Consent, leg.Reauthentication, leg.AutheliaError) + + legs.errorURL = leg.ErrorURL + legs.screenshot = conformanceLatest(legs.screenshot, override.screenshotOf(leg)) + + if override.Assert != nil { + if err = override.Assert(leg); err != nil { + return progressed, err + } + } + + if err = r.client.Visit(ctx, id, uri); err != nil { + return progressed, err + } + } + + return progressed, nil +} + +func (r *ConformanceRunner) fillPlaceholders(ctx context.Context, id, screenshot string) (filled, pending bool, err error) { + entries, err := r.client.Log(ctx, id) + if err != nil { + return false, false, err + } + + for _, entry := range entries { + if entry.Upload == "" { + continue + } + + if screenshot == "" { + pending = true + + continue + } + + r.trace.Logf("Module '%s' uploading evidence image '%s'", id, entry.Upload) + + if err = r.client.UploadPlaceholder(ctx, id, entry.Upload, screenshot); err != nil { + return filled, pending, err + } + + filled = true + } + + return filled, pending, nil +} + +func (r *ConformanceRunner) awaitPlaceholderSettled(ctx context.Context, id string, interval, timeout time.Duration) (current string, err error) { + deadline := time.Now().Add(timeout) + + for { + var info *ConformanceTestInfo + + if info, err = r.client.Info(ctx, id); err != nil { + return "", fmt.Errorf("error reading the status of module '%s' while waiting for it to take up its evidence image: %w", id, err) + } + + if info.Status != conformanceStatusWaiting || time.Now().After(deadline) { + return info.Status, nil + } + + select { + case <-ctx.Done(): + return "", fmt.Errorf("module '%s' did not leave the %s state after its evidence image was uploaded: %w", id, conformanceStatusWaiting, ctx.Err()) + case <-time.After(interval): + } + } +} + +func (r *ConformanceRunner) moduleState(ctx context.Context, id string, filled bool) (state string, err error) { + if filled { + if state, err = r.awaitPlaceholderSettled(ctx, id, conformanceUploadSettleInterval, conformanceUploadSettleTimeout); err != nil { + return "", err + } + + r.trace.Logf("Module '%s' is in state '%s' after its evidence image was uploaded", id, state) + + return state, nil + } + + var info *ConformanceTestInfo + + if info, err = r.client.Info(ctx, id); err != nil { + return "", err + } + + return info.Status, nil +} diff --git a/internal/suites/conformance_runner_errors_test.go b/internal/suites/conformance_runner_errors_test.go new file mode 100644 index 000000000..9a98f8cc7 --- /dev/null +++ b/internal/suites/conformance_runner_errors_test.go @@ -0,0 +1,305 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceRunner_ReportsTheStepWhichFailed(t *testing.T) { + created := conformanceReply(http.StatusCreated, `{"id":"m1"}`) + + testCases := []struct { + name string + routes map[string]http.HandlerFunc + expected string + id string + }{ + { + "ShouldReportAModuleWhichCouldNotBeCreated", + map[string]http.HandlerFunc{"/api/runner": conformanceReply(http.StatusInternalServerError, "broken")}, + "error creating module 'oidcc-server'", + "", + }, + { + "ShouldReportAModuleWhichCouldNotBeWaitedOn", + map[string]http.HandlerFunc{ + "/api/runner": created, + "/api/runner/m1/wait-state": conformanceReply(http.StatusBadRequest, "bad"), + }, + "expected status 200 but got 400", + "m1", + }, + { + "ShouldReportAConfiguredModuleWhichCouldNotBeStarted", + map[string]http.HandlerFunc{ + "/api/runner": created, + "/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"CONFIGURED"}`), + "POST /api/runner/m1": conformanceReply(http.StatusInternalServerError, "broken"), + }, + "expected status 200 but got 500", + "m1", + }, + { + "ShouldReportAModuleWhoseStatusCouldNotBeRead", + map[string]http.HandlerFunc{ + "/api/runner": created, + "/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`), + "/api/info/m1": conformanceReply(http.StatusInternalServerError, "broken"), + }, + "expected status 200 but got 500", + "m1", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + outcome := conformanceRunOne(t, newConformanceFakeClient(t, tc.routes)) + + require.Error(t, outcome.Err) + assert.ErrorContains(t, outcome.Err, tc.expected) + assert.Equal(t, tc.id, outcome.ID) + }) + } +} + +func TestConformanceRunner_StartsAConfiguredModule(t *testing.T) { + var started atomic.Bool + + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/runner": conformanceReply(http.StatusCreated, `{"id":"m1"}`), + "/api/runner/m1/wait-state": func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Query().Get("states"), conformanceStatusConfigured) { + _, _ = w.Write([]byte(`{"state":"CONFIGURED"}`)) + + return + } + + _, _ = w.Write([]byte(`{"state":"FINISHED"}`)) + }, + "POST /api/runner/m1": func(w http.ResponseWriter, _ *http.Request) { + started.Store(true) + + _, _ = w.Write([]byte(`{}`)) + }, + "/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`), + }) + + outcome := conformanceRunOne(t, client) + + require.NoError(t, outcome.Err) + assert.True(t, started.Load()) + assert.Equal(t, "PASSED", outcome.Result) +} + +func TestConformanceRunner_InteractReportsTheCallWhichFailed(t *testing.T) { + status := conformanceReply(http.StatusOK, `{"urls":[]}`) + log := conformanceReply(http.StatusOK, `[]`) + + testCases := []struct { + name string + routes map[string]http.HandlerFunc + expected string + }{ + { + "ShouldReportTheBrowserStatus", + map[string]http.HandlerFunc{"/api/runner/browser/m1": conformanceReply(http.StatusInternalServerError, "status broken")}, + "status broken", + }, + { + "ShouldReportTheLog", + map[string]http.HandlerFunc{ + "/api/runner/browser/m1": status, + "/api/log/m1": conformanceReply(http.StatusInternalServerError, "log broken"), + }, + "log broken", + }, + { + "ShouldReportTheStatus", + map[string]http.HandlerFunc{ + "/api/runner/browser/m1": status, + "/api/log/m1": log, + "/api/info/m1": conformanceReply(http.StatusInternalServerError, "info broken"), + }, + "info broken", + }, + { + "ShouldReportTheWaitForAModuleBetweenLegs", + map[string]http.HandlerFunc{ + "/api/runner/browser/m1": status, + "/api/log/m1": log, + "/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"RUNNING"}`), + "/api/runner/m1/wait-state": conformanceReply(http.StatusBadRequest, "wait broken"), + }, + "wait broken", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + runner := NewConformanceRunner(newConformanceFakeClient(t, tc.routes), &ConformanceBrowser{}, "plan1", nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + assert.ErrorContains(t, runner.interact(ctx, "m1", ConformanceOverride{}), tc.expected) + }) + } +} + +func TestConformanceRunner_FillPlaceholdersReportsAnUploadWhichFailed(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/log/m1": conformanceReply(http.StatusOK, `[{"msg":"upload","upload":"ph1","result":"REVIEW"}]`), + "/api/log/m1/images/ph1": conformanceReply(http.StatusInternalServerError, "upload broken"), + }) + + filled, pending, err := NewConformanceRunner(client, nil, "plan1", nil).fillPlaceholders(context.Background(), "m1", conformanceTestScreenshot) + + assert.ErrorContains(t, err, "upload broken") + assert.False(t, filled) + assert.False(t, pending) +} + +func TestConformanceRunner_InteractStopsWhenTheContextHasExpired(t *testing.T) { + runner := NewConformanceRunner(nil, &ConformanceBrowser{}, "plan1", nil) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + assert.ErrorContains(t, runner.interact(ctx, "m1", ConformanceOverride{}), "module 'm1' did not leave the WAITING state") +} + +func TestConformanceRunner_InteractPollsAWaitingModuleAgain(t *testing.T) { + var infos atomic.Int32 + + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/runner/browser/m1": conformanceReply(http.StatusOK, `{"urls":[]}`), + "/api/log/m1": conformanceReply(http.StatusOK, `[]`), + "/api/info/m1": func(w http.ResponseWriter, _ *http.Request) { + if infos.Add(1) == 1 { + _, _ = w.Write([]byte(`{"_id":"m1","status":"WAITING"}`)) + + return + } + + _, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`)) + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + require.NoError(t, NewConformanceRunner(client, &ConformanceBrowser{}, "plan1", nil).interact(ctx, "m1", ConformanceOverride{})) + assert.Equal(t, int32(2), infos.Load()) +} + +func TestConformanceRunner_ModuleState(t *testing.T) { + t.Run("ShouldSettleAfterAnEvidenceImage", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED"}`), + }) + + runner := NewConformanceRunner(client, nil, "plan1", nil) + runner.trace = NewConformanceTrace(true) + + state, err := runner.moduleState(context.Background(), "m1", true) + + require.NoError(t, err) + assert.Equal(t, conformanceStatusFinished, state) + require.Len(t, runner.trace.Lines(), 1) + assert.Contains(t, runner.trace.Lines()[0], "Module 'm1' is in state 'FINISHED' after its evidence image was uploaded") + }) + + t.Run("ShouldReportAStatusWhichCouldNotBeReadWhileSettling", func(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/info/m1": conformanceReply(http.StatusInternalServerError, "broken"), + }) + + _, err := NewConformanceRunner(client, nil, "plan1", nil).moduleState(context.Background(), "m1", true) + + assert.ErrorContains(t, err, "while waiting for it to take up its evidence image") + }) +} + +func TestConformanceDescribeErrorPageFallsBackForAnUnparsableURL(t *testing.T) { + assert.Equal(t, "%zz", conformanceDescribeErrorPage("%zz")) +} + +func TestConformanceRunner_HandsTheModuleTraceToItsBrowser(t *testing.T) { + client := newConformanceFakeClient(t, map[string]http.HandlerFunc{ + "/api/runner": conformanceReply(http.StatusCreated, `{"id":"m1"}`), + "/api/runner/m1/wait-state": conformanceReply(http.StatusOK, `{"state":"FINISHED"}`), + "/api/info/m1": conformanceReply(http.StatusOK, `{"_id":"m1","status":"FINISHED","result":"PASSED"}`), + }) + + browser := &ConformanceBrowser{} + runner := NewConformanceRunner(client, browser, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}) + runner.debug = true + + for outcome := range runner.Run(context.Background()) { + require.NoError(t, outcome.Err) + } + + assert.Same(t, runner.trace, browser.trace, "what the driver does is recorded in the module's trace") +} + +func newConformanceFakeClient(t *testing.T, routes map[string]http.HandlerFunc) *ConformanceClient { + t.Helper() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route, ok := routes[r.Method+" "+r.URL.Path] + if !ok { + route, ok = routes[r.URL.Path] + } + + if !ok { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + + return + } + + route(w, r) + })) + + t.Cleanup(server.Close) + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + return client +} + +func conformanceReply(status int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + } +} + +func conformanceRunOne(t *testing.T, client *ConformanceClient) ConformanceOutcome { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + + return outcomes[0] +} diff --git a/internal/suites/conformance_runner_test.go b/internal/suites/conformance_runner_test.go new file mode 100644 index 000000000..fbf481608 --- /dev/null +++ b/internal/suites/conformance_runner_test.go @@ -0,0 +1,659 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceRunner_ReportsOutcomesInPlanOrder(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { //nolint:staticcheck // a tagged switch would obscure the multi-condition cases below. + case r.URL.Path == "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"` + r.URL.Query().Get("test") + `-id"}`)) //nolint:gosec // test double echoing a query param set by this same test, not attacker controlled. + case r.URL.Path == "/api/runner/oidcc-server-id/wait-state", + r.URL.Path == "/api/runner/oidcc-scope-address-id/wait-state": + _, _ = w.Write([]byte(`{"state":"FINISHED"}`)) + case r.URL.Path == "/api/info/oidcc-server-id": + _, _ = w.Write([]byte(`{"_id":"oidcc-server-id","status":"FINISHED","result":"PASSED"}`)) + case r.URL.Path == "/api/info/oidcc-scope-address-id": + _, _ = w.Write([]byte(`{"_id":"oidcc-scope-address-id","status":"FINISHED","result":"WARNING"}`)) + case r.URL.Path == "/api/log/oidcc-server-id", r.URL.Path == "/api/log/oidcc-scope-address-id": + _, _ = w.Write([]byte(`[]`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + modules := []ConformancePlanModule{{TestModule: "oidcc-server"}, {TestModule: "oidcc-scope-address"}} + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", modules).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 2) + + assert.Equal(t, "Server", outcomes[0].Name) + assert.Equal(t, "PASSED", outcomes[0].Result) + require.NoError(t, outcomes[0].Err) + + assert.Equal(t, "ScopeAddress", outcomes[1].Name) + assert.Equal(t, "WARNING", outcomes[1].Result) +} + +func TestConformanceRunner_SkipsUnattendedModules(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("the runner must not contact the server for a skipped module, got %s", r.URL.Path) + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + modules := []ConformancePlanModule{{TestModule: "oidcc-server-rotate-keys"}} + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", modules).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + assert.True(t, outcomes[0].Skipped) + assert.Equal(t, "ServerRotateKeys", outcomes[0].Name) + assert.NotEmpty(t, outcomes[0].Reason) +} + +func TestConformanceRunner_FillPlaceholdersUploadsOnlyUnfilledEntries(t *testing.T) { + t.Run("UploadsOnlyTheEntryWithAnUnfilledPlaceholder", func(t *testing.T) { + var ( + uploadCount int + uploadedID string + uploadedBody []byte + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/log/m1": + _, _ = w.Write([]byte(`[{"msg":"info","result":"INFO"},{"msg":"upload","upload":"ph1","result":"REVIEW"}]`)) + case "/api/log/m1/images/ph1": + uploadCount++ + uploadedID = "ph1" + uploadedBody, _ = io.ReadAll(r.Body) + + _, _ = w.Write([]byte(`{}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + filled, pending, err := runner.fillPlaceholders(ctx, "m1", conformanceTestScreenshot) + require.NoError(t, err) + + assert.True(t, filled) + assert.False(t, pending) + assert.Equal(t, 1, uploadCount) + assert.Equal(t, "ph1", uploadedID) + assert.Equal(t, conformanceTestScreenshot, string(uploadedBody)) + }) + + t.Run("UploadsNothingWhenNoPlaceholderIsUnfilled", func(t *testing.T) { + var uploadCalled bool + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/log/m1": + _, _ = w.Write([]byte(`[{"msg":"info","result":"INFO"}]`)) + case "/api/log/m1/images/ph1": + uploadCalled = true + + _, _ = w.Write([]byte(`{}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + filled, pending, err := runner.fillPlaceholders(ctx, "m1", conformanceTestScreenshot) + require.NoError(t, err) + + assert.False(t, filled) + assert.False(t, pending) + assert.False(t, uploadCalled) + }) + + t.Run("WaitsForTheScreenshotBeforeUploading", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/log/m1": + _, _ = w.Write([]byte(`[{"msg":"upload","upload":"ph1","result":"REVIEW"}]`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + filled, pending, err := runner.fillPlaceholders(ctx, "m1", "") + require.NoError(t, err) + + assert.False(t, filled) + assert.True(t, pending) + }) +} + +func TestConformanceOverrideAssertions(t *testing.T) { + testCases := []struct { + name string + assert func(leg ConformanceLeg) error + leg ConformanceLeg + expected string + }{ + { + "ShouldIgnoreTheFirstLegWhenReauthenticationIsExpected", + conformanceAssertReauthentication, + ConformanceLeg{Index: 0, FirstFactor: false}, + "", + }, + { + "ShouldIgnoreTheFirstLegEvenWhenItPrompted", + conformanceAssertReauthentication, + ConformanceLeg{Index: 0, FirstFactor: true}, + "", + }, + { + "ShouldPassWhenTheSecondLegPrompted", + conformanceAssertReauthentication, + ConformanceLeg{Index: 1, FirstFactor: true}, + "", + }, + { + "ShouldFailWhenTheSecondLegDidNotPrompt", + conformanceAssertReauthentication, + ConformanceLeg{Index: 1, FirstFactor: false}, + "expected Authelia to ask for the password again on the second authorization but it did not", + }, + { + "ShouldPassWhenTheSecondLegReauthenticatedOnTheConsentForm", + conformanceAssertReauthentication, + ConformanceLeg{Index: 1, Consent: true, Reauthentication: true}, + "", + }, + { + "ShouldFailWhenTheSecondLegShowedConsentWithoutAskingForThePassword", + conformanceAssertReauthentication, + ConformanceLeg{Index: 1, Consent: true, Reauthentication: false}, + "expected Authelia to ask for the password again on the second authorization but it did not", + }, + { + "ShouldFailWhenTheSecondLegReauthenticatedOnTheConsentFormButShouldNotHave", + conformanceAssertNoReauthentication, + ConformanceLeg{Index: 1, Consent: true, Reauthentication: true}, + "expected Authelia to reuse the existing session on the second authorization but it asked for the password again", + }, + { + "ShouldIgnoreTheFirstLegWhenNoReauthenticationIsExpected", + conformanceAssertNoReauthentication, + ConformanceLeg{Index: 0, FirstFactor: true}, + "", + }, + { + "ShouldPassWhenTheSecondLegDidNotPrompt", + conformanceAssertNoReauthentication, + ConformanceLeg{Index: 1, FirstFactor: false}, + "", + }, + { + "ShouldFailWhenTheSecondLegPrompted", + conformanceAssertNoReauthentication, + ConformanceLeg{Index: 1, FirstFactor: true}, + "expected Authelia to reuse the existing session on the second authorization but it asked for the password again", + }, + { + "ShouldNotBeConfusedByConsentOnTheSecondLeg", + conformanceAssertNoReauthentication, + ConformanceLeg{Index: 1, FirstFactor: false, Consent: true}, + "", + }, + { + "ShouldPassWhenTheLegEndedOnAnAutheliaErrorPage", + conformanceAssertErrorPage, + ConformanceLeg{Index: 0, AutheliaError: true}, + "", + }, + { + "ShouldFailWhenTheLegReachedTheCallbackInsteadOfAnErrorPage", + conformanceAssertErrorPage, + ConformanceLeg{Index: 0}, + "expected Authelia to show an error page rather than redirect to the client but it did not", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.assert(tc.leg) + + if tc.expected == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expected) + } + }) + } +} + +func TestConformanceOverrides_AreWiredToTheirAssertions(t *testing.T) { + for module, expected := range map[string]ConformanceLeg{ + "oidcc-prompt-login": {Index: conformanceReauthenticationLeg, FirstFactor: false}, + "oidcc-max-age-1": {Index: conformanceReauthenticationLeg, FirstFactor: false}, + "oidcc-max-age-10000": {Index: conformanceReauthenticationLeg, FirstFactor: true}, + + "oidcc-ensure-registered-redirect-uri": {Index: 0}, + } { + override, ok := conformanceOverrides[module] + + require.Truef(t, ok, "the '%s' module has no override", module) + require.NotNilf(t, override.Assert, "the '%s' module has no assertion", module) + + assert.Errorf(t, override.Assert(expected), "the '%s' module's assertion cannot fail", module) + } +} + +func TestConformanceRunner_VisitURLsDoesNotRedriveOrRecountAVisitedURL(t *testing.T) { + legs := &conformanceLegs{visited: map[string]bool{"https://conformance.example.com/first": true}, count: 1} + + override := ConformanceOverride{Assert: func(leg ConformanceLeg) error { + return fmt.Errorf("the assertion must not run for an already driven leg, got %+v", leg) + }} + + runner := NewConformanceRunner(nil, nil, "plan1", nil) + + progressed, err := runner.visitURLs(context.Background(), "m1", override, legs, []string{"https://conformance.example.com/first"}) + require.NoError(t, err) + + assert.False(t, progressed) + assert.Equal(t, 1, legs.count) +} + +func TestConformanceOverrides_ClearCookiesMatchesTheModulesThatRequireIt(t *testing.T) { + expected := []string{ + "oidcc-display-page", + "oidcc-display-popup", + "oidcc-login-hint", + "oidcc-prompt-none-not-logged-in", + "oidcc-ui-locales", + } + + var actual []string + + for module, override := range conformanceOverrides { + if override.ClearCookies { + actual = append(actual, module) + } + } + + sort.Strings(actual) + + assert.Equal(t, expected, actual) +} + +func TestConformanceRunner_TreatsAnInterruptedModuleAsFinished(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"m1"}`)) + case "/api/runner/m1/wait-state": + if !strings.Contains(r.URL.Query().Get("states"), conformanceStatusInterrupted) { + time.Sleep(time.Millisecond * 10) + + _, _ = w.Write([]byte(`{"timeout":true}`)) + + return + } + + _, _ = w.Write([]byte(`{"state":"INTERRUPTED"}`)) + case "/api/info/m1": + _, _ = w.Write([]byte(`{"_id":"m1","status":"INTERRUPTED","result":"FAILED"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-max-age-1"}}).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + require.NoError(t, outcomes[0].Err) + + assert.Equal(t, conformanceStatusInterrupted, outcomes[0].Status) + assert.Equal(t, "FAILED", outcomes[0].Result) +} + +func TestConformanceRunner_InteractKeepsDrivingAModuleWhichIsRunningBetweenLegs(t *testing.T) { + var ( + mu sync.Mutex + browserPolls, infos int + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + + switch r.URL.Path { + case "/api/runner/browser/m1": + browserPolls++ + + _, _ = w.Write([]byte(`{"urls":[]}`)) + case "/api/log/m1": + _, _ = w.Write([]byte(`[]`)) + case "/api/info/m1": + infos++ + + if infos == 1 { + _, _ = w.Write([]byte(`{"_id":"m1","status":"RUNNING"}`)) + + return + } + + _, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED"}`)) + case "/api/runner/m1/wait-state": + assert.Equal(t, "WAITING,FINISHED,INTERRUPTED", r.URL.Query().Get("states")) + + _, _ = w.Write([]byte(`{"state":"WAITING"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + runner := NewConformanceRunner(client, &ConformanceBrowser{}, "plan1", nil) + + require.NoError(t, runner.interact(ctx, "m1", ConformanceOverride{})) + + mu.Lock() + defer mu.Unlock() + + assert.Equal(t, 2, browserPolls, "the browser status must be read again once the module is back in WAITING") +} + +func TestConformanceRunner_ReportsTheStatusOfAModuleWhichRanOutOfTime(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"m1"}`)) + case "/api/runner/m1/wait-state": + if strings.Contains(r.URL.Query().Get("states"), conformanceStatusConfigured) { + _, _ = w.Write([]byte(`{"state":"FINISHED"}`)) + + return + } + + time.Sleep(time.Millisecond * 10) + + _, _ = w.Write([]byte(`{"timeout":true}`)) + case "/api/info/m1": + _, _ = w.Write([]byte(`{"_id":"m1","status":"WAITING"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + require.Error(t, outcomes[0].Err) + + assert.Equal(t, conformanceStatusWaiting, outcomes[0].Status) +} + +func TestConformanceLegs_Stalled(t *testing.T) { + t.Run("ShouldNameAutheliasErrorWhenTheLastLegEndedOnItsErrorPage", func(t *testing.T) { + legs := &conformanceLegs{errorURL: "https://login.example.com:8080/consent/completion?error=invalid_request_object&error_description=The+request+parameter+contains+an+invalid+Request+Object.&error_hint=Could+not+be+validated.&error_debug=Expected+typ+JWT."} + + err := legs.stalled("m1") + + var stall *ConformanceErrorPageStallError + + require.ErrorAs(t, err, &stall) + assert.EqualError(t, err, "module 'm1' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: "+ + "error 'invalid_request_object', description 'The request parameter contains an invalid Request Object.', hint 'Could not be validated.', debug 'Expected typ JWT.'") + }) + + t.Run("ShouldFallBackToTheURLWhenTheErrorPageCarriesNoParameters", func(t *testing.T) { + legs := &conformanceLegs{errorURL: "https://login.example.com:8080/"} + + assert.EqualError(t, legs.stalled("m1"), "module 'm1' is waiting for the flow to return to the client, but Authelia ended it on an error page and the module asked for no evidence image of it: https://login.example.com:8080/") + }) + + t.Run("ShouldReportAPlainStallOtherwise", func(t *testing.T) { + legs := &conformanceLegs{} + + err := legs.stalled("m1") + + var stall *ConformanceErrorPageStallError + + assert.False(t, errors.As(err, &stall)) + assert.EqualError(t, err, "module 'm1' stayed in the WAITING state with nothing left to visit or fill") + }) +} + +func TestConformanceOverride_ScreenshotOf(t *testing.T) { + leg := func(index int) ConformanceLeg { + return ConformanceLeg{Index: index, LoginScreenshot: fmt.Sprintf("login-%d", index), ErrorScreenshot: fmt.Sprintf("error-%d", index)} + } + + reauthentication := ConformanceOverride{Screenshot: ConformanceScreenshotReauthentication} + + assert.Empty(t, reauthentication.screenshotOf(leg(0))) + assert.Equal(t, "login-1", reauthentication.screenshotOf(leg(1))) + + errorPage := ConformanceOverride{Screenshot: ConformanceScreenshotErrorPage} + + assert.Equal(t, "error-0", errorPage.screenshotOf(leg(0))) + assert.Empty(t, errorPage.screenshotOf(ConformanceLeg{Index: 0, LoginScreenshot: "login-0"})) + + assert.Empty(t, ConformanceOverride{}.screenshotOf(leg(1)), "a module with no screenshot kind is given none") +} + +func TestConformanceOverrides_ScreenshotEachPlaceholderModule(t *testing.T) { + expected := map[string]ConformanceScreenshot{ + "oidcc-prompt-login": ConformanceScreenshotReauthentication, + "oidcc-max-age-1": ConformanceScreenshotReauthentication, + + "oidcc-ensure-registered-redirect-uri": ConformanceScreenshotErrorPage, + "oidcc-ensure-request-object-with-redirect-uri": ConformanceScreenshotErrorPage, + } + + actual := map[string]ConformanceScreenshot{} + + for module, override := range conformanceOverrides { + if override.Screenshot != ConformanceScreenshotNone { + actual[module] = override.Screenshot + } + } + + assert.Equal(t, expected, actual) +} + +func TestConformanceOverrides_RequestObjectModules(t *testing.T) { + _, ok := conformanceOverrides["oidcc-unsigned-request-object-supported-correctly-or-rejected-as-unsupported"] + assert.False(t, ok) + + override := conformanceOverrides["oidcc-ensure-request-object-with-redirect-uri"] + + assert.Nil(t, override.Assert, "either outcome is accepted by the module") + assert.Equal(t, "error", override.screenshotOf(ConformanceLeg{AutheliaError: true, ErrorScreenshot: "error"})) + assert.Empty(t, override.screenshotOf(ConformanceLeg{}), "a leg which reached the client has no error page to capture") +} + +func TestConformanceLegs_StalledOnAPlaceholderWithoutAScreenshot(t *testing.T) { + legs := &conformanceLegs{pendingPlaceholder: true, errorURL: "https://login.example.com:8080/consent/completion?error=invalid_request"} + + err := legs.stalled("m1") + + var stall *ConformanceErrorPageStallError + + assert.False(t, errors.As(err, &stall), "a module waiting on its placeholder is not stranded for want of one") + assert.EqualError(t, err, "module 'm1' is waiting for an evidence image, but no leg showed the page it asks for") +} + +func TestConformanceRunner_LetsAFailedModuleFinishBeforeMovingOn(t *testing.T) { + testCases := []struct { + name string + settle string + expected string + }{ + {"ShouldWaitForAModuleWhichIsFinishing", `{"state":"FINISHED"}`, "FINISHED"}, + {"ShouldNotWaitOnAModuleWhichIsWaitingOnTheSuite", `{"state":"WAITING"}`, "WAITING"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + mu sync.Mutex + settled []string + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + + switch r.URL.Path { + case "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"m1"}`)) + case "/api/runner/m1/wait-state": + states := r.URL.Query().Get("states") + + if states == "CONFIGURED,WAITING,FINISHED,INTERRUPTED" && len(settled) == 0 && !strings.Contains(r.URL.RawQuery, "settle") { + settled = append(settled, "create:"+states) + + _, _ = w.Write([]byte(`{"state":"WAITING"}`)) + + return + } + + settled = append(settled, "settle:"+states) + + _, _ = w.Write([]byte(tc.settle)) + case "/api/info/m1": + _, _ = w.Write([]byte(`{"_id":"m1","status":"` + tc.expected + `","result":"FAILED"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + defer server.Close() + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + var outcomes []ConformanceOutcome + + for outcome := range NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}).Run(ctx) { + outcomes = append(outcomes, outcome) + } + + require.Len(t, outcomes, 1) + require.Error(t, outcomes[0].Err) + assert.Equal(t, tc.expected, outcomes[0].Status) + + mu.Lock() + defer mu.Unlock() + + assert.Equal(t, []string{"create:CONFIGURED,WAITING,FINISHED,INTERRUPTED", "settle:CONFIGURED,WAITING,FINISHED,INTERRUPTED"}, settled) + }) + } +} diff --git a/internal/suites/conformance_selectors_test.go b/internal/suites/conformance_selectors_test.go new file mode 100644 index 000000000..249077557 --- /dev/null +++ b/internal/suites/conformance_selectors_test.go @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "io/fs" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceSelectorsExistInThePortal(t *testing.T) { + portal := conformancePortalSource(t) + + for _, tc := range []struct { + name string + selector string + }{ + {"FirstFactorStage", conformanceSelectorFirstFactor}, + {"ConsentStage", conformanceSelectorConsent}, + {"ConsentAccept", conformanceSelectorConsentAccept}, + {"ConsentAuthenticate", conformanceSelectorConsentAuthenticate}, + {"ConsentReauthenticationField", conformanceSelectorConsentReauthentication}, + {"ConsentReauthenticationPassword", conformanceSelectorConsentPassword}, + {"Username", conformanceSelectorUsername}, + {"Password", conformanceSelectorPassword}, + {"SignIn", conformanceSelectorSignIn}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, id := range conformanceSelectorIdentifiers(tc.selector) { + assert.Containsf(t, portal, id, "the portal no longer renders anything with the id %q, which %q depends on", id, tc.selector) + } + }) + } + + t.Run("AutheliaErrorToast", func(t *testing.T) { + assert.Contains(t, portal, "notification", "the toast no longer carries the notification class") + assert.Contains(t, portal, "data-[type=error]", "the toast no longer distinguishes an error by data-type") + }) + + t.Run("AutheliaCompletionError", func(t *testing.T) { + assert.Contains(t, portal, `data-testid={"openid-completion-outcome"}`, "the completion view no longer carries its outcome test id") + assert.Contains(t, portal, "data-outcome={outcome}", "the completion view no longer exposes its outcome") + assert.Contains(t, portal, `error ? "error"`, "the completion view no longer names the error outcome 'error'") + }) +} + +func conformanceSelectorIdentifiers(selector string) (ids []string) { + for _, part := range strings.Fields(selector) { + if !strings.HasPrefix(part, "#") { + continue + } + + ids = append(ids, strings.TrimPrefix(part, "#")) + } + + return ids +} + +func conformancePortalSource(t *testing.T) string { + t.Helper() + + builder := &strings.Builder{} + + root, err := os.OpenRoot("../../web/src") + require.NoError(t, err) + + defer root.Close() + + require.NoError(t, fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + if entry.IsDir() || !strings.HasSuffix(path, ".tsx") { + return nil + } + + data, err := root.ReadFile(path) + if err != nil { + return err + } + + builder.Write(data) + + return nil + })) + + require.NotEmpty(t, builder.String(), "no portal sources were read, so this test proves nothing") + + return builder.String() +} diff --git a/internal/suites/conformance_settle_test.go b/internal/suites/conformance_settle_test.go new file mode 100644 index 000000000..482c52a11 --- /dev/null +++ b/internal/suites/conformance_settle_test.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceRunner_AwaitPlaceholderSettledReturnsAsSoonAsTheStatusChanges(t *testing.T) { + client, calls, closeServer := conformanceSettleServer(t, 2) + defer closeServer() + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + start := time.Now() + + status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Millisecond*20, time.Second*2) + require.NoError(t, err) + + assert.Equal(t, "FINISHED", status) + assert.Equal(t, int32(3), calls.Load(), "polls until it changes and then stops") + assert.Less(t, time.Since(start), time.Second, "does not wait out the whole window once it has an answer") +} + +func TestConformanceRunner_AwaitPlaceholderSettledChecksBeforeWaitingAtAll(t *testing.T) { + client, calls, closeServer := conformanceSettleServer(t, 0) + defer closeServer() + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + start := time.Now() + + status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Second*5, time.Second*30) + require.NoError(t, err) + + assert.Equal(t, "FINISHED", status) + assert.Equal(t, int32(1), calls.Load()) + assert.Less(t, time.Since(start), time.Second, "a module which already moved on is not made to wait for the interval") +} + +func TestConformanceRunner_AwaitPlaceholderSettledGivesUpAtTheTimeoutAndReportsWhatItSaw(t *testing.T) { + client, calls, closeServer := conformanceSettleServer(t, 1000) + defer closeServer() + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + status, err := runner.awaitPlaceholderSettled(context.Background(), "m1", time.Millisecond*20, time.Millisecond*100) + require.NoError(t, err) + + assert.Equal(t, conformanceStatusWaiting, status, "reports the status it last saw rather than erroring") + assert.Greater(t, calls.Load(), int32(2), "polled repeatedly across the window") +} + +func TestConformanceRunner_AwaitPlaceholderSettledStopsWhenTheContextExpires(t *testing.T) { + client, _, closeServer := conformanceSettleServer(t, 1000) + defer closeServer() + + runner := NewConformanceRunner(client, nil, "plan1", nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50) + defer cancel() + + _, err := runner.awaitPlaceholderSettled(ctx, "m1", time.Millisecond*20, time.Minute) + require.Error(t, err) + assert.Contains(t, err.Error(), "m1") + assert.Contains(t, err.Error(), "evidence image") +} + +func TestConformanceUploadSettleMatchesTheIntendedCadence(t *testing.T) { + assert.Equal(t, time.Second, conformanceUploadSettleInterval) + assert.Equal(t, time.Second*10, conformanceUploadSettleTimeout) +} + +func conformanceSettleServer(t *testing.T, waits int32) (client *ConformanceClient, calls *atomic.Int32, close func()) { + t.Helper() + + calls = &atomic.Int32{} + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/info/m1", r.URL.Path) + + status := "FINISHED" + + if calls.Add(1) <= waits { + status = "WAITING" + } + + _, _ = fmt.Fprintf(w, `{"_id":"m1","status":%q,"result":"REVIEW"}`, status) + })) + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + return client, calls, server.Close +} diff --git a/internal/suites/conformance_trace.go b/internal/suites/conformance_trace.go new file mode 100644 index 000000000..f36862bc5 --- /dev/null +++ b/internal/suites/conformance_trace.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "fmt" + "time" +) + +// ConformanceTrace records what the suite did for one module, for that module's subtest to log. The runner works on +// its own goroutine, which outlives any one subtest, so it records rather than logging directly. It records nothing +// unless the suite was asked to debug. +type ConformanceTrace struct { + enabled bool + lines []string +} + +// NewConformanceTrace returns a ConformanceTrace which records only when enabled. +func NewConformanceTrace(enabled bool) *ConformanceTrace { + return &ConformanceTrace{enabled: enabled} +} + +// Logf records a line, stamped with the time in UTC so it can be lined up with the container logs. +func (t *ConformanceTrace) Logf(format string, args ...any) { + if t == nil || !t.enabled { + return + } + + t.lines = append(t.lines, time.Now().UTC().Format("15:04:05.000")+" "+fmt.Sprintf(format, args...)) +} + +// Lines returns the recorded lines. +func (t *ConformanceTrace) Lines() []string { + if t == nil { + return nil + } + + return t.lines +} diff --git a/internal/suites/conformance_trace_test.go b/internal/suites/conformance_trace_test.go new file mode 100644 index 000000000..8ed4c912d --- /dev/null +++ b/internal/suites/conformance_trace_test.go @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "net/http" + "net/http/httptest" + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConformanceTrace(t *testing.T) { + t.Run("ShouldRecordNothingUnlessDebugging", func(t *testing.T) { + trace := NewConformanceTrace(false) + + trace.Logf("driving leg %d", 0) + + assert.Empty(t, trace.Lines()) + }) + + t.Run("ShouldRecordEachLineStampedWithTheTime", func(t *testing.T) { + trace := NewConformanceTrace(true) + + trace.Logf("driving leg %d", 0) + trace.Logf("finished leg %d", 0) + + lines := trace.Lines() + + require.Len(t, lines, 2) + assert.Regexp(t, regexp.MustCompile(`^\d{2}:\d{2}:\d{2}\.\d{3} driving leg 0$`), lines[0]) + assert.Regexp(t, regexp.MustCompile(`^\d{2}:\d{2}:\d{2}\.\d{3} finished leg 0$`), lines[1]) + }) + + t.Run("ShouldBeSafeWithoutATrace", func(t *testing.T) { + var trace *ConformanceTrace + + assert.NotPanics(t, func() { trace.Logf("driving leg %d", 0) }) + assert.Empty(t, trace.Lines()) + }) +} + +func TestSuiteDebug(t *testing.T) { + t.Setenv("SUITE_DEBUG", "true") + assert.True(t, suiteDebug()) + + t.Setenv("SUITE_DEBUG", "") + assert.False(t, suiteDebug()) +} + +func TestConformanceRunner_TracesEachModuleOnlyWhenDebugging(t *testing.T) { + for _, debug := range []bool{false, true} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/runner": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"m1"}`)) + case "/api/runner/m1/wait-state": + _, _ = w.Write([]byte(`{"state":"FINISHED"}`)) + case "/api/info/m1": + _, _ = w.Write([]byte(`{"_id":"m1","status":"FINISHED","result":"PASSED"}`)) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + + client, err := NewConformanceClient(server.URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + + runner := NewConformanceRunner(client, nil, "plan1", []ConformancePlanModule{{TestModule: "oidcc-server"}}) + runner.debug = debug + + var outcomes []ConformanceOutcome + + for outcome := range runner.Run(ctx) { + outcomes = append(outcomes, outcome) + } + + cancel() + server.Close() + + require.Len(t, outcomes, 1) + + if debug { + require.NotEmpty(t, outcomes[0].Trace, "a debugging run records what the module did") + assert.Contains(t, outcomes[0].Trace[0], "created as 'm1' in state 'FINISHED'") + } else { + assert.Empty(t, outcomes[0].Trace, "a run which is not debugging records nothing") + } + } +} diff --git a/internal/suites/conformance_users_test.go b/internal/suites/conformance_users_test.go new file mode 100644 index 000000000..d1928a0a6 --- /dev/null +++ b/internal/suites/conformance_users_test.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/authelia/authelia/v4/internal/authentication" +) + +func TestOIDCConformanceUserSuppliesEveryStandardClaim(t *testing.T) { + database := authentication.NewFileUserDatabase("OIDCConformance/users.yml", false, false, nil) + + require.NoError(t, database.Load()) + + user, err := database.GetUserDetails(testUsername) + require.NoError(t, err) + + details := user.ToExtendedUserDetails() + + // scope=profile, less the claims Authelia derives rather than reads: preferred_username comes from the username + // and updated_at is generated. + assert.NotEmpty(t, details.GetDisplayName(), "backs the 'name' claim") + assert.NotEmpty(t, details.GetGivenName(), "backs the 'given_name' claim") + assert.NotEmpty(t, details.GetFamilyName(), "backs the 'family_name' claim") + assert.NotEmpty(t, details.GetMiddleName(), "backs the 'middle_name' claim") + assert.NotEmpty(t, details.GetNickname(), "backs the 'nickname' claim") + assert.NotEmpty(t, details.GetProfile(), "backs the 'profile' claim") + assert.NotEmpty(t, details.GetPicture(), "backs the 'picture' claim") + assert.NotEmpty(t, details.GetWebsite(), "backs the 'website' claim") + assert.NotEmpty(t, details.GetGender(), "backs the 'gender' claim") + assert.NotEmpty(t, details.GetBirthdate(), "backs the 'birthdate' claim") + assert.NotEmpty(t, details.GetZoneInfo(), "backs the 'zoneinfo' claim") + assert.NotEmpty(t, details.GetLocale(), "backs the 'locale' claim") + + // scope=email, less email_verified which Authelia always reports. + assert.NotEmpty(t, details.GetEmails(), "backs the 'email' claim") + + // scope=phone. Authelia emits phone_number_verified only when a number is present, so the number carries both + // claims of this scope. + assert.NotEmpty(t, details.GetPhoneNumberRFC3966(), "backs the 'phone_number' and 'phone_number_verified' claims") + + // scope=address, whose single claim is an object Authelia assembles from these five. + assert.NotEmpty(t, details.GetStreetAddress(), "backs the 'address' claim") + assert.NotEmpty(t, details.GetLocality(), "backs the 'address' claim") + assert.NotEmpty(t, details.GetRegion(), "backs the 'address' claim") + assert.NotEmpty(t, details.GetPostalCode(), "backs the 'address' claim") + assert.NotEmpty(t, details.GetCountry(), "backs the 'address' claim") +} diff --git a/internal/suites/coverage_test.go b/internal/suites/coverage_test.go new file mode 100644 index 000000000..01eb536bb --- /dev/null +++ b/internal/suites/coverage_test.go @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWriteCoverage(t *testing.T) { + dir := filepath.Join(t.TempDir(), ".nyc_output") + + require.NoError(t, writeCoverage(dir, `{"/node/src/app/src/index.tsx":{"path":"/node/src/app/src/index.tsx"}}`)) + require.NoError(t, writeCoverage(dir, `{}`)) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 2, "each document's coverage is written to a file of its own, so the report merges them rather than one replacing another") + + wd, err := os.Getwd() + require.NoError(t, err) + + web := strings.TrimSuffix(wd, "internal/suites") + "web/" + + var found bool + + for _, entry := range entries { + assert.True(t, strings.HasPrefix(entry.Name(), "coverage-") && strings.HasSuffix(entry.Name(), ".json"), entry.Name()) + + data, err := os.ReadFile(filepath.Join(dir, entry.Name())) + require.NoError(t, err) + + if strings.Contains(string(data), "index.tsx") { + found = true + + assert.NotContains(t, string(data), "/node/src/app/", "the paths the frontend was built at must be rewritten to this checkout") + assert.Contains(t, string(data), web+"src/index.tsx") + } + } + + assert.True(t, found) +} + +func TestConformanceCoverageScript(t *testing.T) { + assert.Contains(t, conformanceCoverageScript, "'beforeunload'") + assert.Contains(t, conformanceCoverageScript, "window."+conformanceCoverageBinding+"(") + assert.Contains(t, conformanceCoverageScript, "window.__coverage__") +} + +func TestWriteCoverageReportsADirectoryItCannotCreate(t *testing.T) { + file := filepath.Join(t.TempDir(), "file") + + require.NoError(t, os.WriteFile(file, nil, 0600)) + + assert.Error(t, writeCoverage(filepath.Join(file, ".nyc_output"), `{}`)) +} + +func TestEmitBuildkiteMarker(t *testing.T) { + capture := func(t *testing.T, marker string) string { + t.Helper() + + r, w, err := os.Pipe() + require.NoError(t, err) + + stdout := os.Stdout + os.Stdout = w + + emitBuildkiteMarker(marker) + + os.Stdout = stdout + + require.NoError(t, w.Close()) + + data, err := io.ReadAll(r) + require.NoError(t, err) + + return string(data) + } + + assert.Equal(t, "--- OIDC Conformance Plan: basic\n", capture(t, "--- OIDC Conformance Plan: basic")) + assert.Empty(t, capture(t, ""), "no marker is written outside Buildkite") +} diff --git a/internal/suites/example/compose/nginx/portal/compose.yml b/internal/suites/example/compose/nginx/portal/compose.yml index 4e06e2125..eb0a56d44 100644 --- a/internal/suites/example/compose/nginx/portal/compose.yml +++ b/internal/suites/example/compose/nginx/portal/compose.yml @@ -16,6 +16,7 @@ services: - secure.example.com - login.example.com - duo.example.com + - conformance.example.com # Set the IP to be able to query on port 443 ipv4_address: ${SUITE_SUBNET:-192.168.240}.100 ... diff --git a/internal/suites/example/compose/nginx/portal/nginx.conf b/internal/suites/example/compose/nginx/portal/nginx.conf index 3357b77d6..8776ba053 100644 --- a/internal/suites/example/compose/nginx/portal/nginx.conf +++ b/internal/suites/example/compose/nginx/portal/nginx.conf @@ -450,6 +450,142 @@ http { } } + # OpenID Connect 1.0 conformance suite. This replaces the nginx image the conformance project publishes, whose + # configuration is baked in and only answers on that container's own address. The three listeners below mirror + # that image: 8443 is the ordinary one, 8444 asks for a client certificate so the mutual TLS tests have somewhere + # to land, and 8445 narrows the ciphers to the FAPI 2.0 set. Only 8443 is exercised today, but a suite that grows + # a profile reaching for the other two finds them where it expects rather than failing part way through a plan. + # + # Each listener repeats the proxy body rather than sharing one: nginx would need an include file mounted beside + # this one to factor it out, and the ports differ in the handshake, not in what they forward. The buffer sizes, + # timeouts and forwarded headers are what the published image sets and the suite depends on, so they are + # reproduced here rather than left to the defaults the blocks above share. + server { + listen 8443 ssl; + server_name ~^conformance\.example([0-9])*\.com$; + + # The mutual TLS tests have to arrive on a listener that asks for a client certificate, which this one does + # not, so the suite is told it reached the wrong host and reports that rather than timing out. + ssl_verify_client off; + + resolver 127.0.0.11 ipv6=off; + set $upstream_endpoint http://conformance-server:8080; + + ssl_certificate /pki/public.chain.pem; + ssl_certificate_key /pki/private.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + # The suite sends request objects and authorization requests that overflow the default 1k header buffer. + client_header_buffer_size 32k; + large_client_header_buffers 4 32k; + + location / { + # The suite has requests that block waiting for callbacks for up to 60s, which the default 60s proxy + # timeout races against and turns into sporadic 504s. + proxy_read_timeout 120s; + proxy_send_timeout 120s; + + set $mtls_wrong_host ""; + if ($request_uri ~* "^/test-mtls") { + set $mtls_wrong_host "true"; + } + proxy_set_header X-Test-Mtls-Called-On-Wrong-Host $mtls_wrong_host; + + # Host is deliberately $host rather than $http_host: the suite rebuilds its own base URL from + # X-Forwarded-Host and X-Forwarded-Port, so a port carried in either of the first two would be doubled. + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Uri $request_uri; + proxy_set_header X-Ssl-Cipher $ssl_cipher; + proxy_set_header X-Ssl-Protocol $ssl_protocol; + proxy_set_header X-Ssl-Cert $ssl_client_cert; + proxy_set_header Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme"; + + proxy_pass_request_headers on; + proxy_pass $upstream_endpoint; + } + } + + # Mutual TLS. The client certificate is requested but not checked against a CA: the suite inspects it itself out + # of X-Ssl-Cert, which is why this is optional_no_ca rather than on. + server { + listen 8444 ssl; + server_name ~^conformance\.example([0-9])*\.com$; + + ssl_verify_client optional_no_ca; + + resolver 127.0.0.11 ipv6=off; + set $upstream_endpoint http://conformance-server:8080; + + ssl_certificate /pki/public.chain.pem; + ssl_certificate_key /pki/private.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + client_header_buffer_size 32k; + large_client_header_buffers 4 32k; + + location / { + proxy_read_timeout 120s; + proxy_send_timeout 120s; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Uri $request_uri; + proxy_set_header X-Ssl-Cipher $ssl_cipher; + proxy_set_header X-Ssl-Protocol $ssl_protocol; + proxy_set_header X-Ssl-Cert $ssl_client_cert; + proxy_set_header Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme"; + + proxy_pass_request_headers on; + proxy_pass $upstream_endpoint; + } + } + + # The FAPI 2.0 Final security profile cipher set. + server { + listen 8445 ssl; + server_name ~^conformance\.example([0-9])*\.com$; + + ssl_verify_client optional_no_ca; + ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers on; + + resolver 127.0.0.11 ipv6=off; + set $upstream_endpoint http://conformance-server:8080; + + ssl_certificate /pki/public.chain.pem; + ssl_certificate_key /pki/private.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + client_header_buffer_size 32k; + large_client_header_buffers 4 32k; + + location / { + proxy_read_timeout 120s; + proxy_send_timeout 120s; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Uri $request_uri; + proxy_set_header X-Ssl-Cipher $ssl_cipher; + proxy_set_header X-Ssl-Protocol $ssl_protocol; + proxy_set_header X-Ssl-Cert $ssl_client_cert; + proxy_set_header Forwarded "by=$server_addr;for=$remote_addr;host=$host;proto=$scheme"; + + proxy_pass_request_headers on; + proxy_pass $upstream_endpoint; + } + } + # Fake Web Mail used to receive emails sent by Authelia. server { listen 8080 ssl; diff --git a/internal/suites/hosts.go b/internal/suites/hosts.go index 19764746c..c5d785714 100644 --- a/internal/suites/hosts.go +++ b/internal/suites/hosts.go @@ -57,6 +57,9 @@ func HostEntries() []HostEntry { {Domain: "oidc.example.com", IP: portal}, {Domain: "oidc-public.example.com", IP: portal}, + // OpenID Connect 1.0 conformance suite. + {Domain: "conformance.example.com", IP: portal}, + // For Traefik suite. {Domain: "traefik.example.com", IP: portal}, diff --git a/internal/suites/pngdiff.go b/internal/suites/pngdiff.go index a729d731f..9ea8e9e1c 100644 --- a/internal/suites/pngdiff.go +++ b/internal/suites/pngdiff.go @@ -119,7 +119,7 @@ func AssertVisualSnapshot(t *testing.T, repoRoot, name string, screenshot []byte } if diff.Percentage <= tolerancePercentage { - t.Logf("snapshot %s differs by %d/%d pixels (%.2f%%) — within tolerance (%.2f%%)", + t.Logf("snapshot %s differs by %d/%d pixels (%.2f%%) - within tolerance (%.2f%%)", baselinePath, diff.DifferingPixels, diff.TotalPixels, diff.Percentage, tolerancePercentage) _ = os.Remove(actualPath) diff --git a/internal/suites/suite_activedirectory.go b/internal/suites/suite_activedirectory.go index 4acc7c9a8..3efd4ade9 100644 --- a/internal/suites/suite_activedirectory.go +++ b/internal/suites/suite_activedirectory.go @@ -45,11 +45,12 @@ func init() { GlobalRegistry.Register(activedirectorySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, + OnError: displayAutheliaLogs, TestTimeout: 120 * time.Second, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, - OnError: displayAutheliaLogs, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia against an Active Directory implementation of LDAP, backed by a Samba domain controller.", }) } diff --git a/internal/suites/suite_bypass_all.go b/internal/suites/suite_bypass_all.go index 46374213f..a4d750904 100644 --- a/internal/suites/suite_bypass_all.go +++ b/internal/suites/suite_bypass_all.go @@ -44,12 +44,12 @@ func init() { GlobalRegistry.Register(bypassAllSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 1 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: "This suite has been created to test Authelia with a bypass policy on all resources", }) } diff --git a/internal/suites/suite_caddy.go b/internal/suites/suite_caddy.go index d8bf6c20b..ab12c5189 100644 --- a/internal/suites/suite_caddy.go +++ b/internal/suites/suite_caddy.go @@ -58,11 +58,12 @@ func init() { GlobalRegistry.Register(caddySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 150 * time.Second, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia as an authorization server behind the Caddy reverse proxy.", }) } diff --git a/internal/suites/suite_cli.go b/internal/suites/suite_cli.go index 54b79d674..c828c971d 100644 --- a/internal/suites/suite_cli.go +++ b/internal/suites/suite_cli.go @@ -44,11 +44,13 @@ func init() { GlobalRegistry.Register(cliSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 3 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: `This suite has been created to test the Authelia command line interface, covering the +configuration validation, password hashing, certificate generation, storage and access control subcommands.`, }) } diff --git a/internal/suites/suite_docker.go b/internal/suites/suite_docker.go index adde24f32..cda16ac1f 100644 --- a/internal/suites/suite_docker.go +++ b/internal/suites/suite_docker.go @@ -43,13 +43,12 @@ func init() { GlobalRegistry.Register(dockerSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 1 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, - + TearDownTimeout: 1 * time.Minute, Description: `This suite has been created to test the distributable version of Authelia It's often useful to test this one before the Kube one.`, }) diff --git a/internal/suites/suite_duo_push.go b/internal/suites/suite_duo_push.go index f4fcb8057..c0b6df553 100644 --- a/internal/suites/suite_duo_push.go +++ b/internal/suites/suite_duo_push.go @@ -47,13 +47,12 @@ func init() { GlobalRegistry.Register(duoPushSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 4 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, - + TearDownTimeout: 1 * time.Minute, Description: `This suite has been created to test Authelia against the Duo API for push notifications. It allows a user to validate second factor with a mobile phone.`, diff --git a/internal/suites/suite_envoy.go b/internal/suites/suite_envoy.go index 4ee566da1..aa81ccdf3 100644 --- a/internal/suites/suite_envoy.go +++ b/internal/suites/suite_envoy.go @@ -58,11 +58,12 @@ func init() { GlobalRegistry.Register(envoySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 150 * time.Second, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia as an authorization server behind the Envoy reverse proxy.", }) } diff --git a/internal/suites/suite_haproxy.go b/internal/suites/suite_haproxy.go index e6de8d248..915290ee5 100644 --- a/internal/suites/suite_haproxy.go +++ b/internal/suites/suite_haproxy.go @@ -58,11 +58,12 @@ func init() { GlobalRegistry.Register(haproxySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia as an authorization server behind the HAProxy reverse proxy.", }) } diff --git a/internal/suites/suite_high_availability.go b/internal/suites/suite_high_availability.go index 33cdac126..0ea20357c 100644 --- a/internal/suites/suite_high_availability.go +++ b/internal/suites/suite_high_availability.go @@ -48,12 +48,12 @@ func init() { GlobalRegistry.Register(highAvailabilitySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, + OnError: displayAutheliaLogs, TestTimeout: 6 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, - OnError: displayAutheliaLogs, + TearDownTimeout: 1 * time.Minute, Description: `This suite is made to test Authelia in a *complete* environment, that is, with all components making Authelia highly available.`, }) diff --git a/internal/suites/suite_kubernetes.go b/internal/suites/suite_kubernetes.go index c48871a43..8f434331f 100644 --- a/internal/suites/suite_kubernetes.go +++ b/internal/suites/suite_kubernetes.go @@ -102,10 +102,10 @@ func init() { GlobalRegistry.Register(kubernetesSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 12 * time.Minute, + SetUpTimeout: 2 * time.Minute, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: "This suite has been created to test Authelia in a Kubernetes context and using Traefik as the ingress controller.", }) } diff --git a/internal/suites/suite_ldap.go b/internal/suites/suite_ldap.go index 0b63225f8..3cc6ce821 100644 --- a/internal/suites/suite_ldap.go +++ b/internal/suites/suite_ldap.go @@ -46,11 +46,13 @@ func init() { GlobalRegistry.Register(ldapSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, + OnError: displayAutheliaLogs, TestTimeout: 150 * time.Second, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, - OnError: displayAutheliaLogs, + TearDownTimeout: 1 * time.Minute, + Description: `This suite has been created to test Authelia against an OpenLDAP backend, including the +phpLDAPadmin administration interface.`, }) } diff --git a/internal/suites/suite_mariadb.go b/internal/suites/suite_mariadb.go index 40800d6cd..caf38b018 100644 --- a/internal/suites/suite_mariadb.go +++ b/internal/suites/suite_mariadb.go @@ -46,10 +46,12 @@ func init() { GlobalRegistry.Register(mariadbSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, + TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia with MariaDB as the storage backend and LDAP as the user backend.", }) } diff --git a/internal/suites/suite_multi_cookie_domain.go b/internal/suites/suite_multi_cookie_domain.go index 77a3dbe13..f78499810 100644 --- a/internal/suites/suite_multi_cookie_domain.go +++ b/internal/suites/suite_multi_cookie_domain.go @@ -44,12 +44,12 @@ func init() { GlobalRegistry.Register(multiCookieDomainSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, - OnError: displayAutheliaLogs, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, - TearDown: teardown, + OnError: displayAutheliaLogs, TestTimeout: 4 * time.Minute, - TearDownTimeout: 2 * time.Minute, + TearDown: teardown, + TearDownTimeout: 1 * time.Minute, Description: `This suite is used to test Authelia in a multi cookie domain configuration with in-memory sessions and a local sqlite db stored on disk`, }) diff --git a/internal/suites/suite_mysql.go b/internal/suites/suite_mysql.go index 1c3994bd6..d7d22f7ab 100644 --- a/internal/suites/suite_mysql.go +++ b/internal/suites/suite_mysql.go @@ -46,10 +46,12 @@ func init() { GlobalRegistry.Register(mysqlSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, + TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia with MySQL as the storage backend and LDAP as the user backend.", }) } diff --git a/internal/suites/suite_network_acl.go b/internal/suites/suite_network_acl.go index c48259ad4..0e8e36f5e 100644 --- a/internal/suites/suite_network_acl.go +++ b/internal/suites/suite_network_acl.go @@ -46,12 +46,12 @@ func init() { GlobalRegistry.Register(networkACLSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 1 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: `This suite has been created to test Authelia with basic feature in a non highly-available setup. Authelia basically use an in-memory cache to store user sessions and persist data on disk instead of using a remote database. Also, the user accounts are stored in file-based database.`, diff --git a/internal/suites/suite_oidc.go b/internal/suites/suite_oidc.go index 7a5986742..c5ee8ec02 100644 --- a/internal/suites/suite_oidc.go +++ b/internal/suites/suite_oidc.go @@ -52,11 +52,13 @@ func init() { GlobalRegistry.Register(oidcSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: `This suite has been created to test Authelia as an OpenID Connect 1.0 Provider using a +client application registered against it, with Redis backed sessions.`, }) } diff --git a/internal/suites/suite_oidc_conformance.go b/internal/suites/suite_oidc_conformance.go new file mode 100644 index 000000000..641183c78 --- /dev/null +++ b/internal/suites/suite_oidc_conformance.go @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "time" + + "go.yaml.in/yaml/v4" + + "github.com/authelia/authelia/v4/internal/configuration/schema" + "github.com/authelia/authelia/v4/internal/oidc/conformance" + "github.com/authelia/authelia/v4/internal/random" + "github.com/authelia/authelia/v4/internal/utils" +) + +const ( + oidcConformanceSuiteName = "OIDCConformance" + oidcConformanceBaseURL = "https://conformance.example.com:8443" + oidcConformanceAutheliaURL = "https://login.example.com:8080" + oidcConformancePlansFile = "conformance-plans.json" + oidcConformanceClientsFile = "conformance-clients.yml" + oidcConformanceMongoDBHostEnv = "SUITE_OIDC_CONFORMANCE_MONGODB_HOST" + oidcConformanceMongoDBComposeFile = "OIDCConformance/compose.mongodb.yml" +) + +// OIDCConformancePlanFile is one entry of the plans file exchanged between the setup and test processes. The API plan +// name and the variant are carried as their own fields rather than being read back off the embedded +// [conformance.Plan]: that type tags both of them json:"-" because authelia-gen holds them in memory and sends them as +// query parameters while writing only the plan body to disk, so a round trip through this file would otherwise decode +// them as their zero values and the suite would create every plan nameless and unvarianted. +type OIDCConformancePlanFile struct { + // Name is the builder name of the profile, which is what the suite's per profile test methods look plans up by. + Name string `json:"name"` + + // PlanName is the conformance suite's own name for the plan, sent as the planName query parameter. + PlanName string `json:"plan_name"` + + // Variant is the plan's variant, sent as the variant query parameter. + Variant *conformance.PlanVariant `json:"variant,omitempty"` + + // Plan is the plan body, sent as the request body. + Plan conformance.Plan `json:"plan"` +} + +type oidcConformanceClients struct { + IdentityProviders struct { + OIDC struct { + Clients []schema.IdentityProvidersOpenIDConnectClient `yaml:"clients"` + } `yaml:"oidc"` + } `yaml:"identity_providers"` +} + +func oidcConformanceGenerate() (err error) { + suiteURL, err := url.ParseRequestURI(oidcConformanceBaseURL) + if err != nil { + return err + } + + autheliaURL, err := url.ParseRequestURI(oidcConformanceAutheliaURL) + if err != nil { + return err + } + + version := random.New().StringCustom(8, random.CharSetAlphabeticLower+random.CharSetNumeric) + + clients := &oidcConformanceClients{} + clients.IdentityProviders.OIDC.Clients = []schema.IdentityProvidersOpenIDConnectClient{} + + var plans []OIDCConformancePlanFile + + release := oidcConformanceRelease(oidcConformanceRevision()) + + for _, builder := range conformance.Builders(version, "implicit", "one_factor", "authelia", suiteURL, autheliaURL) { + suite := builder.Build() + + if release != "" { + suite.Plan.Description = builder.Description(release) + } + + clients.IdentityProviders.OIDC.Clients = append(clients.IdentityProviders.OIDC.Clients, suite.Clients...) + plans = append(plans, OIDCConformancePlanFile{Name: builder.Name, PlanName: suite.Plan.Name, Variant: suite.Plan.Variant, Plan: suite.Plan}) + } + + if err = oidcConformanceWriteYAML(SuiteTmpPath(oidcConformanceClientsFile), clients); err != nil { + return err + } + + return oidcConformanceWriteJSON(SuiteTmpPath(oidcConformancePlansFile), plans) +} + +func oidcConformanceRevision() (commit, tag string) { + commit, _, _ = utils.RunCommandAndReturnOutput("git rev-parse HEAD") + tag, _, _ = utils.RunCommandAndReturnOutput("git describe --tags --exact-match HEAD") + + return commit, tag +} + +func oidcConformanceRelease(commit, tag string) string { + switch { + case commit == "": + return "" + case tag == "": + return "commit " + commit + default: + return fmt.Sprintf("%s (commit %s)", tag, commit) + } +} + +func oidcConformanceComposeFiles(mongodbHost string) (files []string) { + files = []string{ + "internal/suites/compose.yml", + "internal/suites/OIDCConformance/compose.yml", + "internal/suites/example/compose/authelia/compose.backend.{}.yml", + "internal/suites/example/compose/authelia/compose.frontend.{}.yml", + "internal/suites/example/compose/nginx/backend/compose.yml", + "internal/suites/example/compose/nginx/portal/compose.yml", + "internal/suites/example/compose/smtp/compose.yml", + "internal/suites/example/compose/redis/compose.yml", + "internal/suites/example/compose/postgres/compose.yml", + } + + if mongodbHost == "" { + files = append(files, "internal/suites/"+oidcConformanceMongoDBComposeFile) + } + + return files +} + +func oidcConformanceLogServices(mongodbHost string) (services []string) { + services = []string{"authelia-backend", "authelia-frontend", "postgres"} + + if mongodbHost == "" { + services = append(services, "conformance-mongodb") + } + + return append(services, "conformance-server") +} + +func oidcConformanceWriteYAML(path string, value any) (err error) { + f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return err + } + + defer func() { + if cerr := f.Close(); cerr != nil && err == nil { + err = fmt.Errorf("error closing '%s': %w", path, cerr) + } + }() + + encoder := yaml.NewEncoder(f) + encoder.SetIndent(2) + + defer func() { + if cerr := encoder.Close(); cerr != nil && err == nil { + err = fmt.Errorf("error flushing '%s': %w", path, cerr) + } + }() + + return encoder.Encode(value) +} + +func oidcConformanceWriteJSON(path string, value any) (err error) { + f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return err + } + + defer func() { + if cerr := f.Close(); cerr != nil && err == nil { + err = fmt.Errorf("error closing '%s': %w", path, cerr) + } + }() + + return json.NewEncoder(f).Encode(value) +} + +func oidcConformanceReadPlans() (plans []OIDCConformancePlanFile, err error) { + data, err := os.ReadFile(SuiteTmpPath(oidcConformancePlansFile)) + if err != nil { + return nil, fmt.Errorf("error reading the generated conformance plans, was the suite set up?: %w", err) + } + + if err = json.Unmarshal(data, &plans); err != nil { + return nil, err + } + + return plans, nil +} + +func init() { + mongodbHost := os.Getenv(oidcConformanceMongoDBHostEnv) + + dockerEnvironment := NewDockerEnvironment(oidcConformanceComposeFiles(mongodbHost)) + + setup := func(suitePath string) (err error) { + if err = oidcConformanceGenerate(); err != nil { + return err + } + + if err = dockerEnvironment.Up(); err != nil { + return err + } + + if err = waitUntilAutheliaIsReady(dockerEnvironment, oidcConformanceSuiteName); err != nil { + return err + } + + client, err := NewConformanceClient(oidcConformanceBaseURL) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + if err = client.WaitReady(ctx); err != nil { + return err + } + + return updateDevEnvFileForDomain(BaseDomain, dockerEnvironment) + } + + displayLogs := func() error { + return dockerEnvironment.PrintLogs(oidcConformanceLogServices(mongodbHost)...) + } + + teardown := func(suitePath string) error { + return dockerEnvironment.Down() + } + + GlobalRegistry.Register(oidcConformanceSuiteName, Suite{ + SetUp: setup, + SetUpTimeout: 2 * time.Minute, + OnSetupTimeout: displayLogs, + OnError: displayLogs, + TestTimeout: 8 * time.Minute, + TearDown: teardown, + TearDownTimeout: 1 * time.Minute, + Description: "This suite runs the OpenID Foundation conformance suite against Authelia for every profile Authelia is OpenID Certified for.", + }) +} diff --git a/internal/suites/suite_oidc_conformance_test.go b/internal/suites/suite_oidc_conformance_test.go new file mode 100644 index 000000000..3bf146192 --- /dev/null +++ b/internal/suites/suite_oidc_conformance_test.go @@ -0,0 +1,448 @@ +// SPDX-FileCopyrightText: 2026 Authelia +// +// SPDX-License-Identifier: Apache-2.0 + +package suites + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.yaml.in/yaml/v4" + + "github.com/authelia/authelia/v4/internal/oidc/conformance" + "github.com/authelia/authelia/v4/internal/utils" +) + +var ( + conformanceAcceptedResults = []string{"PASSED", "REVIEW", "SKIPPED"} + conformanceAcceptedWarnings = map[string]string{ + // ValidateIdTokenACRClaimAgainstAcrValuesRequest (OIDCC-3.1.2.1, OIDCC-15.1). + "oidcc-ensure-request-with-acr-values-succeeds": "Authelia does not select an acr from acr_values, which is a voluntary request, so the ID Token carries the fallback acr '0' rather than one of the requested values", + } +) + +func conformanceResultAccepted(module, result string) bool { + if utils.IsStringInSlice(result, conformanceAcceptedResults) { + return true + } + + _, ok := conformanceAcceptedWarnings[module] + + return ok && result == "WARNING" +} + +// conformanceOutcomeAccepted reports whether a module passes: the suite drove it without error, it was not interrupted, +// and its result is accepted. A module is interrupted when it stops before running to completion, so its result says +// nothing about the conformance of what it did not get to check. +func conformanceOutcomeAccepted(outcome ConformanceOutcome) bool { + return outcome.Err == nil && outcome.Status != conformanceStatusInterrupted && conformanceResultAccepted(outcome.Module, outcome.Result) +} + +const ( + conformanceDiagnosticTimeout = time.Second * 30 + conformancePlanTimeout = time.Minute * 75 + conformanceDrainTimeout = time.Second * 45 + conformanceArchiveDir = "oidc-conformance-plans" +) + +// OIDCConformanceSuite runs the OpenID Foundation conformance suite against Authelia, one test per certified profile. +type OIDCConformanceSuite struct { + *RodSuite + + client *ConformanceClient + cancel context.CancelFunc + browsers []*ConformanceBrowser + outcomes map[string]<-chan ConformanceOutcome + planIDs map[string]string +} + +// NewOIDCConformanceSuite returns a new *OIDCConformanceSuite. +func NewOIDCConformanceSuite() *OIDCConformanceSuite { + return &OIDCConformanceSuite{ + RodSuite: NewRodSuite(oidcConformanceSuiteName), + outcomes: map[string]<-chan ConformanceOutcome{}, + planIDs: map[string]string{}, + } +} + +// SetupSuite creates every plan and starts a runner per plan. +func (s *OIDCConformanceSuite) SetupSuite() { + s.BaseSuite.SetupSuite() + + complete := false + + defer func() { + if !complete { + s.release() + } + }() + + plans, err := oidcConformanceReadPlans() + s.Require().NoError(err) + + s.client, err = NewConformanceClient(oidcConformanceBaseURL) + s.Require().NoError(err) + + session, err := NewRodSession() + s.Require().NoError(err) + + s.RodSession = session + + ctx, cancel := context.WithTimeout(context.Background(), conformancePlanTimeout) + + s.cancel = cancel + + for _, plan := range plans { + plan := plan + + created, err := s.client.CreatePlan(ctx, plan.PlanName, plan.Variant, &plan.Plan) + s.Require().NoErrorf(err, "error creating the '%s' plan", plan.Name) + + s.planIDs[plan.Name] = created.ID + + browser, err := NewConformanceBrowser(s.RodSession) + s.Require().NoError(err) + + s.browsers = append(s.browsers, browser) + + runner := NewConformanceRunner(s.client, browser, created.ID, created.Modules) + runner.debug = suiteDebug() + + s.outcomes[plan.Name] = runner.Run(ctx) + } + + complete = true +} + +func (s *OIDCConformanceSuite) release() { + if s.cancel != nil { + s.cancel() + } + + s.drainRunners() + + for _, browser := range s.browsers { + browser.Close() + } + + if s.RodSession != nil { + if err := s.Stop(); err != nil { + s.T().Logf("error stopping rod session: %v", err) + } + } +} + +func (s *OIDCConformanceSuite) drainRunners() { + for name, outcomes := range s.outcomes { + timer := time.NewTimer(conformanceDrainTimeout) + + for drained := false; !drained; { + select { + case _, ok := <-outcomes: + if !ok { + drained = true + } + case <-timer.C: + s.T().Logf("timed out after %s waiting for the '%s' plan's runner to finish", conformanceDrainTimeout, name) + + drained = true + } + } + + timer.Stop() + } +} + +// TearDownSuite exports each plan's log for the CI artifacts, then releases the runners, browser contexts and rod +// session via release. +func (s *OIDCConformanceSuite) TearDownSuite() { + emitBuildkiteMarker(buildkiteGroup("OIDC Conformance Finalization: teardown")) + + if s.client != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + + for name, id := range s.planIDs { + path := conformanceArchivePath(name) + + if err := s.client.ExportPlanHTML(ctx, id, path); err != nil { + s.T().Logf("Error exporting the '%s' plan log: %v", name, err) + + continue + } + + s.T().Logf("Exported the '%s' plan log to '%s'", name, path) + } + } + + s.release() +} + +func (s *OIDCConformanceSuite) assertPlan(name string) { + outcomes, ok := s.outcomes[name] + s.Require().Truef(ok, "the '%s' plan was not created", name) + + emitBuildkiteMarker(buildkiteGroup("OIDC Conformance Plan: " + name)) + + count, accepted, failed, skipped, expanded := 0, 0, 0, 0, false + + for outcome := range outcomes { + outcome := outcome + count++ + + s.Run(outcome.Name, func() { + t := s.T() + + if len(outcome.Trace) != 0 { + t.Log("Trace:\n" + strings.Join(outcome.Trace, "\n")) + } + + if outcome.Skipped { + skipped++ + + t.Skipf("%s cannot run unattended: %s", outcome.Module, outcome.Reason) + + return + } + + if !conformanceOutcomeAccepted(outcome) { + failed++ + + s.reportFailure(t, name, outcome) + + if !expanded { + emitBuildkiteMarker(buildkiteExpandGroup()) + + expanded = true + } + } + + require.NoError(t, outcome.Err) + require.NotEqualf(t, conformanceStatusInterrupted, outcome.Status, + "module '%s' was interrupted with the result '%s' and the status '%s'", outcome.Module, outcome.Result, outcome.Status) + require.Truef(t, conformanceResultAccepted(outcome.Module, outcome.Result), + "module '%s' finished with the result '%s' and the status '%s'", outcome.Module, outcome.Result, outcome.Status) + + accepted++ + + if outcome.Result == "WARNING" { + s.reportAcceptedWarning(outcome) + } + }) + } + + s.T().Log(conformancePlanSummary(name, accepted, failed, skipped)) + + s.Require().NotZerof(count, "the '%s' plan produced no modules", name) +} + +func (s *OIDCConformanceSuite) reportAcceptedWarning(outcome ConformanceOutcome) { + reason := conformanceAcceptedWarnings[outcome.Module] + + if s.client == nil || outcome.ID == "" { + s.T().Logf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s.", outcome.Module, outcome.Status, reason) + + return + } + + ctx, cancel := context.WithTimeout(context.Background(), conformanceDiagnosticTimeout) + defer cancel() + + entries, err := s.client.Log(ctx, outcome.ID) + if err != nil { + s.T().Logf("Module '%s' finished with the accepted result 'WARNING' and the status '%s': %s. Its log could not be read: %v", outcome.Module, outcome.Status, reason, err) + + return + } + + s.T().Log(ConformanceAcceptedWarningReport(outcome.Module, outcome.Status, reason, entries)) +} + +func (s *OIDCConformanceSuite) reportFailure(t *testing.T, plan string, outcome ConformanceOutcome) { + t.Logf("Module '%s' of the '%s' plan finished with the result '%s' and the status '%s'.", outcome.Module, plan, outcome.Result, outcome.Status) + + if outcome.Err != nil { + t.Logf("Suite error: %v", outcome.Err) + } + + if outcome.ID == "" { + t.Log("The module was never created, so the conformance server has no log for it.") + + return + } + + if s.client == nil { + return + } + + t.Logf("Module id %s, in %s", outcome.ID, conformanceArchiveReference(plan, outcome.LogURL)) + + ctx, cancel := context.WithTimeout(context.Background(), conformanceDiagnosticTimeout) + defer cancel() + + entries, err := s.client.Log(ctx, outcome.ID) + if err != nil { + t.Logf("The module's log could not be read for this report: %v", err) + + return + } + + var stall *ConformanceErrorPageStallError + + switch detail := ConformanceDiagnostics(entries); { + case detail != "": + t.Logf("Conformance log entries that did not pass:%s", detail) + case errors.As(outcome.Err, &stall): + t.Logf("The module logged no failing entries because it is still waiting for the callback Authelia's error page never sent.") + default: + t.Logf("The module logged no failing entries, which points at the suite driving it rather than at the provider.") + } +} + +// TestConfig runs the Config OP certification profile. +func (s *OIDCConformanceSuite) TestConfig() { + s.assertPlan(conformance.NameConfig) +} + +// TestBasic runs the Basic OP certification profile. +func (s *OIDCConformanceSuite) TestBasic() { + s.assertPlan(conformance.NameBasic) +} + +// TestBasicFormPost runs the Basic OP certification profile with the form post response mode. +func (s *OIDCConformanceSuite) TestBasicFormPost() { + s.assertPlan(conformance.NameBasicFormPost) +} + +// TestHybrid runs the Hybrid OP certification profile. +func (s *OIDCConformanceSuite) TestHybrid() { + s.assertPlan(conformance.NameHybrid) +} + +// TestHybridFormPost runs the Hybrid OP certification profile with the form post response mode. +func (s *OIDCConformanceSuite) TestHybridFormPost() { + s.assertPlan(conformance.NameHybridFormPost) +} + +// TestImplicit runs the Implicit OP certification profile. +func (s *OIDCConformanceSuite) TestImplicit() { + s.assertPlan(conformance.NameImplicit) +} + +// TestImplicitFormPost runs the Implicit OP certification profile with the form post response mode. +func (s *OIDCConformanceSuite) TestImplicitFormPost() { + s.assertPlan(conformance.NameImplicitFormPost) +} + +func TestOIDCConformanceSuite(t *testing.T) { + if testing.Short() { + t.Skip("skipping suite test in short mode") + } + + suite.Run(t, NewOIDCConformanceSuite()) +} + +func TestOIDCConformanceGenerateRoundTrip(t *testing.T) { + t.Setenv("SUITE_TMP_PATH", t.TempDir()) + + require.NoError(t, oidcConformanceGenerate()) + + plans, err := oidcConformanceReadPlans() + require.NoError(t, err) + + expected := map[string]struct { + plan string + variant bool + }{ + conformance.NameConfig: {"oidcc-config-certification-test-plan", false}, + conformance.NameBasic: {"oidcc-basic-certification-test-plan", true}, + conformance.NameBasicFormPost: {"oidcc-formpost-basic-certification-test-plan", true}, + conformance.NameHybrid: {"oidcc-hybrid-certification-test-plan", true}, + conformance.NameHybridFormPost: {"oidcc-formpost-hybrid-certification-test-plan", true}, + conformance.NameImplicit: {"oidcc-implicit-certification-test-plan", true}, + conformance.NameImplicitFormPost: {"oidcc-formpost-implicit-certification-test-plan", true}, + } + + require.Len(t, plans, len(expected)) + + data, err := os.ReadFile(SuiteTmpPath(oidcConformanceClientsFile)) + require.NoError(t, err) + + clients := &oidcConformanceClients{} + require.NoError(t, yaml.Unmarshal(data, clients)) + + ids := map[string]bool{} + + for _, client := range clients.IdentityProviders.OIDC.Clients { + ids[client.ID] = true + } + + require.NotEmpty(t, ids) + + for _, plan := range plans { + want, ok := expected[plan.Name] + + require.Truef(t, ok, "unexpected profile '%s'", plan.Name) + + assert.Equalf(t, want.plan, plan.PlanName, "the '%s' profile lost its conformance plan name", plan.Name) + assert.NotEmptyf(t, plan.Plan.Alias, "the '%s' profile lost its alias", plan.Name) + assert.NotEmptyf(t, plan.Plan.Server.DiscoveryURL, "the '%s' profile lost its discovery url", plan.Name) + + if want.variant { + require.NotNilf(t, plan.Variant, "the '%s' profile lost its variant", plan.Name) + assert.Equal(t, "discovery", plan.Variant.ServerMetadata) + assert.Equal(t, "static_client", plan.Variant.ClientRegistration) + } else { + assert.Nilf(t, plan.Variant, "the '%s' profile gained a variant", plan.Name) + } + + if plan.Name == conformance.NameConfig { + assert.Nil(t, plan.Plan.Client) + + continue + } + + require.NotNilf(t, plan.Plan.Client, "the '%s' profile lost its client", plan.Name) + assert.NotEmpty(t, plan.Plan.Client.Secret) + assert.Truef(t, ids[plan.Plan.Client.ID], "the '%s' profile's client '%s' is not in the generated configuration", plan.Name, plan.Plan.Client.ID) + require.NotNilf(t, plan.Plan.ClientAlternate, "the '%s' profile lost its alternate client", plan.Name) + require.NotNilf(t, plan.Plan.ClientSecretPost, "the '%s' profile lost its secret post client", plan.Name) + + assert.Truef(t, ids[plan.Plan.ClientAlternate.ID], "the '%s' profile's alternate client '%s' is not in the generated configuration", plan.Name, plan.Plan.ClientAlternate.ID) + assert.Truef(t, ids[plan.Plan.ClientSecretPost.ID], "the '%s' profile's secret post client '%s' is not in the generated configuration", plan.Name, plan.Plan.ClientSecretPost.ID) + } +} + +func conformanceArchiveReference(plan, liveURL string) string { + if build, job := os.Getenv("BUILDKITE_BUILD_URL"), os.Getenv("BUILDKITE_JOB_ID"); os.Getenv("CI") == t && build != "" && job != "" { + return fmt.Sprintf("%s/%s in the job's artifacts: %s/waterfall?jid=%s&tab=artifacts", conformanceArchiveDir, conformanceArchiveName(plan), build, job) + } + + return fmt.Sprintf("%s, or the live log while the suite is up: %s", conformanceArchivePath(plan), liveURL) +} + +func conformancePlanSummary(plan string, accepted, failed, skipped int) string { + return fmt.Sprintf("Plan '%s': %d accepted, %d failed, %d skipped", plan, accepted, failed, skipped) +} + +func conformanceArchiveName(plan string) string { + return fmt.Sprintf("conformance-%s.zip", plan) +} + +func conformanceArchivePath(plan string) string { + if os.Getenv("CI") == t { + return filepath.Join("../..", conformanceArchiveDir, conformanceArchiveName(plan)) + } + + return filepath.Join(os.TempDir(), "authelia-suites-"+conformanceArchiveDir, composeProjectName(), conformanceArchiveName(plan)) +} diff --git a/internal/suites/suite_oidc_traefik.go b/internal/suites/suite_oidc_traefik.go index 39f91e6ac..7671a36f2 100644 --- a/internal/suites/suite_oidc_traefik.go +++ b/internal/suites/suite_oidc_traefik.go @@ -72,11 +72,13 @@ func init() { GlobalRegistry.Register(oidcTraefikSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: `This suite has been created to test Authelia as an OpenID Connect 1.0 Provider behind the +Traefik reverse proxy, with Redis backed sessions.`, }) } diff --git a/internal/suites/suite_one_factor_only.go b/internal/suites/suite_one_factor_only.go index 52dd8ff89..dea9bc17f 100644 --- a/internal/suites/suite_one_factor_only.go +++ b/internal/suites/suite_one_factor_only.go @@ -42,12 +42,12 @@ func init() { GlobalRegistry.Register(oneFactorOnlySuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 1 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: "This suite has been created to test Authelia in a one-factor only configuration", }) } diff --git a/internal/suites/suite_pam.go b/internal/suites/suite_pam.go index cfbc56d39..3f852b214 100644 --- a/internal/suites/suite_pam.go +++ b/internal/suites/suite_pam.go @@ -48,12 +48,12 @@ func init() { GlobalRegistry.Register(pamSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 3 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: "PAM module integration tests for SSH authentication via pam_authelia", }) } diff --git a/internal/suites/suite_pathprefix.go b/internal/suites/suite_pathprefix.go index 02ad70ac3..7836ea7db 100644 --- a/internal/suites/suite_pathprefix.go +++ b/internal/suites/suite_pathprefix.go @@ -64,11 +64,12 @@ func init() { GlobalRegistry.Register(pathPrefixSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 150 * time.Second, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia served from a path prefix rather than the root of a domain.", }) } diff --git a/internal/suites/suite_postgres.go b/internal/suites/suite_postgres.go index 22174ffd3..c5910c2a0 100644 --- a/internal/suites/suite_postgres.go +++ b/internal/suites/suite_postgres.go @@ -46,10 +46,12 @@ func init() { GlobalRegistry.Register(postgresSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, + TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia with PostgreSQL as the storage backend and LDAP as the user backend.", }) } diff --git a/internal/suites/suite_short_timeouts.go b/internal/suites/suite_short_timeouts.go index b7bcf2822..fe4fb221a 100644 --- a/internal/suites/suite_short_timeouts.go +++ b/internal/suites/suite_short_timeouts.go @@ -43,12 +43,12 @@ func init() { GlobalRegistry.Register(shortTimeoutsSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 3 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, Description: `This suite has been created to configure Authelia with short timeouts for sessions expiration in order to test the inactivity feature and the remember me feature.`, }) diff --git a/internal/suites/suite_standalone.go b/internal/suites/suite_standalone.go index 13d717460..811146a8c 100644 --- a/internal/suites/suite_standalone.go +++ b/internal/suites/suite_standalone.go @@ -51,12 +51,12 @@ func init() { GlobalRegistry.Register(standaloneSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, - OnError: displayAutheliaLogs, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, - TearDown: teardown, + OnError: displayAutheliaLogs, TestTimeout: 5 * time.Minute, - TearDownTimeout: 2 * time.Minute, + TearDown: teardown, + TearDownTimeout: 1 * time.Minute, Description: `This suite is used to test Authelia in a standalone configuration with in-memory sessions and a local sqlite db stored on disk`, }) diff --git a/internal/suites/suite_traefik2.go b/internal/suites/suite_traefik2.go index bbbbe15c1..1213d29eb 100644 --- a/internal/suites/suite_traefik2.go +++ b/internal/suites/suite_traefik2.go @@ -66,11 +66,12 @@ func init() { GlobalRegistry.Register(traefik2SuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia as an authorization server behind version 2 of the Traefik reverse proxy.", }) } diff --git a/internal/suites/suite_traefik3.go b/internal/suites/suite_traefik3.go index 979d5eed9..7ce2bd112 100644 --- a/internal/suites/suite_traefik3.go +++ b/internal/suites/suite_traefik3.go @@ -66,11 +66,12 @@ func init() { GlobalRegistry.Register(traefik3SuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, OnError: displayAutheliaLogs, TestTimeout: 2 * time.Minute, TearDown: teardown, - TearDownTimeout: 2 * time.Minute, + TearDownTimeout: 1 * time.Minute, + Description: "This suite has been created to test Authelia as an authorization server behind version 3 of the Traefik reverse proxy.", }) } diff --git a/internal/suites/suite_two_factor.go b/internal/suites/suite_two_factor.go index ffaca9ffd..98458fe47 100644 --- a/internal/suites/suite_two_factor.go +++ b/internal/suites/suite_two_factor.go @@ -52,12 +52,12 @@ func init() { GlobalRegistry.Register(twoFactorSuiteName, Suite{ SetUp: setup, - SetUpTimeout: 5 * time.Minute, - OnError: displayAutheliaLogs, + SetUpTimeout: 2 * time.Minute, OnSetupTimeout: displayAutheliaLogs, - TearDown: teardown, + OnError: displayAutheliaLogs, TestTimeout: 4 * time.Minute, - TearDownTimeout: 2 * time.Minute, + TearDown: teardown, + TearDownTimeout: 1 * time.Minute, Description: `This suite is used to test Authelia in a two factor configuration with in-memory sessions and a local sqlite db stored on disk`, }) diff --git a/internal/suites/utils.go b/internal/suites/utils.go index 2fdd48a94..0e4240a11 100644 --- a/internal/suites/utils.go +++ b/internal/suites/utils.go @@ -22,7 +22,6 @@ import ( "time" "github.com/go-rod/rod" - "github.com/go-rod/rod/lib/input" "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -95,21 +94,10 @@ const diagnosticsConsole = `() => JSON.stringify({ entries: window.__diagnostics__ || [], }, null, 2)` +const coverageDir = "../../web/.nyc_output" + var errPageNotCreated = errors.New("the page was not created") -// StringToKeys returns the input.Key values which represent the given string. -func StringToKeys(value string) []input.Key { - n := len(value) - - keys := make([]input.Key, n) - - for i := 0; i < n; i++ { - keys[i] = input.Key(value[i]) - } - - return keys -} - // ValidateBrowserPath validates the appropriate chromium browser path. func ValidateBrowserPath(path string) (browserPath string, err error) { var info os.FileInfo @@ -161,8 +149,6 @@ func (rs *RodSession) collectCoverage(page *rod.Page) { return } - coverageDir := "../../web/.nyc_output" - resp, err := page.Eval("() => JSON.stringify(window.__coverage__)") if err != nil { log.Errorf("Error collecting coverage: %v", err) @@ -175,15 +161,54 @@ func (rs *RodSession) collectCoverage(page *rod.Page) { _ = os.MkdirAll(coverageDir, 0775) if coverageData != "" { - if err = os.WriteFile(fmt.Sprintf("%s/coverage-%s.json", coverageDir, uuid.New().String()), []byte(coverageData), 0664); err != nil { //nolint:gosec + if err = writeCoverage(coverageDir, coverageData); err != nil { log.Errorf("Error writing coverage: %v", err) - - return } + } +} - if err = filepath.Walk("../../web/.nyc_output", fixCoveragePath); err != nil { - log.Errorf("Error rewriting coverage paths: %v", err) - } +func writeCoverage(dir, data string) (err error) { + if err = os.MkdirAll(dir, 0775); err != nil { + return err + } + + wd, err := os.Getwd() + if err != nil { + return err + } + + data = strings.ReplaceAll(data, "/node/src/app/", strings.TrimSuffix(wd, "internal/suites")+"web/") + + return os.WriteFile(filepath.Join(dir, fmt.Sprintf("coverage-%s.json", uuid.New().String())), []byte(data), 0664) //nolint:gosec +} + +func suiteDebug() bool { + return os.Getenv("SUITE_DEBUG") == t +} + +func buildkiteGroups() bool { + return os.Getenv("BUILDKITE") == t && suiteDebug() +} + +func buildkiteGroup(title string) string { + if !buildkiteGroups() { + return "" + } + + return "--- " + title +} + +func buildkiteExpandGroup() string { + if !buildkiteGroups() { + return "" + } + + return "^^^ +++" +} + +func emitBuildkiteMarker(marker string) { + if marker != "" { + fmt.Println(marker) //nolint:forbidigo } } @@ -559,39 +584,6 @@ func (s *RodSuite) VerifyPageElementAttributeValue(t *testing.T, page *rod.Page, assert.Equal(t, value, *attr) } -func fixCoveragePath(path string, file os.FileInfo, err error) error { - if err != nil { - return err - } - - if file.IsDir() { - return nil - } - - coverage, err := filepath.Match("*.json", file.Name()) - if err != nil { - return err - } - - if coverage { - var data []byte - - if data, err = os.ReadFile(path); err != nil { //nolint:gosec - return err - } - - wd, _ := os.Getwd() - ciPath := strings.TrimSuffix(wd, "internal/suites") - content := strings.ReplaceAll(string(data), "/node/src/app/", ciPath+"web/") - - if err = os.WriteFile(path, []byte(content), 0); err != nil { //nolint:gosec - return err - } - } - - return nil -} - func getDomainEnvInfo(domain string) (info map[string]string, err error) { info = make(map[string]string)