ci: add scorecard linter for file based checks (#13178)

Scorecard findings only showed up in the weekly workflow run against
`master`, well after the change had merged. The `lint.sh scorecard`
subcommand runs the four file based checks up front, so lefthook catches
them at commit time and reviewdog annotates them on the pull request.

The checks run against a `git checkout-index` export rather than the
working tree. Scanning the tree directly walks every `node_modules` and
does not finish; the export takes about 15 seconds. Only checks that
lost points are reported, as `scorecard` also warns about things it does
not penalize, such as the job level `contents` write the contributors
workflow needs to push the card.

`scorecard` and `jq` are now required tools. CI agents need `scorecard`
installed for the reviewdog runner.

---------

Signed-off-by: Amir Zarrinkafsh <3339418+nightah@users.noreply.github.com>

.buildkite/steps/lint.sh

@@ -14,10 +14,19 @@
 #                            are forwarded to shellcheck.
 #   lint.sh typos ...        Run typos, reporting an abnormal exit as a finding
 #                            so reviewdog does not discard the reason.
+#   lint.sh scorecard        Run the file based OpenSSF Scorecard checks against
+#                            the git index, reporting each finding as
+#                            path:line:col: [check] message.
 #   lint.sh -flag ...        Anything else is forwarded to reviewdog.
 
 set -uo pipefail
 
+# The file based scorecard checks that describe something fixable in a tracked file.
+SCORECARD_CHECKS='Dangerous-Workflow,Dependency-Update-Tool,Pinned-Dependencies,Token-Permissions'
+
+# A finding describing the project rather than a file is reported against the workflow scorecard runs in.
+SCORECARD_ANCHOR='.github/workflows/scorecard.yml'
+
 discover_shell_files() {
   # A file is considered a shell script if any of:
   #   - its path matches *.sh
@@ -86,6 +95,52 @@ run_typos() {
   return ${rc}
 }
 
+run_scorecard() {
+  local dir out err rc findings
+  dir=$(mktemp -d)
+  out=$(mktemp)
+  err=$(mktemp)
+
+  # scorecard --local walks every file on disk, node_modules included, which takes minutes. Exporting the index
+  # scans only tracked content, and under lefthook exactly what is about to be committed.
+  git checkout-index -a --prefix="${dir}/"
+  (cd "${dir}" && scorecard --local=. --checks="${SCORECARD_CHECKS}" --format=json --show-details) >"${out}" 2>"${err}"
+  rc=$?
+
+  # scorecard exits 0 when it finds problems, so a finding is turned into a failure here. An abnormal
+  # exit is re-emitted as a finding for the same reason as typos above.
+  if [ ${rc} -ne 0 ]; then
+    if [ -s "${err}" ]; then
+      sed -e "1s|^|.reviewdog.yml:1:1: scorecard failed (exit ${rc}): |" "${err}"
+    else
+      echo ".reviewdog.yml:1:1: scorecard failed (exit ${rc}) without writing a reason"
+    fi
+  else
+    # Only checks that lost points are reported. scorecard also warns about findings it does not penalize, such as
+    # the job level 'contents' write the contributors workflow needs to push the card, and failing on those would
+    # block work the scored check considers fine. A score below zero is scorecard declaring the check inconclusive.
+    #
+    # Probe outcomes are not usable for this as their polarity is per probe: pinsDependencies reports False for an
+    # unpinned dependency, while hasDangerousWorkflowScriptInjection reports True for an injection. A detail ending
+    # in path:line, optionally a line range and scorecard remediation text, is reported there, and one describing
+    # the project as a whole against the scorecard workflow.
+    findings=$(jq -r --arg anchor "${SCORECARD_ANCHOR}" '
+      .checks[] | select(.score >= 0 and .score < 10) | .name as $check | (.details // [])[]
+      | select(startswith("Warn: ")) | ltrimstr("Warn: ") | . as $detail
+      | (capture("^(?<msg>.*): (?<path>[^ :]+):(?<line>[0-9]+)(-[0-9]+)?(?<extra>: .*)?$") // {msg: $detail, path: $anchor, line: "0"})
+      | "\(.path):\([(.line | tonumber), 1] | max):1: [\($check)] \(.msg)\(.extra // "")"
+    ' "${out}")
+    if [ -n "${findings}" ]; then
+      printf '%s\n' "${findings}"
+      rc=1
+    fi
+  fi
+
+  rm -rf "${dir}" "${out}" "${err}"
+
+  return ${rc}
+}
+
 cd "$(git rev-parse --show-toplevel)" || exit 1
 
 if [[ $# -eq 0 ]]; then
@@ -117,6 +172,8 @@ elif [[ $1 == "shellcheck" ]]; then
 elif [[ $1 == "typos" ]]; then
   shift
   run_typos "$@"
+elif [[ $1 == "scorecard" ]]; then
+  run_scorecard
 else
   reviewdog "$@"
 fi

.lefthook.yml

@@ -11,7 +11,7 @@ pre-commit:
       run: |
         COUNT=0
 
-        for TOOL in golangci-lint pnpm reuse shellcheck trufflehog typos yamllint zizmor; do
+        for TOOL in golangci-lint jq pnpm reuse scorecard shellcheck trufflehog typos yamllint zizmor; do
           if ! command -v ${TOOL} >/dev/null 2>&1; then
             if [ ${COUNT} -eq 0 ]; then
               MISSING=${TOOL}
@@ -59,6 +59,15 @@ pre-commit:
             run: reuse lint --lines && echo "0 issues."
             glob: "{REUSE.toml,LICENSES/*}"
 
+          - name: scorecard
+            run: .buildkite/steps/lint.sh scorecard && echo "0 issues."
+            glob:
+              - "*Dockerfile*"
+              - ".github/workflows/*"
+              - ".buildkite/hooks/*"
+              - ".renovaterc*"
+              - "*.sh"
+
           - name: shellcheck
             run: shellcheck --format=gcc {staged_files} && echo "0 issues."
             file_types:

.reviewdog.yml

@@ -43,6 +43,13 @@ runner:
       - '%f:%l:%c: %m'
     level: error
 
+  scorecard:
+    cmd: .buildkite/steps/lint.sh scorecard
+    errorformat:
+      - '%E%f:%l:%c: %m'
+      - '%C%.%#'
+    level: error
+
   reuse:
     # reuse reports 'path: message' with no position, so a 1:1 position is added for reviewdog.
     cmd: reuse lint --lines | sed -e 's|^\([^:]*\):|\1:1:1:|'