diff options
| author | Craig Jennings <c@cjennings.net> | 2026-07-16 11:19:45 -0500 |
|---|---|---|
| committer | Craig Jennings <c@cjennings.net> | 2026-07-16 11:19:45 -0500 |
| commit | 794a8bd606006cc21351be098a30cbbfe55750e4 (patch) | |
| tree | ef3a113221cc49610a3e6f1e607af0f2f62fab91 | |
| parent | cf3eadc5dfeff5145feb891a2e61d1ada9a94df0 (diff) | |
| download | rulesets-794a8bd606006cc21351be098a30cbbfe55750e4.tar.gz rulesets-794a8bd606006cc21351be098a30cbbfe55750e4.zip | |
fix(githooks): match fixed-case secret tokens case-sensitively
The secret scan ran every pattern through grep -iE, so AKIA[0-9A-Z]{16} matched any mixed-case 20-char run. Random base64 in an embedded image blob hits that about 6% of the time per 100KB, enough to block a real commit and force --no-verify. takuzu hit it on a PNG sprite. AWS keys are uppercase, sk- keys lowercase, and PEM headers fixed, so those three now match case-sensitively. Only the keyword=value patterns still need -i.
I measured both causes first. Across ~10MB of random base64 the old pattern produced 7 matches and would have false-positived 6 of 100 sprite-sized blobs. Case-sensitive matching produced none.
takuzu's report also proposed skipping any added line containing ";base64,". I left that out. It guards against an uppercase AKIA run surviving the case fix, which never occurred in the sample and runs about one in a million per blob. The cost is real: minified bundles put a whole file on one line, so a data URI and a live key can share it, and skipping the line hides the key. A test covers that.
The three variants share one scan block, so elisp, bash, and go all carry the change. Splitting one grep into two meant a line matching both passes got reported twice, which reads as two separate leaks. awk dedupes it.
| -rwxr-xr-x | languages/bash/githooks/pre-commit | 22 | ||||
| -rwxr-xr-x | languages/elisp/githooks/pre-commit | 22 | ||||
| -rwxr-xr-x | languages/go/githooks/pre-commit | 22 | ||||
| -rw-r--r-- | scripts/tests/pre-commit-secret-scan.bats | 144 |
4 files changed, 195 insertions, 15 deletions
diff --git a/languages/bash/githooks/pre-commit b/languages/bash/githooks/pre-commit index e41c41c..880d5cf 100755 --- a/languages/bash/githooks/pre-commit +++ b/languages/bash/githooks/pre-commit @@ -9,11 +9,23 @@ cd "$REPO_ROOT" || exit 1 # --- 1. Secret scan --- # Patterns for common credentials. Scans only added lines in the staged diff. -SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])' - -secret_hits="$(git diff --cached -U0 --diff-filter=AM \ - | grep '^+' | grep -v '^+++' \ - | grep -iEn "$SECRET_PATTERNS" || true)" +# +# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys +# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i, +# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an +# embedded image blob hits ~6% of the time per 100KB and blocks real commits. +# Only the keyword=value patterns need -i. +SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)' +SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']' + +added_lines="$(git diff --cached -U0 --diff-filter=AM \ + | grep '^+' | grep -v '^+++' || true)" + +cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)" +ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)" +# awk dedupes lines both passes matched, keeping first-seen order. +secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \ + | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)" if [ -n "$secret_hits" ]; then echo "pre-commit: potential secret in staged changes:" >&2 diff --git a/languages/elisp/githooks/pre-commit b/languages/elisp/githooks/pre-commit index 909cde2..27f280c 100755 --- a/languages/elisp/githooks/pre-commit +++ b/languages/elisp/githooks/pre-commit @@ -9,11 +9,23 @@ cd "$REPO_ROOT" # --- 1. Secret scan --- # Patterns for common credentials. Scans only added lines in the staged diff. -SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])' - -secret_hits="$(git diff --cached -U0 --diff-filter=AM \ - | grep '^+' | grep -v '^+++' \ - | grep -iEn "$SECRET_PATTERNS" || true)" +# +# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys +# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i, +# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an +# embedded image blob hits ~6% of the time per 100KB and blocks real commits. +# Only the keyword=value patterns need -i. +SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)' +SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']' + +added_lines="$(git diff --cached -U0 --diff-filter=AM \ + | grep '^+' | grep -v '^+++' || true)" + +cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)" +ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)" +# awk dedupes lines both passes matched, keeping first-seen order. +secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \ + | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)" if [ -n "$secret_hits" ]; then echo "pre-commit: potential secret in staged changes:" >&2 diff --git a/languages/go/githooks/pre-commit b/languages/go/githooks/pre-commit index a3d6f3f..a6297c8 100755 --- a/languages/go/githooks/pre-commit +++ b/languages/go/githooks/pre-commit @@ -9,11 +9,23 @@ cd "$REPO_ROOT" # --- 1. Secret scan --- # Patterns for common credentials. Scans only added lines in the staged diff. -SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])' - -secret_hits="$(git diff --cached -U0 --diff-filter=AM \ - | grep '^+' | grep -v '^+++' \ - | grep -iEn "$SECRET_PATTERNS" || true)" +# +# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys +# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i, +# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an +# embedded image blob hits ~6% of the time per 100KB and blocks real commits. +# Only the keyword=value patterns need -i. +SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)' +SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']' + +added_lines="$(git diff --cached -U0 --diff-filter=AM \ + | grep '^+' | grep -v '^+++' || true)" + +cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)" +ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)" +# awk dedupes lines both passes matched, keeping first-seen order. +secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \ + | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)" if [ -n "$secret_hits" ]; then echo "pre-commit: potential secret in staged changes:" >&2 diff --git a/scripts/tests/pre-commit-secret-scan.bats b/scripts/tests/pre-commit-secret-scan.bats new file mode 100644 index 0000000..013129e --- /dev/null +++ b/scripts/tests/pre-commit-secret-scan.bats @@ -0,0 +1,144 @@ +#!/usr/bin/env bats +# Tests for the secret-scan block shared by the elisp, bash, and go pre-commit +# hooks. The block greps added lines in the staged diff for credential +# patterns; a hit blocks the commit (exit 1), a clean scan falls through to the +# variant's language check (exit 0). +# +# Every case stages a .txt file, so the language checks that follow the scan +# (check-parens, shellcheck, gofmt) all skip and the scan is what's under test. +# +# The two boundary cases exist because of a live false-positive in a downstream +# project: an embedded PNG sprite data URI blocked a real commit and forced +# --no-verify. Root cause was `grep -iE` applying case-insensitivity to the +# fixed-case AWS token AKIA[0-9A-Z]{16}, so any mixed-case 20-char run inside a +# random base64 blob matched. Measured at ~6% of 100KB blobs; case-sensitive +# matching drops it to 0 across ~10MB. + +VARIANTS="elisp bash go" + +setup() { + REPO="$(mktemp -d)" + cd "$REPO" || return 1 + git init -q . + git config user.email t@example.com + git config user.name Test +} + +teardown() { + cd /tmp || true + [ -n "${REPO:-}" ] && rm -rf "$REPO" +} + +# Stage $2 as the content of file $1 (default staged.txt). +stage() { + local file="${2:-staged.txt}" + printf '%s\n' "$1" > "$file" + git add "$file" +} + +# Run a variant's hook in the temp repo. $1 = variant name. +run_hook() { + bash "${BATS_TEST_DIRNAME}/../../languages/$1/githooks/pre-commit" +} + +# ---- Normal: real secrets still block, clean content still passes ---- + +@test "secret-scan: clean content passes in every variant" { + stage 'const greeting = "hello world";' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 0 ] || { echo "$v blocked clean content: $output"; return 1; } + done +} + +@test "secret-scan: a real uppercase AWS access key blocks in every variant" { + stage 'aws_key = "AKIAIOSFODNN7EXAMPLE"' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 1 ] || { echo "$v missed an AWS key"; return 1; } + [[ "$output" == *"potential secret"* ]] + done +} + +@test "secret-scan: a keyword=value credential blocks in every variant" { + stage 'api_key: "sk_live_9f3b2a7c1e4d8f0a6b5"' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 1 ] || { echo "$v missed an api_key assignment"; return 1; } + done +} + +@test "secret-scan: a PEM private-key header blocks in every variant" { + stage '-----BEGIN RSA PRIVATE KEY-----' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 1 ] || { echo "$v missed a PEM header"; return 1; } + done +} + +# ---- Boundary: the case-sensitivity fix ---- + +@test "secret-scan: a lowercase akia-like run does not block (AWS keys are uppercase)" { + # Under `grep -iE` this matched the AKIA token and blocked a legitimate commit. + stage 'const blob = "akiaiosfodnn7examplexyz";' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 0 ] || { echo "$v false-positived on a lowercase run: $output"; return 1; } + done +} + +@test "secret-scan: an embedded base64 data URI carrying a mixed-case akia run does not block" { + # The live failure: a sprite blob whose random base64 contained a mixed-case + # 20-char run. Case-sensitive matching is what clears it. + stage 'const SPRITE = "data:image/png;base64,iVBORw0KGgoAkIaIOSFODNN7ExAMPLEqQmCC";' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 0 ] || { echo "$v false-positived on a sprite data URI: $output"; return 1; } + done +} + +# ---- Boundary: the scan must not go blind on data-URI lines ---- + +@test "secret-scan: a real credential sharing a line with a base64 data URI still blocks" { + # Minified bundles put a whole file on one line, so a data URI and a real key + # can share it. Skipping any line containing ';base64,' would hide the key. + stage 'const S="data:image/png;base64,iVBORw0KGgoAAAANS";const c={api_key:"sk_live_9f3b2a7c1e4d8f0a6b5"};' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 1 ] || { echo "$v went blind on a data-URI line and missed the key"; return 1; } + done +} + +@test "secret-scan: a line matching both passes is reported once, not twice" { + # The scan runs a case-sensitive and a case-insensitive pass. A line carrying + # both an AWS key and a keyword=value credential hits both; reporting it twice + # reads as two separate leaks. + stage 'api_key = "AKIAIOSFODNN7EXAMPLE_padding"' + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 1 ] + hits="$(printf '%s\n' "$output" | grep -c 'AKIAIOSFODNN7EXAMPLE_padding')" + [ "$hits" -eq 1 ] || { echo "$v reported the line $hits times, want 1"; return 1; } + done +} + +# ---- Error / edge ---- + +@test "secret-scan: an empty staged diff passes" { + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 0 ] || { echo "$v failed on an empty diff: $output"; return 1; } + done +} + +@test "secret-scan: a secret only on a removed line does not block" { + # The scan reads added lines. Deleting a key should never block the deletion. + stage 'aws_key = "AKIAIOSFODNN7EXAMPLE"' + git commit -qm "seed" --no-verify + printf 'clean\n' > staged.txt + git add staged.txt + for v in $VARIANTS; do + run run_hook "$v" + [ "$status" -eq 0 ] || { echo "$v blocked a removal: $output"; return 1; } + done +} |
