aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCraig Jennings <c@cjennings.net>2026-07-16 11:19:45 -0500
committerCraig Jennings <c@cjennings.net>2026-07-16 11:19:45 -0500
commit794a8bd606006cc21351be098a30cbbfe55750e4 (patch)
treeef3a113221cc49610a3e6f1e607af0f2f62fab91
parentcf3eadc5dfeff5145feb891a2e61d1ada9a94df0 (diff)
downloadrulesets-794a8bd606006cc21351be098a30cbbfe55750e4.tar.gz
rulesets-794a8bd606006cc21351be098a30cbbfe55750e4.zip
fix(githooks): match fixed-case secret tokens case-sensitively
The secret scan ran every pattern through grep -iE, so AKIA[0-9A-Z]{16} matched any mixed-case 20-char run. Random base64 in an embedded image blob hits that about 6% of the time per 100KB, enough to block a real commit and force --no-verify. takuzu hit it on a PNG sprite. AWS keys are uppercase, sk- keys lowercase, and PEM headers fixed, so those three now match case-sensitively. Only the keyword=value patterns still need -i. I measured both causes first. Across ~10MB of random base64 the old pattern produced 7 matches and would have false-positived 6 of 100 sprite-sized blobs. Case-sensitive matching produced none. takuzu's report also proposed skipping any added line containing ";base64,". I left that out. It guards against an uppercase AKIA run surviving the case fix, which never occurred in the sample and runs about one in a million per blob. The cost is real: minified bundles put a whole file on one line, so a data URI and a live key can share it, and skipping the line hides the key. A test covers that. The three variants share one scan block, so elisp, bash, and go all carry the change. Splitting one grep into two meant a line matching both passes got reported twice, which reads as two separate leaks. awk dedupes it.
-rwxr-xr-xlanguages/bash/githooks/pre-commit22
-rwxr-xr-xlanguages/elisp/githooks/pre-commit22
-rwxr-xr-xlanguages/go/githooks/pre-commit22
-rw-r--r--scripts/tests/pre-commit-secret-scan.bats144
4 files changed, 195 insertions, 15 deletions
diff --git a/languages/bash/githooks/pre-commit b/languages/bash/githooks/pre-commit
index e41c41c..880d5cf 100755
--- a/languages/bash/githooks/pre-commit
+++ b/languages/bash/githooks/pre-commit
@@ -9,11 +9,23 @@ cd "$REPO_ROOT" || exit 1
# --- 1. Secret scan ---
# Patterns for common credentials. Scans only added lines in the staged diff.
-SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])'
-
-secret_hits="$(git diff --cached -U0 --diff-filter=AM \
- | grep '^+' | grep -v '^+++' \
- | grep -iEn "$SECRET_PATTERNS" || true)"
+#
+# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys
+# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i,
+# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an
+# embedded image blob hits ~6% of the time per 100KB and blocks real commits.
+# Only the keyword=value patterns need -i.
+SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)'
+SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']'
+
+added_lines="$(git diff --cached -U0 --diff-filter=AM \
+ | grep '^+' | grep -v '^+++' || true)"
+
+cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)"
+ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)"
+# awk dedupes lines both passes matched, keeping first-seen order.
+secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \
+ | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)"
if [ -n "$secret_hits" ]; then
echo "pre-commit: potential secret in staged changes:" >&2
diff --git a/languages/elisp/githooks/pre-commit b/languages/elisp/githooks/pre-commit
index 909cde2..27f280c 100755
--- a/languages/elisp/githooks/pre-commit
+++ b/languages/elisp/githooks/pre-commit
@@ -9,11 +9,23 @@ cd "$REPO_ROOT"
# --- 1. Secret scan ---
# Patterns for common credentials. Scans only added lines in the staged diff.
-SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])'
-
-secret_hits="$(git diff --cached -U0 --diff-filter=AM \
- | grep '^+' | grep -v '^+++' \
- | grep -iEn "$SECRET_PATTERNS" || true)"
+#
+# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys
+# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i,
+# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an
+# embedded image blob hits ~6% of the time per 100KB and blocks real commits.
+# Only the keyword=value patterns need -i.
+SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)'
+SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']'
+
+added_lines="$(git diff --cached -U0 --diff-filter=AM \
+ | grep '^+' | grep -v '^+++' || true)"
+
+cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)"
+ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)"
+# awk dedupes lines both passes matched, keeping first-seen order.
+secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \
+ | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)"
if [ -n "$secret_hits" ]; then
echo "pre-commit: potential secret in staged changes:" >&2
diff --git a/languages/go/githooks/pre-commit b/languages/go/githooks/pre-commit
index a3d6f3f..a6297c8 100755
--- a/languages/go/githooks/pre-commit
+++ b/languages/go/githooks/pre-commit
@@ -9,11 +9,23 @@ cd "$REPO_ROOT"
# --- 1. Secret scan ---
# Patterns for common credentials. Scans only added lines in the staged diff.
-SECRET_PATTERNS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----|(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"'])'
-
-secret_hits="$(git diff --cached -U0 --diff-filter=AM \
- | grep '^+' | grep -v '^+++' \
- | grep -iEn "$SECRET_PATTERNS" || true)"
+#
+# Two passes because case-sensitivity differs. AWS keys are uppercase, sk- keys
+# lowercase, PEM headers fixed, so those match case-SENSITIVELY: under -i,
+# AKIA[0-9A-Z]{16} matches any mixed-case 20-char run, which random base64 in an
+# embedded image blob hits ~6% of the time per 100KB and blocks real commits.
+# Only the keyword=value patterns need -i.
+SECRET_PATTERNS_CS='(AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9_-]{20,}|-----BEGIN (RSA|DSA|EC|OPENSSH|PGP)( PRIVATE)?( KEY| KEY BLOCK)?-----)'
+SECRET_PATTERNS_CI='(api[_-]?key|api[_-]?secret|auth[_-]?token|secret[_-]?key|bearer[_-]?token|access[_-]?token|password)[[:space:]]*[:=][[:space:]]*["'"'"'][^"'"'"']{16,}["'"'"']'
+
+added_lines="$(git diff --cached -U0 --diff-filter=AM \
+ | grep '^+' | grep -v '^+++' || true)"
+
+cs_hits="$(printf '%s\n' "$added_lines" | grep -nE "$SECRET_PATTERNS_CS" || true)"
+ci_hits="$(printf '%s\n' "$added_lines" | grep -niE "$SECRET_PATTERNS_CI" || true)"
+# awk dedupes lines both passes matched, keeping first-seen order.
+secret_hits="$(printf '%s\n%s' "$cs_hits" "$ci_hits" \
+ | grep -v '^[[:space:]]*$' | awk '!seen[$0]++' || true)"
if [ -n "$secret_hits" ]; then
echo "pre-commit: potential secret in staged changes:" >&2
diff --git a/scripts/tests/pre-commit-secret-scan.bats b/scripts/tests/pre-commit-secret-scan.bats
new file mode 100644
index 0000000..013129e
--- /dev/null
+++ b/scripts/tests/pre-commit-secret-scan.bats
@@ -0,0 +1,144 @@
+#!/usr/bin/env bats
+# Tests for the secret-scan block shared by the elisp, bash, and go pre-commit
+# hooks. The block greps added lines in the staged diff for credential
+# patterns; a hit blocks the commit (exit 1), a clean scan falls through to the
+# variant's language check (exit 0).
+#
+# Every case stages a .txt file, so the language checks that follow the scan
+# (check-parens, shellcheck, gofmt) all skip and the scan is what's under test.
+#
+# The two boundary cases exist because of a live false-positive in a downstream
+# project: an embedded PNG sprite data URI blocked a real commit and forced
+# --no-verify. Root cause was `grep -iE` applying case-insensitivity to the
+# fixed-case AWS token AKIA[0-9A-Z]{16}, so any mixed-case 20-char run inside a
+# random base64 blob matched. Measured at ~6% of 100KB blobs; case-sensitive
+# matching drops it to 0 across ~10MB.
+
+VARIANTS="elisp bash go"
+
+setup() {
+ REPO="$(mktemp -d)"
+ cd "$REPO" || return 1
+ git init -q .
+ git config user.email t@example.com
+ git config user.name Test
+}
+
+teardown() {
+ cd /tmp || true
+ [ -n "${REPO:-}" ] && rm -rf "$REPO"
+}
+
+# Stage $2 as the content of file $1 (default staged.txt).
+stage() {
+ local file="${2:-staged.txt}"
+ printf '%s\n' "$1" > "$file"
+ git add "$file"
+}
+
+# Run a variant's hook in the temp repo. $1 = variant name.
+run_hook() {
+ bash "${BATS_TEST_DIRNAME}/../../languages/$1/githooks/pre-commit"
+}
+
+# ---- Normal: real secrets still block, clean content still passes ----
+
+@test "secret-scan: clean content passes in every variant" {
+ stage 'const greeting = "hello world";'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 0 ] || { echo "$v blocked clean content: $output"; return 1; }
+ done
+}
+
+@test "secret-scan: a real uppercase AWS access key blocks in every variant" {
+ stage 'aws_key = "AKIAIOSFODNN7EXAMPLE"'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 1 ] || { echo "$v missed an AWS key"; return 1; }
+ [[ "$output" == *"potential secret"* ]]
+ done
+}
+
+@test "secret-scan: a keyword=value credential blocks in every variant" {
+ stage 'api_key: "sk_live_9f3b2a7c1e4d8f0a6b5"'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 1 ] || { echo "$v missed an api_key assignment"; return 1; }
+ done
+}
+
+@test "secret-scan: a PEM private-key header blocks in every variant" {
+ stage '-----BEGIN RSA PRIVATE KEY-----'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 1 ] || { echo "$v missed a PEM header"; return 1; }
+ done
+}
+
+# ---- Boundary: the case-sensitivity fix ----
+
+@test "secret-scan: a lowercase akia-like run does not block (AWS keys are uppercase)" {
+ # Under `grep -iE` this matched the AKIA token and blocked a legitimate commit.
+ stage 'const blob = "akiaiosfodnn7examplexyz";'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 0 ] || { echo "$v false-positived on a lowercase run: $output"; return 1; }
+ done
+}
+
+@test "secret-scan: an embedded base64 data URI carrying a mixed-case akia run does not block" {
+ # The live failure: a sprite blob whose random base64 contained a mixed-case
+ # 20-char run. Case-sensitive matching is what clears it.
+ stage 'const SPRITE = "data:image/png;base64,iVBORw0KGgoAkIaIOSFODNN7ExAMPLEqQmCC";'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 0 ] || { echo "$v false-positived on a sprite data URI: $output"; return 1; }
+ done
+}
+
+# ---- Boundary: the scan must not go blind on data-URI lines ----
+
+@test "secret-scan: a real credential sharing a line with a base64 data URI still blocks" {
+ # Minified bundles put a whole file on one line, so a data URI and a real key
+ # can share it. Skipping any line containing ';base64,' would hide the key.
+ stage 'const S="data:image/png;base64,iVBORw0KGgoAAAANS";const c={api_key:"sk_live_9f3b2a7c1e4d8f0a6b5"};'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 1 ] || { echo "$v went blind on a data-URI line and missed the key"; return 1; }
+ done
+}
+
+@test "secret-scan: a line matching both passes is reported once, not twice" {
+ # The scan runs a case-sensitive and a case-insensitive pass. A line carrying
+ # both an AWS key and a keyword=value credential hits both; reporting it twice
+ # reads as two separate leaks.
+ stage 'api_key = "AKIAIOSFODNN7EXAMPLE_padding"'
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 1 ]
+ hits="$(printf '%s\n' "$output" | grep -c 'AKIAIOSFODNN7EXAMPLE_padding')"
+ [ "$hits" -eq 1 ] || { echo "$v reported the line $hits times, want 1"; return 1; }
+ done
+}
+
+# ---- Error / edge ----
+
+@test "secret-scan: an empty staged diff passes" {
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 0 ] || { echo "$v failed on an empty diff: $output"; return 1; }
+ done
+}
+
+@test "secret-scan: a secret only on a removed line does not block" {
+ # The scan reads added lines. Deleting a key should never block the deletion.
+ stage 'aws_key = "AKIAIOSFODNN7EXAMPLE"'
+ git commit -qm "seed" --no-verify
+ printf 'clean\n' > staged.txt
+ git add staged.txt
+ for v in $VARIANTS; do
+ run run_hook "$v"
+ [ "$status" -eq 0 ] || { echo "$v blocked a removal: $output"; return 1; }
+ done
+}