Skip to content

Instantly share code, notes, and snippets.

@alefebvre-ls

alefebvre-ls/.git-verification-patterns.hjson

Last active October 31, 2016 13:32
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alefebvre-ls/4861a4d48be208f08e3e5bf2bf4e7997 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alefebvre-ls/4861a4d48be208f08e3e5bf2bf4e7997 to your computer and use it in GitHub Desktop.
Download ZIP
Python-based Git pre-commit hook to verify for secrets
Raw
.git-verification-patterns.hjson
patterns:
[
{
name: AWS Secret Key, Github Personal Access Token
pattern: (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\s*(:|=>|=)\s*(\"|')?[A-Za-z0-9/\+=]{40}([^a-zA-Z0-9]+|$)
min_occurences: 1
white_listed_pattern: "[^a-z0-9]+(0{40})[^a-z0-9]*"
},
{
name: AWS Access Key ID
pattern: (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)?_?(ID|id|Id)?(\"|')?\s*(:|=>|=)\s*(\"|')?[A-Za-z0-9/\+=]{20}([^a-zA-Z0-9]+|$)
min_occurences: 1
white_listed_pattern: "[^a-z0-9]+(0{20})[^a-z0-9]*"
},
{
name: DigitalOcean Access Token
pattern: (\"|')?(DigitalOcean)?_?(Access)?_?(Token)?_?(\"|')?\s*(:|=>|=)\s*(\"|')?[a-zA-Z0-9]{64}
min_occurences: 1,
white_listed_pattern: "[^a-z0-9]*0{64}[^a-zA-Z0-9]*"
},
{
name: Base64 Encoded HTTP Auth header
pattern: Authorization\s+"?Basic\s+[a-zA-Z0-9+/]+={0,2}"?
min_occurences: 1,
white_listed_pattern: Authorization "?Basic MDAwMDAwMDAwMDAwMDowMDAwMDAwMDAwMAo="?
},
{
name: SSH Private Key File
pattern: BEGIN RSA PRIVATE KEY
min_occurences: 1
white_listed_pattern: null
},
{
name: SSH Private Key File
pattern: END RSA PRIVATE KEY
min_occurences: 1
white_listed_pattern: null
},
{
name: SSH Private Key File
pattern: ^[^\s]{65}$
min_occurences: 3
white_listed_pattern: null
},
{
name: SSL Certificate
pattern: -*BEGIN ENCRYPTED PRIVATE KEY-*
min_occurences: 1
white_listed_pattern: null
},
{
name: SSL Certificate
pattern: -*BEGIN PRIVATE KEY-*
min_occurences: 1
white_listed_pattern: null
}
]
Raw
LICENSE
MIT License
Copyright (c) 2016 Lightspeed POS Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Raw
pre-commit
#!/usr/bin/python
"""
This Git hook verifies for secrets in the code
This file should be placed in the ~/.git-templates/hooks/ directory so that each newly initialized
repository
"""
import os
import os.path
import re
import sys
import subprocess
import collections
import configparser
import contextlib
import hjson
import sqlite3 as sql
import uuid
import time
from os.path import expanduser
ExecutionResult = collections.namedtuple(
'ExecutionResult',
'status, stdout, stderr'
)
DEBUG = False
stats = {
"files_verified": 0,
"lines_flagged": 0,
"lines_verified": 0,
"false_positives": 0
}
db_name = '{}/.git-verification-log.db'.format(expanduser("~"))
cur = None
conn = None
try:
conn = sql.connect(db_name)
with conn:
cur = conn.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS verifications(id TEXT, file TEXT, line_number INT, line_contents TEXT, match_pattern TEXT, false_positive INT, date INT)")
except sql.Error, e:
exit_with_message(1, "Error opening DB {}".format(db_name))
def run(cmd):
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout, stderr = process.communicate()
status = process.poll()
return ExecutionResult(status, stdout, stderr)
def load_config():
contents = get_file_contents('{}/.git-verification-patterns.hjson'.format(expanduser("~")))
return hjson.loads(contents)
def get_file_contents(file_path):
fh = open(file_path, 'r')
contents = fh.read()
fh.close()
return contents
def get_file_contents_as_lines(file_path):
# Split each line apart so we'll be able to find out the line number
return get_file_contents(file_path).split('\n')
'''
Log the verificaiton results to the local SQLite DB
'''
def log_verification():
pass
def get_current_commit():
if run('git rev-parse --verify HEAD'.split()).status:
return '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
else:
return 'HEAD'
def get_modified_files(against):
cmd = "git diff --cached --name-only {}".format(against)
if DEBUG:
print "RUNNING: {}".format(cmd)
files = run(cmd.split())
if files.status != 0:
exit_with_message(1, "Error getting list of files: {}".format(files.stderr));
file_list = list()
for f in files.stdout.split('\n'):
if f.strip() == "":
continue
if os.path.isfile(f):
file_list.append(f)
return file_list
def show_match(file, match, ln_num, pattern, desc):
print("------------------ MATCH FOUND ------------------")
print(' File: {} [Line {}]'.format(file,(ln_num+1)))
print(' Match: {}'.format(match))
print(' Pattern: {}'.format(pattern))
print(' Desc: {}'.format(desc))
sys.stdin = open('/dev/tty')
answer = raw_input('Commit anyway? [y/N] ')
if answer.strip().lower() == 'y':
stats['false_positives'] += 1
# Log the false positive match in sqlite to build stats
cur.execute("INSERT INTO verifications VALUES(?, ?, ?, ?, ?, ?, ?)", (str(uuid.uuid1()), file, ln_num, match, pattern, 1, int(time.time()) ) )
conn.commit()
else:
stats['lines_flagged'] += 1
print ""
def exit_with_message(exit_code,msg):
print msg
if DEBUG:
sys.exit(1)
else:
sys.exit(exit_code)
# ------------------- Run the verification ----------------------
try:
print "======================================================="
print "Running Verification for tokens, secret keys, etc..."
print "======================================================="
print ""
# Load the hjson config file
conf = load_config()
# Get the current commit hash to verify against
commit_hash = get_current_commit()
files = get_modified_files(commit_hash)
base_dir = os.getcwd()
user_email = run('git config user.email'.split())
if os.environ['WORKENV'] == '':
print "Missing WORKENV environment variable!"
sys.exit(1)
#valid_envs_list = os.environ['VALID_WORKENVS'].split(',')
valid_envs_list = map(lambda str : str.strip(), os.environ['VALID_WORKENVS'].split(','))
if re.match( r'.*/github_personal/.*', base_dir, re.M|re.I) and os.environ['WORKENV'] not in valid_envs_list:
print "Must switch to personal user before committing to personal project!"
sys.exit(1)
elif re.match( r'.*/github/.*', base_dir, re.M|re.I) and os.environ['WORKENV'] not in valid_envs_list:
print "Must switch to work user before committing to work/corporate based project!"
sys.exit(1)
# Now itterate through all the files
for f in files:
stats['files_verified'] += 1
match_occurences = {}
if DEBUG:
print "Verifying: {}".format(f)
# Load the file as a list of lines
file_lines = get_file_contents_as_lines(f)
ln_num = 0
# Itterate over each line
for l in file_lines:
# Itterate over each pattern
for p in conf['patterns']:
if p['min_occurences'] > 1:
match_occurences[p['name']] = 0
matchObj = re.search(p['pattern'], l, re.I) # re.M|re.I
if matchObj:
show_match(f, l, ln_num, p['pattern'], p['name'])
ln_num += 1
stats['lines_verified'] += 1
# Close the Sqlite connection as it's no longer needed
conn.close()
print "------------------ STATS ------------------"
print ' Total Files verified: {}'.format(stats['files_verified'])
print ' Total Lines verified: {}'.format(stats['lines_verified'])
print ' Lines flagged: {}'.format(stats['lines_flagged'])
print ' False positives: {}'.format(stats['false_positives'])
if stats['lines_flagged'] >= 1:
exit_with_message(1, ' Status: {}\n'.format("COMMIT FAILED!"))
else:
exit_with_message(0, ' Status: {}\n'.format("COMMIT OK"))
except subprocess.CalledProcessError:
# There was a problem calling "git status".
exit_with_message(12, "Error executing command")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment