Last active
June 13, 2022 20:04
-
-
Save dannguyen/57423dbcb1713d31b659 to your computer and use it in GitHub Desktop.
A Bash script, using the jq JSON-parser, to scrape all the NHTSA 5 star vehcile ratings from its API http://www.nhtsa.gov/webapi/Default.aspx?SafetyRatings/API/5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # jq JSON parser is awesome: | |
| # http://stedolan.github.io/jq/ | |
| # The NHTSA API is pretty clunky, requiring you to get a list of all the years, then all the models in that year, then all the makes per model, and then | |
| # finally, you get the vehicle IDs needed to query the endpoint for one vehicle at a time. | |
| # | |
| # I query for JSON for most of the loop, and in the end, I get the Vehicle data in CSV format | |
| BURL='http://www.nhtsa.gov/webapi/api/SafetyRatings' | |
| # get all the years first | |
| curl -s "$BURL?format=json" | jq -r '.Results[] .ModelYear' | \ | |
| while read year; do | |
| echo "$year" | |
| echo "######" | |
| curl -s "$BURL/modelyear/$year?format=json" | jq -r '.Results[] .Make' | sed 's/ /%20/g' | while read -r carmake; do | |
| # Get the year and make | |
| echo " $carmake" | |
| echo " =======" | |
| curl -s "$BURL/modelyear/$year/make/$carmake?format=json" | jq -r '.Results[] .Model' | sed 's/ /%20/g' | \ | |
| while read -r model; do | |
| echo " $model" | |
| echo " -------" | |
| # Get the year, make, and model | |
| curl -s "$BURL/modelyear/$year/make/$carmake/model/$model?format=json" | jq -r '.Results[] .VehicleId' | \ | |
| while read -r id; do | |
| echo " $id: $year - $carmake - $model" | |
| curl -s "$BURL/VehicleId/$id?format=csv" -o "$id.csv" | |
| done | |
| done | |
| done | |
| done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment