Last active
July 27, 2017 19:44
-
-
Save deepfriedheroin/656999b15491ab96fac1eb9f0aa1f391 to your computer and use it in GitHub Desktop.
Scrape and download all roms from the eye
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| from bs4 import BeautifulSoup as bs | |
| from colorama import Fore | |
| import os | |
| import pathlib | |
| import requests as req | |
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | |
| PUB_URL = 'https://the-eye.eu/public/rom/' | |
| ROM_PATH = os.path.join(os.path.dirname(__file__), 'roms') | |
| FOLDERS = ['gba','gc','n64','nes','NoIntro'] | |
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | |
| def unescape(s): | |
| s = s.replace("%20", " ") | |
| s = s.replace("%21", "!") | |
| s = s.replace("%22", "") | |
| s = s.replace("%23", "#") | |
| s = s.replace("%24", "$") | |
| s = s.replace("%25", "%") | |
| s = s.replace("%26", "&") | |
| s = s.replace("%27", "") | |
| s = s.replace("%28", "(") | |
| s = s.replace("%29", ")") | |
| s = s.replace("%", " ") | |
| return s | |
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | |
| def download_roms(folders): | |
| for folder in folders: | |
| rom_dir = os.path.join(ROM_PATH, folder) | |
| url = PUB_URL + folder + '/' | |
| res = req.get(url).text | |
| soup = bs(res, 'html.parser') | |
| links = [a for a in soup.pre.find_all('a')] | |
| dl_links = links[1:len(links)] | |
| pathlib.Path(rom_dir).mkdir(parents=True, exist_ok=True) | |
| for link in dl_links: | |
| fn_link = link['href'] | |
| fn = unescape(fn_link) | |
| full_path = os.path.join(rom_dir, fn) | |
| if not os.path.isfile(full_path): | |
| rom = url + fn_link | |
| res = req.get(rom) | |
| print(Fore.WHITE + '✚ ' + 'Downloading: ' + rom_dir + ' ' + fn + '\n') | |
| with open(full_path, "wb") as file: | |
| file.write(res.content) | |
| if os.path.isfile(full_path): | |
| print(Fore.GREEN + '✔ ' + 'Downloaded: ' + rom_dir + ' ' + fn + '\n') | |
| return | |
| # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | |
| if __name__ == "__main__": | |
| download_roms(FOLDERS) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment