Last active
October 6, 2022 17:33
-
-
Save kboghe/58b7f487e4435be863cd84732accde55 to your computer and use it in GitHub Desktop.
for-catch loop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| from bs4 import BeautifulSoup | |
| import random | |
| headers_browser = {'Connection': 'close',"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", | |
| "Accept-Encoding": "gzip","Accept-Language": "en-US,en;q=0.9,es;q=0.8", "Upgrade-Insecure-Requests": "1","User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"} | |
| for i in range(3): # loop the try-part (i.e. opening the link) until it works, but only try it 4 times at most# | |
| try: #try the following:# | |
| random_sleep_link = random.uniform(10, 15) #sleep for a random chosen amount of seconds between 10 and 15 seconds# | |
| time.sleep(random_sleep_link) | |
| page = requests.get(url,headers= headers_browser) #access the URL using the header settings defined earlier# | |
| except requests.exceptions.RequestException as e: #if anything weird happens...# | |
| random_sleep_except = random.uniform(240,360) | |
| print("I've encountered an error! I'll pause for"+str(random_sleep_except/60) + " minutes and try again \n") | |
| time.sleep(random_sleep_except) #sleep the script for x seconds and....# | |
| continue #...start the loop again from the beginning# | |
| else: #if the try-part works...# | |
| break #...break out of the loop# | |
| else: #if x amount of retries on the try-part don't work...# | |
| raise Exception("Something really went wrong here... I'm sorry.") #...raise an exception and stop the script# | |
| # if the script survived this part...# | |
| bsObj = BeautifulSoup(page.text,"lxml") #this means we're good to go and can parse the page into BeautifulSoup!# |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment