-
-
Save gourav3017/5b28e52a38c7506243add50ed40f7ded to your computer and use it in GitHub Desktop.
A simple python script to check broken links of a wesite
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from bs4 import BeautifulSoup,SoupStrainer | |
| import urllib.request | |
| import colorama,re,queue,threading | |
| from colorama import Fore | |
| from urllib.parse import * | |
| class check_link(): | |
| def __init__(self,address): | |
| self.address=address | |
| def check(self,address): | |
| try: | |
| req=urllib.request.Request(url=address) | |
| resp=urllib.request.urlopen(req) | |
| if resp.status in [400,404,403,408,409,501,502,503]:print (Fore.RED+resp.status+"-"+resp.reason+"-->"+address) | |
| else: print (Fore.GREEN+"no problem in-->"+address) | |
| except Exception as e: | |
| print (Fore.YELLOW+"{}-{}".format(e,address)) | |
| pass | |
| def pattern_adjust(a): | |
| try: | |
| if re.match('^#' ,a):return 0 | |
| r=urlsplit(a) | |
| if r.scheme=='' and (r.netloc!='' or r.path!=''): | |
| d=urlunsplit(r) | |
| if re.match('^//' ,d): | |
| m= re.search('(?<=//)\S+', d) | |
| d=m.group(0) | |
| m="https://"+d | |
| return m | |
| elif r.scheme=='' and r.netloc=='': | |
| return address+a | |
| else:return a | |
| except Exception as e: | |
| pass | |
| def extract_link(address): | |
| tags= {'a':'href', 'img':'src', 'script':'src', 'link':'href' } | |
| for key,value in iter(tags.items()): | |
| try: | |
| res=urllib.request.urlopen(address) | |
| response=res.read().decode('utf-8') #needs improvement | |
| for link in BeautifulSoup(response,"html.parser",parse_only=SoupStrainer(key)): | |
| if link.has_attr(value): | |
| p=pattern_adjust(link[value]) | |
| if p!=0 and str(p)!='None': | |
| newcheck=check_link(p) | |
| newcheck.check(p) | |
| if p not in hyperlinks: | |
| hyperlinks.add(p) | |
| if website.split('.')[1] in p:#needs improvement | |
| if not website.endswith(('.png','.jpeg','.js','jpg')): | |
| q.put(p) | |
| except Exception as e: | |
| print (e,address) | |
| def threader(): | |
| while True: | |
| value=q.get() | |
| result=extract_link(value) | |
| q.task_done() | |
| if __name__=="__main__": | |
| colorama.init() | |
| q=queue.Queue() | |
| global hyperlinks,website | |
| hyperlinks=set() | |
| website=input("Please enter the website address: ") | |
| for x in range(30): | |
| t=threading.Thread(target=threader) | |
| t.deamon=True | |
| t.start() | |
| q.put(website.strip()) | |
| q.join() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment