PYTHON PROGRAM RELATED TO INFORMATION RETRIEVAL AND WEB SEARCH

import io import re import requests from bs4 import BeautifulSoup from PyPDF2 import PdfFileReader from urllib.request import urlopen def checkWords(content): if len(content.split(" ")) > 50: return True else: return False def save(content, name): global output, counts if counts < 10001: content = re.sub(' +', ' ', content) if checkWords(content): with open(output+name+".txt", "w") as file: file.write(content) file.close() counts += 1 print(counts) def customCrawler(site): global counts if counts < 10001: response = requests.get(site, allow_redirects=False, timeout=25) html = BeautifulSoup(response.text,"html.parser") htmlText = html.get_text() if(len(htmlText ) > 0): save(htmlText , "output_html_{0}".format(str(counts))) for link in html.find_all("a"): try: hrefLink = link.attrs['href'] if hrefLink.startswith("#") == True and hrefLink.startswith("//") == True: continue if hrefLink.startswith("/") and len(hrefLink.count(".php")) == 1: hrefLink = site+hrefLink if hrefLink not in fetched_urls and "google" not in hrefLink and "youtube" not in hrefLink : if hrefLink.endswith(".pdf"): data = urlopen(hrefLink).read() temp = io.BytesIO(data) pdfFile = PdfFileReader(temp) combine = pdfFile.getPage(0).extractText().replace("\n", "") combine = re.sub(' +', ' ', combine) if len(combine.split(" ")) > 50: save(combine, "output_pdf_"+str(counts)) elif hrefLink.endswith(".txt"): r = requests.get(hrefLink, allow_redirects=False, timeout=20) s = BeautifulSoup(r.text,"html.parser") combine = s.get_text() if(len(combine) > 0): save(combine, "output_html_"+str(counts)) elif ("memphis" in hrefLink) and hrefLink.endswith(".mp3") == False and hrefLink.endswith(".mp4") == False: fetched_urls.append(hrefLink) customCrawler(hrefLink) except: pass else: return "files fetched." site="https://www.memphis.edu/" output = r"C:/Users/hsunki/OneDrive - The University of Memphis/Desktop/output/" fetched_urls = [] counts = 0 # calling function customCrawler(site)