osmanyz · April 24, 2020 10:40 · osmanyz · Apr 24, 2020
diff --git a/README.md b/README.md
diff --git a/main.py b/main.py
 import math
 import urllib3
 import re
 from bs4 import BeautifulSoup
 import json


 # func
 def get_score(el):
    if el[0] is None:
        return None

    matches = re.finditer(r"has (.*) Positive Feedback", el[0].text, re.MULTILINE)
    for matchNum, match in enumerate(matches, start=1):
        for groupNum in range(0, len(match.groups())):
            groupNum = groupNum + 1
            return match.group(groupNum)
    return None

 # main parser
 http = urllib3.PoolManager()
 url = 'https://www.ebay.com/sns?_pgn=1&store_search=turkish'
 response = http.request('GET', url)
 parser = BeautifulSoup(response.data, 'html.parser')

 # main result count
 result_count_base_el = parser.find('div', {"class": "sns-results-count"}).find_next('div')
 result_count_el = result_count_base_el.find_next('span', {"class": "BOLD"})
 total_result_count = int(result_count_el.text)
 total_result_page_count = math.ceil(total_result_count / 50)

 # main text result
 result_text_el = result_count_base_el.find_next('span', {"class": "BOLD"}).find_next('span', {"class": "BOLD"}).text

 # stores array
 stores = []

 # main items element
 items_el = parser.find('ul', {"class": "sns-items"}, True).find_all('li', {"class": "sns-item"})

 # get stores
 for item_el in items_el:
    title_el = item_el.find('div', {"class": "sns-item__title"})
    desc_el = item_el.find('div', {"class": "sns-item__desc"})
    # set store
    stores.append({
        'link': title_el.find('a').get('href'),
        'name': title_el.find('a').text,
        'desc': desc_el.text
    })

 # store detail
 storesDetail = []

 # get store detail
 for store in stores:
    store_response = http.request('GET', store['link'])
    store_parser = BeautifulSoup(store_response.data, 'html.parser')
    canonical = store_parser.find('link', {'rel': 'canonical'})
    if canonical is None:
        continue
    # seller user
    store_seller_user_el = store_parser.find('span', {"class": "str-metadata__seller-link"})
    # str-billboard__store
    store_title_meta_el = store_parser.find('div', {"class": "str-billboard__title-container"}).find_next('h1')
    store_title_meta_el = store_title_meta_el.find_next('div', {"class": "str-metadata"})
    # score percent
    store_percent_meta_sibling_el = store_parser.select('div.str-metadata span > span.clipped')
    store_score_el = store_parser.find('span', {'id': "feedback-score-tooltip"})
    store_score_el = store_score_el.find_next('a')
    # image
    store_image = store_parser.find('div', {"class": "str-billboard__store-image"}).find_next('a').find_next('img')
    # flowers
    store_meta_flowers_el = store_parser.find('span', {"class": "str-metadata__followers-number"})
    store_flower_count = 0
    if store_meta_flowers_el is not None:
        store_flower_count = int(store_meta_flowers_el.text)
    # description
    store_desc_el = store_parser.find('div', {"class": "str-billboard__description-container"}).find_next('p')
    store_desc_el = store_desc_el.find_next('span')
    # set store
    storesDetail.append({
        'link': store['link'],
        'name': store['name'],
        'user': store_seller_user_el.text,
        'image': store_image.get('src'),
        'flowers': store_flower_count,
        'score': int(store_score_el.text),
        'desc': store_desc_el.text,
        'score_percent': get_score(store_percent_meta_sibling_el),
    })

 # it's for example
 with open('result.json', 'w') as fp:
    json.dump(storesDetail, fp)

 print("Done!")

diff --git a/requirements.txt b/requirements.txt
 bs4
 urllib3
diff --git a/result.json b/result.json
 [
  {
    "link": "https://www.ebay.com/str/antiqueturkishrug",
    "name": "Antique Turkish Rug",
    "user": "fatma.gul",
    "image": "https://i.ebayimg.com/thumbs/images/g/jIQAAOSwCXVeIjbQ/s-l150.jpg",
    "flowers": 6,
    "score": 2,
    "desc": "Hello everyone, my name is Fatma G\u00fcl. I live in Kayseri. I started to deal with carpets from the child. I have learned a lot from my master who has been a rug seller for 40 years and I continue to learn. I'm 21 years old now and I've just joined the eBay family and I'm planning to get my college pocket money from eBay. If you want a special size, color or variety, please contact me. \r\n",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/authenticturkishdelight",
    "name": "Authentic Turkish Delight",
    "user": "authenticturkishdelight",
    "image": "https://i.ebayimg.com/thumbs/images/g/VRMAAOSwJYVdmKwD/s-l150.png",
    "flowers": 490,
    "score": 620,
    "desc": "OTTOMAN STYLE - AUTHENTIC TURKISH JEWELRY ",
    "score_percent": "98.1%"
  },
  {
    "link": "https://www.ebay.com/str/turkishdecor",
    "name": "TURKISH DECOR",
    "user": "cansu.1",
    "image": "https://i.ebayimg.com/thumbs/images/g/9BsAAOSwNaxeXDl4/s-l150.jpg",
    "flowers": 0,
    "score": 0,
    "desc": "Hello Everyone \ud83d\udc4b My name is Cansu. It has a great importance for the history of Turkish rugs Turkey. Veteran women have been the greatest contributors in creating this history.In the old times, many women weaved carpets for very long working hours to contribute to the home economy. Each carpet they weave has different and special meanings. All of these meanings have equivalents in motifs. As a woman, I am proud to show you the carpets woven by our veteran women. ",
    "score_percent": "0%"
  },
  {
    "link": "https://www.ebay.com/str/turkishartisan",
    "name": "Turkish Artisan",
    "user": "turkish-artisan",
    "image": "https://i.ebayimg.com/thumbs/images/g/EjYAAOSwtC5ehhwl/s-l150.jpg",
    "flowers": 1,
    "score": 1,
    "desc": "",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishevileye",
    "name": "Turkish Evil Eye",
    "user": "neer_41",
    "image": "https://i.ebayimg.com/thumbs/images/g/t6QAAOSwlLdccEuV/s-l150.jpg",
    "flowers": 20,
    "score": 39,
    "desc": "I do my best to publish high quality evil eye products here.  My first priority is customer satisfaction and I take pride in the high quality of my products. \r\n\r\n",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishgoods",
    "name": "Turkish Goods",
    "user": "turkish-goods",
    "image": "https://i.ebayimg.com/thumbs/images/g/dmgAAOSw8S9a61YJ/s-l150.png",
    "flowers": 520,
    "score": 4204,
    "desc": "We are based in Istanbul - Turkey, providing you Turkish Goods with remarkably low prices.",
    "score_percent": "96.7%"
  },
  {
    "link": "https://www.ebay.com/str/turkishkilimstore",
    "name": "Turkish Kilim Store",
    "user": "turkishkilim",
    "image": "https://i.ebayimg.com/thumbs/images/a/(KGrHqV,!hEE86b1hlS5BPQSLdHKrw~~/s-l150.jpg",
    "flowers": 4019,
    "score": 5393,
    "desc": "Welcome to my eBay Store. Please add me to your list of favorite sellers and visit often. Thank you for your business.",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishlinenandtowels",
    "name": "Turkish Linen and Towels",
    "user": "turkishlinentowels",
    "image": "https://i.ebayimg.com/thumbs/images/g/Y5gAAMXQuTNTNPGn/s-l150.jpg",
    "flowers": 81,
    "score": 720,
    "desc": "Authentic Handwoven High Quality Turkish Towels, Foutas, Peshtemals, Blankets, Coverlets and Throws.\r\nShipping discount is applied automatically for any additional item.\r\n100% Customer Satisfaction. ",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishnecklace",
    "name": "Turkish Necklace",
    "user": "ercanercany",
    "image": "https://i.ebayimg.com/thumbs/images/g/fYoAAOSwM~taYKfO/s-l150.jpg",
    "flowers": 605,
    "score": 1317,
    "desc": "TURKISH OTTOMAN STYLE SILVER PLATED NECKLACE , BRACELET , RING and EARRING\r\n\r\n\r\n",
    "score_percent": "98.7%"
  },
  {
    "link": "https://www.ebay.com/str/turkishebuy",
    "name": "Turkish eBuy",
    "user": "turkishebuy",
    "image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
    "flowers": 35,
    "score": 788,
    "desc": "",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/vintageturkishrugs",
    "name": "VINTAGE TURKISH RUGS",
    "user": "vintageturkishrugs",
    "image": "https://i.ebayimg.com/thumbs/images/g/fG8AAOSw5K5eDvF~/s-l150.jpg",
    "flowers": 58,
    "score": 44,
    "desc": "Turkish rug, Vintage Turkish Rug, Oushak Rug, Runner Rug, Oushak Runner Rug, Kilim Rug, Vintage Kilim Rug",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishangle",
    "name": "turkish angle",
    "user": "zayt-37",
    "image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
    "flowers": 9,
    "score": 95,
    "desc": "",
    "score_percent": "98.8%"
  },
  {
    "link": "https://www.ebay.com/str/turkishonlinemarket",
    "name": "turkish_online_market",
    "user": "turkish_online_market",
    "image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
    "flowers": 347,
    "score": 1514,
    "desc": "",
    "score_percent": "97.2%"
  },
  {
    "link": "https://www.ebay.com/str/turkishrugsstore",
    "name": "turkish_rugs_Store",
    "user": "turkish_rugs",
    "image": "https://i.ebayimg.com/thumbs/images/g/hVYAAOSwNRdX~9Bb/s-l150.jpg",
    "flowers": 955,
    "score": 2114,
    "desc": "Welcome to my eBay Store. Please add me to your list of favorite sellers and visit often. Thank you for your business.",
    "score_percent": "100%"
  },
  {
    "link": "https://www.ebay.com/str/turkishtraditionalmarket",
    "name": "turkish_traditional_market",
    "user": "turkish_traditional_market",
    "image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
    "flowers": 122,
    "score": 758,
    "desc": "",
    "score_percent": "94.7%"
  }
 ]
	import math
	import urllib3
	import re
	from bs4 import BeautifulSoup
	import json


	# func
	def get_score(el):
	if el[0] is None:
	return None

	matches = re.finditer(r"has (.*) Positive Feedback", el[0].text, re.MULTILINE)
	for matchNum, match in enumerate(matches, start=1):
	for groupNum in range(0, len(match.groups())):
	groupNum = groupNum + 1
	return match.group(groupNum)
	return None

	# main parser
	http = urllib3.PoolManager()
	url = 'https://www.ebay.com/sns?_pgn=1&store_search=turkish'
	response = http.request('GET', url)
	parser = BeautifulSoup(response.data, 'html.parser')

	# main result count
	result_count_base_el = parser.find('div', {"class": "sns-results-count"}).find_next('div')
	result_count_el = result_count_base_el.find_next('span', {"class": "BOLD"})
	total_result_count = int(result_count_el.text)
	total_result_page_count = math.ceil(total_result_count / 50)

	# main text result
	result_text_el = result_count_base_el.find_next('span', {"class": "BOLD"}).find_next('span', {"class": "BOLD"}).text

	# stores array
	stores = []

	# main items element
	items_el = parser.find('ul', {"class": "sns-items"}, True).find_all('li', {"class": "sns-item"})

	# get stores
	for item_el in items_el:
	title_el = item_el.find('div', {"class": "sns-item__title"})
	desc_el = item_el.find('div', {"class": "sns-item__desc"})
	# set store
	stores.append({
	'link': title_el.find('a').get('href'),
	'name': title_el.find('a').text,
	'desc': desc_el.text
	})

	# store detail
	storesDetail = []

	# get store detail
	for store in stores:
	store_response = http.request('GET', store['link'])
	store_parser = BeautifulSoup(store_response.data, 'html.parser')
	canonical = store_parser.find('link', {'rel': 'canonical'})
	if canonical is None:
	continue
	# seller user
	store_seller_user_el = store_parser.find('span', {"class": "str-metadata__seller-link"})
	# str-billboard__store
	store_title_meta_el = store_parser.find('div', {"class": "str-billboard__title-container"}).find_next('h1')
	store_title_meta_el = store_title_meta_el.find_next('div', {"class": "str-metadata"})
	# score percent
	store_percent_meta_sibling_el = store_parser.select('div.str-metadata span > span.clipped')
	store_score_el = store_parser.find('span', {'id': "feedback-score-tooltip"})
	store_score_el = store_score_el.find_next('a')
	# image
	store_image = store_parser.find('div', {"class": "str-billboard__store-image"}).find_next('a').find_next('img')
	# flowers
	store_meta_flowers_el = store_parser.find('span', {"class": "str-metadata__followers-number"})
	store_flower_count = 0
	if store_meta_flowers_el is not None:
	store_flower_count = int(store_meta_flowers_el.text)
	# description
	store_desc_el = store_parser.find('div', {"class": "str-billboard__description-container"}).find_next('p')
	store_desc_el = store_desc_el.find_next('span')
	# set store
	storesDetail.append({
	'link': store['link'],
	'name': store['name'],
	'user': store_seller_user_el.text,
	'image': store_image.get('src'),
	'flowers': store_flower_count,
	'score': int(store_score_el.text),
	'desc': store_desc_el.text,
	'score_percent': get_score(store_percent_meta_sibling_el),
	})

	# it's for example
	with open('result.json', 'w') as fp:
	json.dump(storesDetail, fp)

	print("Done!")
	[
	{
	"link": "https://www.ebay.com/str/antiqueturkishrug",
	"name": "Antique Turkish Rug",
	"user": "fatma.gul",
	"image": "https://i.ebayimg.com/thumbs/images/g/jIQAAOSwCXVeIjbQ/s-l150.jpg",
	"flowers": 6,
	"score": 2,
	"desc": "Hello everyone, my name is Fatma G\u00fcl. I live in Kayseri. I started to deal with carpets from the child. I have learned a lot from my master who has been a rug seller for 40 years and I continue to learn. I'm 21 years old now and I've just joined the eBay family and I'm planning to get my college pocket money from eBay. If you want a special size, color or variety, please contact me. \r\n",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/authenticturkishdelight",
	"name": "Authentic Turkish Delight",
	"user": "authenticturkishdelight",
	"image": "https://i.ebayimg.com/thumbs/images/g/VRMAAOSwJYVdmKwD/s-l150.png",
	"flowers": 490,
	"score": 620,
	"desc": "OTTOMAN STYLE - AUTHENTIC TURKISH JEWELRY ",
	"score_percent": "98.1%"
	},
	{
	"link": "https://www.ebay.com/str/turkishdecor",
	"name": "TURKISH DECOR",
	"user": "cansu.1",
	"image": "https://i.ebayimg.com/thumbs/images/g/9BsAAOSwNaxeXDl4/s-l150.jpg",
	"flowers": 0,
	"score": 0,
	"desc": "Hello Everyone \ud83d\udc4b My name is Cansu. It has a great importance for the history of Turkish rugs Turkey. Veteran women have been the greatest contributors in creating this history.In the old times, many women weaved carpets for very long working hours to contribute to the home economy. Each carpet they weave has different and special meanings. All of these meanings have equivalents in motifs. As a woman, I am proud to show you the carpets woven by our veteran women. ",
	"score_percent": "0%"
	},
	{
	"link": "https://www.ebay.com/str/turkishartisan",
	"name": "Turkish Artisan",
	"user": "turkish-artisan",
	"image": "https://i.ebayimg.com/thumbs/images/g/EjYAAOSwtC5ehhwl/s-l150.jpg",
	"flowers": 1,
	"score": 1,
	"desc": "",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishevileye",
	"name": "Turkish Evil Eye",
	"user": "neer_41",
	"image": "https://i.ebayimg.com/thumbs/images/g/t6QAAOSwlLdccEuV/s-l150.jpg",
	"flowers": 20,
	"score": 39,
	"desc": "I do my best to publish high quality evil eye products here. My first priority is customer satisfaction and I take pride in the high quality of my products. \r\n\r\n",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishgoods",
	"name": "Turkish Goods",
	"user": "turkish-goods",
	"image": "https://i.ebayimg.com/thumbs/images/g/dmgAAOSw8S9a61YJ/s-l150.png",
	"flowers": 520,
	"score": 4204,
	"desc": "We are based in Istanbul - Turkey, providing you Turkish Goods with remarkably low prices.",
	"score_percent": "96.7%"
	},
	{
	"link": "https://www.ebay.com/str/turkishkilimstore",
	"name": "Turkish Kilim Store",
	"user": "turkishkilim",
	"image": "https://i.ebayimg.com/thumbs/images/a/(KGrHqV,!hEE86b1hlS5BPQSLdHKrw~~/s-l150.jpg",
	"flowers": 4019,
	"score": 5393,
	"desc": "Welcome to my eBay Store. Please add me to your list of favorite sellers and visit often. Thank you for your business.",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishlinenandtowels",
	"name": "Turkish Linen and Towels",
	"user": "turkishlinentowels",
	"image": "https://i.ebayimg.com/thumbs/images/g/Y5gAAMXQuTNTNPGn/s-l150.jpg",
	"flowers": 81,
	"score": 720,
	"desc": "Authentic Handwoven High Quality Turkish Towels, Foutas, Peshtemals, Blankets, Coverlets and Throws.\r\nShipping discount is applied automatically for any additional item.\r\n100% Customer Satisfaction. ",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishnecklace",
	"name": "Turkish Necklace",
	"user": "ercanercany",
	"image": "https://i.ebayimg.com/thumbs/images/g/fYoAAOSwM~taYKfO/s-l150.jpg",
	"flowers": 605,
	"score": 1317,
	"desc": "TURKISH OTTOMAN STYLE SILVER PLATED NECKLACE , BRACELET , RING and EARRING\r\n\r\n\r\n",
	"score_percent": "98.7%"
	},
	{
	"link": "https://www.ebay.com/str/turkishebuy",
	"name": "Turkish eBuy",
	"user": "turkishebuy",
	"image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
	"flowers": 35,
	"score": 788,
	"desc": "",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/vintageturkishrugs",
	"name": "VINTAGE TURKISH RUGS",
	"user": "vintageturkishrugs",
	"image": "https://i.ebayimg.com/thumbs/images/g/fG8AAOSw5K5eDvF~/s-l150.jpg",
	"flowers": 58,
	"score": 44,
	"desc": "Turkish rug, Vintage Turkish Rug, Oushak Rug, Runner Rug, Oushak Runner Rug, Kilim Rug, Vintage Kilim Rug",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishangle",
	"name": "turkish angle",
	"user": "zayt-37",
	"image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
	"flowers": 9,
	"score": 95,
	"desc": "",
	"score_percent": "98.8%"
	},
	{
	"link": "https://www.ebay.com/str/turkishonlinemarket",
	"name": "turkish_online_market",
	"user": "turkish_online_market",
	"image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
	"flowers": 347,
	"score": 1514,
	"desc": "",
	"score_percent": "97.2%"
	},
	{
	"link": "https://www.ebay.com/str/turkishrugsstore",
	"name": "turkish_rugs_Store",
	"user": "turkish_rugs",
	"image": "https://i.ebayimg.com/thumbs/images/g/hVYAAOSwNRdX~9Bb/s-l150.jpg",
	"flowers": 955,
	"score": 2114,
	"desc": "Welcome to my eBay Store. Please add me to your list of favorite sellers and visit often. Thank you for your business.",
	"score_percent": "100%"
	},
	{
	"link": "https://www.ebay.com/str/turkishtraditionalmarket",
	"name": "turkish_traditional_market",
	"user": "turkish_traditional_market",
	"image": "https://i.ebayimg.com/thumbs/images/g/M8gAAOSwng9ZuDLm/s-l150.png",
	"flowers": 122,
	"score": 758,
	"desc": "",
	"score_percent": "94.7%"
	}
	]