python 아마존 bestsellers?ld=NSGoogle

완친파 아마존

 

# 인터넷 쇼핑몰의 분야별 베스트셀러 상품 크롤러

 

#Step 1. 필요한 모듈과 라이브러리를 로딩합니다.

 

from bs4 import BeautifulSoup

from selenium import webdriver

 

import time

import sys

import re

import math

import numpy 

import pandas as pd   

import xlwt 

import random

import os

import urllib.request

import urllib.parse

 

# 학습목표 1 : 사용자에게 다양한 메뉴를 보여 준 후 카테고리값을 입력 받아 해당 카테고리 메뉴를 실행한다.

# Step 2. 사용자에게 카테고리 메뉴를 보여주고 정보를 입력 받습니다.

print("=" *80)

print("     아마존 닷컴의 분야별 Best Seller 상품 정보 추출하기")

print("=" *80)

 

query_txt='아마존닷컴'

query_url='https://www.amazon.com/bestsellers?ld=NSGoogle'

 

res=urllib.request.urlopen(query_url)

 

html = res.read()

 

soup = BeautifulSoup(html, 'html.parser')

 

result = soup.find('ul', id='zg_browseRoot').find('ul')

 

slist = result.find_all('li')

sec_text_list=[]             # 첫번째항목 두번째항목...

sec_list=[]                  # 1.항목 2.항목...

count=1

 

for li in slist:

    sec_cnt=str(count)       # 카운터 스트링숫자

    sec_text = li.get_text() # 항목의 텍스트

    sec_text_list.append(sec_text) # 첫번째항목 두번째항목...

    sec_list.append(sec_cnt+"."+sec_text) # 1.항목 2.항목...

 

    count+=1

 

cnt=1

sec_input=[]

for i in sec_list:

    sec_input.append(i+'\t')

 

    if (cnt%3)==0:

 

        sec_input.append('\n')

 

    cnt+=1

 

sec_input.append('\n\n'+'1.위 분야 중에서 자료를 수집할 분야의 번호를  선택하세요: ')

 

sec=input(str(''.join(sec_input))) or str(40)

 

cnt = int(input('    2.해당 분야에서 크롤링 할 건수는 몇건입니까?(1-100 건 사이 입력): ') or 40) 

 

f_dir = input("    3.파일을 저장할 폴더명만 쓰세요(예:C:\\Users\\miero\\Documents\\data\\):") or 'C:\\Users\\miero\\Documents\\data\\'

print("\n")

 

for c in range(1,cnt+1):

#     print(sec_text_list[c-1])

    if sec==str(c):

        sec_name=sec_text_list[c-1]

 

if cnt > 30 :

      print("    요청 건수가 많아서 시간이 제법 소요되오니 잠시만 기다려 주세요~~")

else :

      print("    요청하신 데이터를 수집하고 있으니 잠시만 기다려 주세요~~")

      

# Step 3. 저장될 파일위치와 이름을 지정 한 후 크롬 드라이버를 실행하여 페이지를 엽니다

now = time.localtime()

s = '%04d-%02d-%02d-%02d-%02d-%02d' % (now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, now.tm_min, now.tm_sec)

 

os.makedirs(f_dir+s+'-'+query_txt+'-'+sec_name)

os.chdir(f_dir+s+'-'+query_txt+'-'+sec_name)

 

ff_dir=f_dir+s+'-'+query_txt+'-'+sec_name

ff_name=f_dir+s+'-'+query_txt+'-'+sec_name+'\\'+s+'-'+query_txt+'-'+sec_name+'.txt'

fc_name=f_dir+s+'-'+query_txt+'-'+sec_name+'\\'+s+'-'+query_txt+'-'+sec_name+'.csv'

fx_name=f_dir+s+'-'+query_txt+'-'+sec_name+'\\'+s+'-'+query_txt+'-'+sec_name+'.xls'

 

s_time = time.time( )

 

path = "b:/python/webdriver/chromedriver/chromedriver.exe"

driver = webdriver.Chrome(path)

    

driver.get(query_url)

time.sleep(5)

 

for c in range(1,cnt+1):

#     print(sec_text_list[c-1])

    if sec==str(c):

        driver.find_element_by_xpath(f"""//*[@id="zg_browseRoot"]/ul/li[{c}]/a""").click( )

time.sleep(1)

 

# 학습목표 2 : 해당 카테고리의 데이터를 수집합니다.

#Step 4. 화면을 스크롤해서 아래로 이동한 후 요청된 데이터를 수집합니다.

 

def scroll_down(driver):

      

      driver.execute_script("window.scrollBy(0,9300);")

      time.sleep(1)

 

scroll_down(driver)

 

# 비트맵 이미지 아이콘을 위한 대체 딕셔너리를 만듭니다

bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)

 

html = driver.page_source

soup = BeautifulSoup(html, 'html.parser')

 

reple_result = soup.select('#zg-center-div > #zg-ordered-list')

slist = reple_result[0].find_all('li')

 

if cnt < 51 :

         

    ranking2=[]

    title3=[]

    price2=[]

    score2=[]

    sat_count2=[]

    store2=[]

       

    count = 0

 

    for li in slist:

            

            f = open(ff_name, 'a',encoding='UTF-8')

            f.write("-----------------------------------------------------"+"\n")

 

            # 판매순위

            print("-" *70)

            try :

             ranking = li.find('span',class_='zg-badge-text').get_text().replace("#","")

            except AttributeError :

             ranking = ''

             print(ranking.replace("#",""))

            else :

             print("1.판매순위:",ranking)

 

             f.write('1.판매순위:'+ ranking + "\n")

 

            #제품 설명 

            try :

             title1 = li.find('div',class_='p13n-sc-truncated').get_text().replace("\n","")

            except AttributeError :

             title1 = ''

             print(title1.replace("\n",""))

             f.write('2.제품소개:'+ title1 + "\n")

            else :

             title2=title1.translate(bmp_map).replace("\n","") 

             print("2.제품소개:", title2.replace("\n",""))

 

             count += 1

             

             f.write('2.제품소개:'+ title2 + "\n")

            

             # 가격

             try :

               price = li.find('span','p13n-sc-price').get_text().replace("\n","")

             except AttributeError :

               price = ''

               

             print("3.가격:", price.replace("\n",""))

             f.write('3.가격:'+ price + "\n")

                  

             try :

                sat_count = li.find('a','a-size-small a-link-normal').get_text().replace(",","")

             except (IndexError , AttributeError) :

                sat_count='0'

                print('4.상품평 수: ',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

             else :

                print('4.상품평 수:',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

 

            #상품 별점 구하기

             try :

               score = li.find('span','a-icon-alt').get_text()

             except AttributeError :

               score=' '

               

             print('5.평점:',score)

             f.write('5.평점:'+ score + "\n")

 

             print("-" *70)

                          

             f.close( )

              

             time.sleep(0.3)

            

             ranking2.append(ranking)

             title3.append(title2.replace("\n",""))

             price2.append(price.replace("\n",""))

 

             try :   

               sat_count2.append(sat_count)

             except IndexError :

               sat_count2.append(0)

 

             score2.append(score)

 

             if count == cnt :

                break

                          

elif cnt >= 51 :    

    

    count = 0

   

    ranking2=[]

    title3=[]

    price2=[]

    score2=[]

    sat_count2=[]

    store2=[]

 

    for li in slist:           

            f = open(ff_name, 'a',encoding='UTF-8')

            f.write("-----------------------------------------------------"+"\n")

 

            # 판매순위

            print("-" *70)

            try :

             ranking = li.find('span',class_='zg-badge-text').get_text().replace("#","")

            except AttributeError :

             ranking = ''

             print(ranking.replace("#",""))

            else :

             print("1.판매순위:",ranking)

 

             f.write('1.판매순위:'+ ranking + "\n")

 

            #제품 설명 

            try :

             title1 = li.find('div',class_='p13n-sc-truncated').get_text().replace("\n","")

            except AttributeError :

             title1 = ''

             print(title1.replace("\n",""))

             f.write('2.제품소개:'+ title1 + "\n")

            else :

             title2=title1.translate(bmp_map).replace("\n","") 

             print("2.제품소개:", title2.replace("\n",""))

 

             count += 1

             

             f.write('2.제품소개:'+ title2 + "\n")

            

             # 가격

             try :

               price = li.find('span','p13n-sc-price').get_text().replace("\n","")

             except AttributeError :

               price = ''

               

             print("3.가격:", price.replace("\n",""))

             f.write('3.가격:'+ price + "\n")

                  

             try :

                sat_count = li.find('a','a-size-small a-link-normal').get_text().replace(",","")

             except (IndexError , AttributeError) :

                sat_count='0'

                print('4.상품평 수: ',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

             else :

                print('4.상품평 수:',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

 

            #상품 별점 구하기

             try :

               score = li.find('span','a-icon-alt').get_text()

             except AttributeError :

               score=' '

               

             print('5.평점:',score)

             f.write('5.평점:'+ score + "\n")

 

             print("-" *70)

                          

             f.close( )

              

             time.sleep(0.5)

 

             ranking2.append(ranking)

             title3.append(title2.replace("\n",""))

             price2.append(price.replace("\n",""))

 

             try :   

               sat_count2.append(sat_count)

             except IndexError :

               sat_count2.append(0)

 

             score2.append(score)

 

    # 1 페이지 정보 추출 후 2 페이지로 넘어가기

    driver.find_element_by_xpath("""//*[@id="zg-center-div"]/div[2]/div/ul/li[3]/a""").click( )

    print("\n")

    print("요청하신 데이터의 수량이 많아 다음 페이지의 데이터를 추출 중이오니 잠시만 기다려 주세요~^^")

    print("\n")       

    

    html = driver.page_source

    soup = BeautifulSoup(html, 'html.parser')

    reple_result = soup.select('#zg-center-div > #zg-ordered-list')

    slist = reple_result[0].find_all('li')

    

    for li in slist:

            

            f = open(ff_name, 'a',encoding='UTF-8')

            f.write("-----------------------------------------------------"+"\n")

 

            # 판매순위

            print("-" *70)

            try :

             ranking = li.find('span',class_='zg-badge-text').get_text().replace("#","")

            except AttributeError :

             ranking = ''

             print(ranking.replace("#",""))

            else :

             print("1.판매순위:",ranking)

 

             f.write('1.판매순위:'+ ranking + "\n")

 

            #제품 설명 

            try :

             title1 = li.find('div',class_='p13n-sc-truncated').get_text().replace("\n","")

            except AttributeError :

             title1 = ''

             print(title1.replace("\n",""))

             f.write('2.제품소개:'+ title1 + "\n")

            else :

             title2=title1.translate(bmp_map).replace("\n","") 

             print("2.제품소개:", title2.replace("\n",""))

 

             count += 1

             

             f.write('2.제품소개:'+ title2 + "\n")

            

             # 가격

             try :

               price = li.find('span','p13n-sc-price').get_text().replace("\n","")

             except AttributeError :

               price = ''

               

             print("3.가격:", price.replace("\n",""))

             f.write('3.가격:'+ price + "\n")

                  

             try :

                sat_count = li.find('a','a-size-small a-link-normal').get_text().replace(",","")

             except IndexError :

                sat_count='0'

                print('4.상품평 수: ',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

             except AttributeError :

                sat_count='0'

                print('4.상품평 수: ',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n") 

             else :

                print('4.상품평 수:',sat_count)

                f.write('4.상품평 수:'+ sat_count + "\n")

 

            #상품 별점 구하기

             try :

               score = li.find('span','a-icon-alt').get_text()

             except AttributeError :

               score=' '

               

             print('5.평점:',score)

             f.write('5.평점:'+ score + "\n")

 

             print("-" *70)

                         

             f.close( )

              

             time.sleep(0.5)

             

             ranking2.append(ranking)

             title3.append(title2.replace("\n",""))

             price2.append(price.replace("\n",""))

 

             try :   

               sat_count2.append(sat_count)

             except IndexError :

               sat_count2.append(0)

 

             score2.append(score)

      

             if count == cnt :

                break

else :

      print(" 검색 건수는 1건 - 최대 100 건까지만 가능합니다")

  

 

#Step 5. 검색 결과를 다양한 형태로 저장하기

              

amazon_best_seller = pd.DataFrame()

amazon_best_seller['판매순위']=ranking2

amazon_best_seller['제품소개']=pd.Series(title3)

amazon_best_seller['판매가격']=pd.Series(price2)

amazon_best_seller['상품평 갯수']=pd.Series(sat_count2)

amazon_best_seller['상품평점']=pd.Series(score2)

# csv 형태로 저장하기

amazon_best_seller.to_csv(fc_name,encoding="utf-8-sig",index=True)

 

# 엑셀 형태로 저장하기

amazon_best_seller.to_excel(fx_name ,index=True)

 

e_time = time.time( )

t_time = e_time - s_time

 

# txt 파일에 크롤링 요약 정보 저장하기

orig_stdout = sys.stdout

f = open(ff_name, 'a',encoding='UTF-8')

sys.stdout = f

 

# Step 6. 요약 정보를 출력하기

print("\n")

print("=" *50)

print("총 소요시간은 %s 초 이며," %t_time)

print("총 저장 건수는 %s 건 입니다 " %count)

print("=" *50)

 

sys.stdout = orig_stdout

f.close( )

 

print("\n") 

print("=" *80)

print("1.요청된 총 %s 건의 리뷰 중에서 실제 크롤링 된 리뷰수는 %s 건입니다" %(cnt,count))

print("2.총 소요시간은 %s 초 입니다 " %round(t_time,1))

print("3.파일 저장 완료: txt 파일명 : %s " %ff_name)

print("4.파일 저장 완료: csv 파일명 : %s " %fc_name)

print("5.파일 저장 완료: xls 파일명 : %s " %fx_name)

print("=" *80)

driver.close()