lecture05 0교시 파이썬 기초 연습문제

웹 크롤링

from bs4 import BeautifulSoup
import urllib.request
import re

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()
url = "https://www.imdb.com/title/tt4154796/reviews?ref_=tt_gl_3"
htmlData = urllib.request.urlopen(url)
# html 데이터 받기
bs = BeautifulSoup(htmlData, 'lxml')
# beautifulsoup 으로 번역
print(BeautifulSoup.prettify(bs))

title_list = bs.findAll('a', {'class': 'title'})
review_list = bs.findAll('div', {'class': 'text show-more__control'})
score_list = bs.findAll('span', {'class': 'rating-other-user-rating'})
# a 태그 class 가 title

for title in title_list:
    print(title.getText())
    # 필요한 제목만 가져오기위해 getText

for i, content in enumerate(review_list):
    print(i+1, content.getText())

for score in score_list:
    print(score.span.getText())
    # span이라는 태크 뒤에 10/10 가 나옴

f = open("./data/review.txt", "w", encoding='UTF8')
for i in range(len(title_list)):
    f.write(clean_str(title_list[i].getText())+" "+clean_str(review_list[i].getText())+"\n")
f.close()
# review에 제목과 내용 저장

f = open("./data/score.txt", "w", encoding='UTF8')
for i in range(len(score_list)):
    f.write(score_list[i].span.getText()+"\n")
f.close()
# score에 점수 저장

 

 

 

동적 크롤링

#동적 크롤링, 스크롤 이동 등 자동으로
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import re

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

driver = webdriver.Chrome("./chromedriver_win32/chromedriver")
driver.implicitly_wait(3)
#time.sleep 은 지정시간 (무조건지연), driver.implicitly_wait은 엔진 파싱 시간 기다림

driver.get('https://www.imdb.com/title/tt4154796/reviews?ref_=tt_ov_rt')

driver.find_element_by_xpath('//*[@id="load-more-trigger"]').click()
time.sleep(10)
driver.find_element_by_xpath('//*[@id="load-more-trigger"]').click()
time.sleep(10)
driver.find_element_by_xpath('//*[@id="load-more-trigger"]').click()
time.sleep(10)
# load more 이라는 트리거 xpath 복사해옴
# time.sleep(10) 10초 쉬기 페이지로드 대기, 봇탐지 피하기, random

click_list = driver.find_elements_by_xpath("//div[@class='expander-icon-wrapper show-more__control']")
# 글이 긴경우 더보기 트리거 xpath 복사해옴
for click in click_list:
    if click.is_displayed():
        click.click()
        time.sleep(1)

time.sleep(1)
req = driver.page_source
# driver로 페이지 소스 받기

bs = BeautifulSoup(req, 'lxml')
# beautifulsoup 로 번역

title_list = bs.findAll('a', 'title')
date_list = bs.findAll('span', 'review-date')
review_list = bs.findAll('div', 'text show-more__control')

score_list = bs.findAll('span', 'rating-other-user-rating')

for title in title_list:
    print(title.getText())

for date in date_list:
    print(date.getText())

for i, content in enumerate(review_list):
    print(i, content.getText()+"\n")

for score in score_list:
    print(score.span.getText())
# span이라는 태크 뒤에 10/10 가 나옴

f = open("./data/review_dynamic.txt", "w", encoding='UTF8')
for i in range(len(title_list)):
    f.write(clean_str(title_list[i].getText())+" "+clean_str(date_list[i].getText())+" "+clean_str(review_list[i].getText())+"\n")
f.close()

f = open("./data/score_dyanamic.txt", "w", encoding='UTF8')
for i in range(len(score_list)):
    f.write(score_list[i].span.getText()+"\n")
f.close()