Crawler：基于BeautifulSoup庫+requests庫實作爬取2018最新電影《後來的我們》熱門短評

輸出結果

實作代碼

# -*- coding: utf-8 -*-

#Py之Crawler：利用BeautifulSoup庫實作爬取2018最新電影《後來的我們》熱門短評

import time

import requests

import csv

from bs4 import BeautifulSoup

head = 'https://movie.douban.com/subject/'

middle = '/comments?start='

zr_tail = '&limit=20&sort=new_score&status=P&percent_type='

names = []

header = {

  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'

}

zr_urls = ['最熱']

def createUrls():

  for j in range(1, 34, 2):

      name = names[j]

      #print(name)

      for i in range(0, 100, 20):

          zr_urls.append(head + str(name) + middle + str(i) + zr_tail)

          #print(zr_urls)得到某個電影短評位址(預設最熱排序)，如

https://movie.douban.com/subject/27063335/comments?start=0&limit=20&sort=new_score&status=P&percent_type=

def readName():

  with open('爬取電影名稱.txt', mode='r', encoding='utf-8') as f:

      for i in f.readlines():

          i = i.strip('\n')

          names.append(i)

          #print(names)  

readName()

createUrls()

get_comments(zr_urls)  #傳入zr_urls