蚌埠學院官網綜合新聞條目抓取

蚌埠學院綜合新聞

import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import json

def get_one_page(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
        } 
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

def page_parser(html):
    soup = BeautifulSoup(html,'lxml')
    
    for td in soup.find_all(name='td',attrs={'height':24}):
        # 超連結
        href = 'http://www.bbc.edu.cn' + td.find(name='td').a.attrs['href']
        # 标題
        title = td.find(name='td').a.font.string
        # 釋出時間
        postTime = td.find(class_='postTime').string
        yield {
            'href':href,
            'title':title,
            'postTime':postTime
        }

def get_pages(url):
    html = get_one_page(url)
    soup = BeautifulSoup(html,'lxml')
    # 擷取總頁碼
    pages = soup.find(name='a',attrs={'title':'進入尾頁'}).attrs['href']
    # 将總頁碼提取出來
    pages = pages.split('/')[8]
    if pages :
         return pages
    return None

def write_to_file(content):
    with open('result.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False) + '\n')

def main(num=0):
    pages = get_pages('http://www.bbc.edu.cn/s/21/t/267/p/22/i/1/list.htm')
    pages = int(pages)
    if num:
        pages=num
    for page in range(1,pages):
        url = 'http://www.bbc.edu.cn/s/21/t/267/p/22/i/'+str(page)+'/list.htm'
        html = get_one_page(url)
        for item in page_parser(html):
            print(item)
    print('抓取了： '+str(pages)+'頁綜合新聞')

if __name__ == '__main__':
    main(20)