Python 爬虫

1. 配置第三方包

# 时间的模块
import datetime
# 数据分析模块，用来处理excel
import pandas as pd
#用来构造xlsx文件的模块
import xlsxwriter as xlw
# 用来爬取数据的模块
from urllib import request
# HTML或XML标签中的内容解析器
from bs4 import BeautifulSoup as bs

2.获取时间序列函数

# 产生时间序列
def dateRange1(start, end):
    datelist1 = [datetime.datetime.strftime(x, '%Y%m') for x in list(
        pd.date_range(start=start, end=end))]
    datelist = sorted(list(set(datelist1)))
    return datelist
# ['202005', '202006', '202007', '202008', '202009', '202010']

3.爬取网页

# 爬取网页数据，解析HTML文件，筛选数据，转换成列表格式数据
def getCommentsById(city, start, end): 
    weather_result = [] 
    # 获取时间序列 
    datelist = dateRange1(start, end)  # [ '202009', '202010']
    for i in datelist:
        url = 'http://lishi.tianqi.com/' + city + '/' + i + '.html'
        # 请求天气数据
        opener = request.Request(url)
        # 添加  HTTP请求头
        opener.add_header(
            'User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
        req = request.urlopen(opener).read()
        # 解析html 数据
        soup = bs(req, 'html.parser')
        
        # 'div .thrui > li ' 筛选html数据 
        weather_m = soup.select('div .thrui > li ')
        # 循环获取的数据  
        for i in weather_m[0:]:  
            tt = []
            for j in range(5):
               t = i.find_all('div')[j].string
               if t is not None:  # 存在None值的进行处理，否则不能写入到excel
                    tt.append(t)
               else:
                    tt.append('None')
            weather_result.append(tt)
            print(weather_result)
    return weather_result

4.输出excel文件

#  将list数据写入到本地excel中
def list_to_excel(weather_result, filename):
    # 创建excel 名称，路径
    workbook = xlw.Workbook('E:\\%s.xlsx' % filename)
    # 添加工作簿
    sheet = workbook.add_worksheet('weather_report')
    # 添加excel头标题文字
    title = ['日期', '最高气温', '最低气温', '天气', '风向',]
    for i in range(len(title)):
        # 将标题文字写入excel表头，字体加粗
        sheet.write_string(0, i, title[i], workbook.add_format({'bold': True}))  
    row, col = 1, 0
    for a, b, c, d , e in weather_result:
        # 依次将数据 写入表格
        sheet.write_string(row, col, a)
        sheet.write_string(row, col + 1, b)
        sheet.write_string(row, col + 2, c)
        sheet.write_string(row, col + 3, d)
        sheet.write_string(row, col + 4, d)
        row += 1
        # 关闭表格
    workbook.close()

5.调用

# 你要查询的城市的名称（拼音），起始时间，结束时间。
data = getCommentsById('hunan', '2020-09', '2020-10')

# 获取的data值，excel的文件名
list_to_excel(data, '湖南天气202009-202010')

 

全部源码

# 时间的模块
import datetime
# 数据分析模块，用来处理excel
import pandas as pd
#用来构造xlsx文件的模块
import xlsxwriter as xlw
# 用来爬取数据的模块
from urllib import request
# HTML或XML标签中的内容解析器
from bs4 import BeautifulSoup as bs

# 产生时间序列
def dateRange1(start, end):
    datelist1 = [datetime.datetime.strftime(x, '%Y%m') for x in list(
        pd.date_range(start=start, end=end))]
    datelist = sorted(list(set(datelist1)))
    return datelist
# ['202005', '202006', '202007', '202008', '202009', '202010']


# 爬取网页数据，解析HTML文件，筛选数据，转换成列表格式数据
def getCommentsById(city, start, end): 
    weather_result = [] 
    # 获取时间序列 
    datelist = dateRange1(start, end)  # [ '202009', '202010']
    for i in datelist:
        url = 'http://lishi.tianqi.com/' + city + '/' + i + '.html'
        # 请求天气数据
        opener = request.Request(url)
        # 添加  HTTP请求头
        opener.add_header(
            'User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
        req = request.urlopen(opener).read()
        # 解析html 数据
        soup = bs(req, 'html.parser')
        
        # 'div .thrui > li ' 筛选html数据 
        weather_m = soup.select('div .thrui > li ')
        # 循环获取的数据  
        for i in weather_m[0:]:  
            tt = []
            for j in range(5):
               t = i.find_all('div')[j].string
               if t is not None:  # 存在None值的进行处理，否则不能写入到excel
                    tt.append(t)
               else:
                    tt.append('None')
            weather_result.append(tt)
            print(weather_result)
    return weather_result

#  将list数据写入到本地excel中
def list_to_excel(weather_result, filename):
    # 创建excel 名称，路径
    workbook = xlw.Workbook('E:\\%s.xlsx' % filename)
    # 添加工作簿
    sheet = workbook.add_worksheet('weather_report')
    # 添加excel头标题文字
    title = ['日期', '最高气温', '最低气温', '天气', '风向',]
    for i in range(len(title)):
        # 将标题文字写入excel表头，字体加粗
        sheet.write_string(0, i, title[i], workbook.add_format({'bold': True}))  
    row, col = 1, 0
    for a, b, c, d , e in weather_result:
        # 依次将数据 写入表格
        sheet.write_string(row, col, a)
        sheet.write_string(row, col + 1, b)
        sheet.write_string(row, col + 2, c)
        sheet.write_string(row, col + 3, d)
        sheet.write_string(row, col + 4, d)
        row += 1
        # 关闭表格
    workbook.close()




# 你要查询的城市的名称（拼音），起始时间，结束时间。
data = getCommentsById('hunan', '2020-09', '2020-10')

# 获取的data值，excel的文件名
list_to_excel(data, '湖南天气202009-202010')