目录

python爬虫环境准备预备知识UrllibBeautifulSoup正则表达式xlwt表格操作 sqlite3数据库操作获取数据解析数据保存数据到表格中保存数据到数据库中数据可视化Flask框架Echarts应用WorldCloud应用 完整代码

python爬虫

网络爬虫：按照一定规则，自动抓取互联网信息的程序或者脚本 学习视频地址：https://www.bilibili.com/video/BV12E411A7ZQ/?p=28

环境准备

导入包配置 import sys from bs4 import BeautifulSoup # 网页解析，获取数据 import re # 正则表达式，进行文字匹配 import urllib.request, urllib.error # 制定URl，获取网页数据 import xlwt # 进行excel操作 import sqlite3 # 进行sqlLite数据库操作 基础文件结构 """ 爬取豆瓣电影排名250电影 """ def main(): url = 'https://movie.douban.com/top250?start=0' savepath='.\\doubanTop250.xls' # 1.爬取网页 datalist = get_data(url) # 3.保存数据 sava_data() def get_data(url): datalist = [] # 2.逐一解析数据 return datalist def sava_data(savepath): print("save") if __name__ == '__main__': main()

预备知识

Urllib

import urllib.request import urllib.parse # 通过get请求访问 response = urllib.request.urlopen("http://www.baidu.com") print(response.read().decode('utf-8')) # 对获取到的源码进行utf-8解码 # 通过post请求访问:http://httpbin.org data = bytes(urllib.parse.urlencode({"hello": "word"}), encoding="utf-8") response = urllib.request.urlopen("http://httpbin.org/post",data=data) print(response.read().decode('utf-8')) # 超时处理 try: response = urllib.request.urlopen("http://httpbin.org/get",timeout=1) print(response.read().decode('utf-8')) except Exception as error: print(error) # 响应头 url = "http://douban.com" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 Edg/85.0.564.68" } req = urllib.request.Request(url=url, headers=headers, method="POST") response = urllib.request.urlopen(req, timeout=3) print(response.status) print(response.read().decode('utf-8'))

BeautifulSoup

将复杂的html转换成一个复杂的树形结构，每个节点都是python对象所有对象可以分文四种： Tag:标签及其内容:(但是只是第一个标签)NavigableString:标签里的内容BeautifulSoup:自身，整个文档Comment:注释:是一个特殊的NavigableString输出的内容不包含注释符号 from bs4 import BeautifulSoup file = open("./../baidu.html", "rb") html = file.read() bs = BeautifulSoup(html, "html.parser") # 1. tag:标签及其内容:(但是只是第一个标签) print(bs.title) print(bs.a) print(bs.a.attrs) # 得到标签的属性以字典的格式储存 print(type(bs.a)) # 2.NavigableString:标签里的内容 print(bs.title.string) print(type(bs.title.string)) # 3.BeautifulSoup:自身，整个文档 print(bs.name) print(type(bs)) # 4.Comment:注释:是一个特殊的NavigableString输出的内容不包含注释符号 print(bs.a.string) 文档的遍历和查找 # 文档的遍历:contents print(bs.head.contents[1]) print("---------------------------------") # 文档的搜索 # 字符串过滤:查找与字符串完全匹配的内容 a_list = bs.find_all("a") # 查找所有的a标签 print(a_list) print("---------------------------------") # 正则表达式搜索:使用search()方法来匹配 import re a_list1 = bs.find_all(re.compile("a")) print(a_list1) print("---------------------------------") # 使用方法来搜索:传入一个函数，根据函数的要求来搜索 def name_is_exists(tag): return tag.has_attr("name") a_list2 = bs.find_all(name_is_exists) print(a_list2) print("---------------------------------") # keywords:参数搜索 # a_list3 = bs.find_all(class_=True) a_list3 = bs.find_all(text="新闻") a_list3 = bs.find_all(text=re.compile("\d"),limit=2) # 根据正则表达式匹配内容，limit限制查找个数 for item in a_list3: print(item) print("---------------------------------") # 选择器类似css中的选择器 print(bs.select("title")) # 通过标签 print(bs.select(".classname")) # 通过类名 print(bs.select("#idname")) # 通过id查找 print(bs.select("a[class='classname']")) # 通过属性查找 print(bs.select("head>title")) # 通过子标签

正则表达式

参考文档链接：https://www.runoob.com/regexp/regexp-syntax.html

python中的re库 # 正则表达式：字符串匹配（判断字符串是否符合一定的标准） import re # 创建模式对象 pat = re.compile("AA") # 此处的AA是正则表达式 a = pat.search("AAA") # search字符串被校验的内容 print(a) a = re.search("abc", "aabcc") # 前面的字符串是规则，后面的字符串是被匹配的对象 print(a) a = re.findall("[a-z]+", "aABCXabuhua") # 找到所有的匹配项 print(a) a = re.sub("a", "A", "abcdefg") #找到a用A替换 print(a) # 建议在正则表达式中，被比较的字符串前面加上r，不用担心转义字符的问题

xlwt表格操作

简单操作 import xlwt work_book = xlwt.Workbook(encoding="utf-8") work_sheet = work_book.add_sheet("sheet1") work_sheet.write(0, 0, 'hello') # 参数：行、列、内容 work_book.save('student.xls')

sqlite3数据库操作

建表语句 import sqlite3 connet = sqlite3.connect("test.db") # 打开或创建文件 # 建表 c = connet.cursor() # 获取游标 sql = ''' create table if not exists doubanTop250 ( id integer not null primary key autoincrement, find_link text not null, find_image text, find_ctitle char(100), find_otitle char(100), find_score int(4), find_judge_number int(10), find_sign char(200), find_description text ) ''' c.execute(sql) # 执行sql语句 connet.commit() # 提交 connet.close() # 关闭数据库 插入语句 import sqlite3 connet = sqlite3.connect("test.db") # 打开或创建文件 # 建表 c = connet.cursor() # 获取游标 sql = ''' insert into doubanTop250 (find_link,find_image,find_ctitle,find_otitle,find_score,find_judge_number,find_sign,find_description) values ("1","1","1","1",1,1,"1","1"); ''' c.execute(sql) # 执行sql语句 connet.commit() # 提交 connet.close() # 关闭数据库 查询语句 import sqlite3 connet = sqlite3.connect("test.db") # 打开或创建文件 # 建表 c = connet.cursor() # 获取游标 sql = ''' select * from doubanTop250; ''' response = c.execute(sql) # 执行sql语句 for i in response: for j in range(0,8): print(i[j]) connet.commit() # 提交 connet.close() # 关闭数据库

获取数据

根据一个URL获取一个页面的html数据 # 得到一个指定页面的信息 def ask_url(url): head = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0" } """ head:模拟浏览器头部信息 "User-Agent":用户代理 """ request = urllib.request.Request(url=url, headers=head) html = "" try: response = urllib.request.urlopen(request) html = response.read().decode("utf-8") except Exception as e: if hasattr(e,"code"): print(e.code) if hasattr(e,"reason"): print(e.reason) return html

解析数据

豆瓣排名分析 # 定义一些全局变量 # 电影链接正则表达式 find_link = re.compile(r'<a href="(.*?)">') # 电影图片链接正则表达式,re.S忽略换行符 # find_image = re.compile(r'<img(.*)src="(.*?)" class="">', re.S) find_image = re.compile(r'<img.*src="(.*?)"', re.S) # 电影标题 find_title = re.compile(r'<span class="title">(.*)</span>') # 影片评分 find_score = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>') # 评价人数 find_judge_number = re.compile(r'<span>(\d*)人评价</span>') # 概况 find_sign = re.compile(r'<span class="inq">(.*)</span>') # 影片相关内容 find_description = re.compile(r'<p class="">(.*?)</p>', re.S) def get_data(url): datalist = [] for i in range(0, 10): # 循环获取10个页面数据 urls = url + str(i*25) html = ask_url(urls) # 保存网页信息 # 2.逐一解析数据 soup = BeautifulSoup(html, "html.parser") for item in soup.find_all('div', class_="item"): # 查找符合要求的字符串，形成列表 data = [] # 保存一部电影的所有信息 item = str(item) # 获取影片详情链接 link = re.findall(find_link, item)[0] # re库通过正则表达式查找指定字符串 data.append(link) img = re.findall(find_image, item)[0] data.append(img) title = re.findall(find_title, item) # 判断中文外文名 if len(title) == 2: ctitle = title[0] data.append(ctitle) otitle = title[1].replace("/","") # 去掉名称前的斜杠 data.append(otitle) else: data.append(title) data.append(" ") # 留空 score = re.findall(find_score, item)[0] data.append(score) judge_number = re.findall(find_judge_number, item)[0] data.append(judge_number) sign = re.findall(find_sign, item) if len(sign) != 0: data.append(sign[0].replace("。", "")) else: data.append(" ") description = re.findall(find_description, item)[0] description = re.sub('<br(\s+)?/>(\s+)?',"",description) # 去掉<br/> description = re.sub('/',"",description) data.append(description.strip()) # 去掉前后的空格 print(data) datalist.append(data) # 处理好的 return datalist

保存数据到表格中

# 保存数据 def sava_data(savepath, datalist): print('------------save------------') work_book = xlwt.Workbook(encoding="utf-8", style_compression=0) work_sheet = work_book.add_sheet("doubanTop250", cell_overwrite_ok=True) col = ("电影详情链接", "图片链接" , "影片中文名", "影片外文名", "评分", "评价数", "概况", "详情") for i in range(0, 8): work_sheet.write(0, i, col[i]) for i in range(0, 250): print("第%d条" %i) data = datalist[i] for j in range(0, 8): work_sheet.write(i+1, j, data[j]) work_book.save(savepath) # 保存数据

保存数据到数据库中

保存数据到数据库中 # 保存数据到sqlite def sava_data_sqlite(datalist): print("------------SQLite------------") init_sqlite() # 初始化数控库 # 插入数据 connet = sqlite3.connect("test.db") # 打开或创建文件 c = connet.cursor() # 获取游标 for data in datalist: for index in range(0, 8): if index not in (4, 5): data[index] = '"'+data[index]+'"' sql = ''' insert into doubanTop250(find_link,find_image,find_ctitle,find_otitle,find_score,find_judge_number,find_sign,find_description) values(%s)'''%",".join(data) print(sql) c.execute(sql) connet.commit() c.close() connet.close() # 建表语句 def init_sqlite(): connet = sqlite3.connect("test.db") # 打开或创建文件 # 建表 c = connet.cursor() # 获取游标 sql = ''' create table if not exists doubanTop250 ( id integer not null primary key autoincrement, find_link text not null, find_image text, find_ctitle char(100), find_otitle char(100), find_score int(4), find_judge_number int(10), find_sign char(200), find_description text ) ''' c.execute(sql) # 执行sql语句 connet.commit() # 提交 connet.close() # 关闭数据库

数据可视化

Flask框架

web框架

新建一个flask框架

app.py的测试文件： from flask import Flask, render_template, request import datetime app = Flask(__name__) # 通过访问路径，获取用户的字符串参数 @app.route('/index/<name>') def hello_word(name): return 'hello word %s'%name # 通过访问路径，获取用户的字符串参数 @app.route('/index/<int:id>') def hello_word2(id): return 'hello %d'%id # 返回给用户渲染后的网页文件 @app.route('/') def index(): time = datetime.date.today() # 向网页页面传递的参数 name = [1, 2, 3] # 列表类型 dic = {"name": "黄耀辉", "age": "18"} # 字典类型 return render_template("index.html", time=time, name=name, dic=dic) @app.route('/register') def register(): return render_template("register.html") @app.route('/result', methods=['POST']) def register_result(): if request.method == 'POST': result = request.form return render_template("result.html", result=result) else: return render_template("error.html") if __name__ == '__main__': app.run(debug=True) html中的数据操作 <body> <div>豆瓣爬虫</div> <div>时间:{{time}}</div> <div> 测试列表： {%for i in name%} <li> {{i}} </li> {% endfor %} </div> <div> 测试字典： <table> {% for key,value in dic.items() %} <tr> <td>{{key}}</td> <td>{{value}}</td> </tr> {% endfor %} </table> </div> </body> 表单提交 <form action="{{url_for('result')}}" method="post"> <p>姓名: <input type="text" name="name"></p> <p>性别: <input type="text" name="sex"></p> <p>年龄: <input type="text" name="age"></p> <p>地址: <input type="text" name="address"></p> <p><input type="submit"></p> </form>

Echarts应用

百度数据可视化应用 官网地址链接

WorldCloud应用

词云 官网地址地址

完整代码

app.py from flask import Flask, render_template, request import sqlite3 import jieba # 分词 from matplotlib import pyplot as plt # 绘图，数据可视化 from wordcloud import WordCloud # 词云 from PIL import Image # 图片处理 import numpy as np # 矩阵运算 app = Flask(__name__) @app.route('/') def home(): return render_template("home.html") @app.route('/movie') def movie(): movies = [] con = sqlite3.connect("./douban/test.db") cur = con.cursor() sql = "select * from doubanTop250" data = cur.execute(sql) for item in data: movies.append(item) cur.close() con.close() return render_template("movie.html", movies=movies) @app.route('/score') def score(): find_score = [] find_number = [] con = sqlite3.connect("./douban/test.db") cur = con.cursor() sql = "select find_score,count(find_score) from doubanTop250 group by find_score" data = cur.execute(sql) for item in data: find_score.append(item[0]) find_number.append(item[1]) cur.close() con.close() return render_template("score.html", score=find_score, number=find_number) @app.route('/word') def word(): # 获取数据 con = sqlite3.connect("./douban/test.db") cur = con.cursor() sql="select find_sign from doubanTop250" data = con.execute(sql) # 拼接 text = "" for item in data: text = text + item[0] cut = jieba.cut(text) string = " ".join(cut) cur.close() con.close() # 找到一张图片 img = Image.open("./static/timg.jpg") # 打开遮罩图片 img_array = np.array(img) # 将图片转换为数组 wold_cloud = WordCloud( background_color="#E4E7ED", mask=img_array, font_path="STKAITI.TTF", # 字体所在位置 ).generate_from_text(string) # 放入词 # 绘制图片 fig = plt.figure(1) plt.imshow(wold_cloud) plt.axis('off') # 不显示坐标轴 # plt.show() # 显示生成的词云图片 plt.savefig("./static/word.jpg", dpi=500) return render_template("word.html") @app.route('/author') def author(): return render_template("author.html") if __name__ == '__main__': app.run(debug=True) 前端页面home.html <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <title>豆瓣top260</title> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="description" content="Premium Bootstrap 4 Landing Page Template" /> <meta name="keywords" content="bootstrap 4, premium, marketing, multipurpose" /> <meta content="Shreethemes" name="author" /> <!-- favicon --> <link rel="shortcut icon" href="images/favicon.ico"> <!-- Bootstrap css --> <link href="static/css/bootstrap.min.css" rel="stylesheet" type="text/css" /> <!-- 图标库 --> <link href="static/css/materialdesignicons.min.css" rel="stylesheet" type="text/css" /> <link rel="stylesheet" href="static/css/unicons.css" /> <!-- Pe7 Icon --> <link rel="stylesheet" type="text/css" href="static/css/pe-icon-7.css"> <!-- Icons --> <link href="static/css/magnific-popup.css" rel="stylesheet" type="text/css" /> <!-- Main css File --> <link href="static/css/style.css" rel="stylesheet" type="text/css" /> <link href="static/css/default.css" rel="stylesheet" id="color-opt"> </head> <body> <!-- Navbar STart 导航栏--> <header id="topnav" class="defaultscroll sticky" style="background: #409EFF"> <div class="container"> <!-- Logo container--> <div> <a class="logo" href="/" target="_blank" style="color: #0b0b0b"> <!-- Landkey --> 豆瓣Top250 </a> </div> <div id="navigation"> <!-- Navigation Menu--> <ul class="navigation-menu"> <li class="has-submenu"> <a href="/">首页</a> </li> <li class="has-submenu"> <a href="/movie">电影</a> </li> <li class="has-submenu"> <a href="/score">评分</a> </li> <li class="has-submenu"> <a href="/word">词云</a> </li> <li class="has-submenu"> <a href="/author">作者</a> </li> </ul><!--end navigation menu--> </div><!--end navigation--> </div><!--end container--> </header><!--end header--> <!-- Navbar End --> <section class="section" id="price" style="background: #E4E7ED;min-height: 600px"> <div class="container"> <div class="row justify-content-center"> <div class="col-12"> <div class="section-title text-center mb-4 pb-2"> <h4 class="title mb-4">豆瓣电影Top250数据分析</h4> <p class="text-muted para-desc mx-auto mb-0">应用Python爬虫、Flask框架、Echarts、Word Cloud等技术实现</p> </div> </div><!--end col--> </div><!--end row--> <div class="row"> <div class="col-lg-3 col-md-6 col-12 mt-4 pt-2"> <a href="/movie"> <div class="card service-wrapper rounded border-0 shadow px-4 py-5"> <div class="icon text-center text-primary h1 shadow rounded bg-white"> <i class="uim uim-airplay"></i> </div> <div class="content mt-4"> <h5 class="title">经典电影</h5> </div> </div> </a> </div><!--end col--> <div class="col-lg-3 col-md-6 col-12 mt-4 pt-2"> <div class="card service-wrapper rounded border-0 shadow px-4 py-5"> <a href="/score"> <div class="icon text-center text-primary h1 shadow rounded bg-white"> <i class="uim uim-circle-layer"></i> </div> <div class="content mt-4"> <h5 class="title">评分统计</h5> </div> </a> </div> </div><!--end col--> <div class="col-lg-3 col-md-6 col-12 mt-4 pt-2"> <div class="card service-wrapper rounded border-0 shadow px-4 py-5"> <a href="/word"> <div class="icon text-center text-primary h1 shadow rounded bg-white"> <i class="uim uim-signal-alt-3"></i> </div> <div class="content mt-4"> <h5 class="title">词汇统计</h5> </div> </a> </div> </div><!--end col--> <div class="col-lg-3 col-md-6 col-12 mt-4 pt-2"> <div class="card service-wrapper rounded border-0 shadow px-4 py-5"> <a href="/author"> <div class="icon text-center text-primary h1 shadow rounded bg-white"> <i class="uim uim-flip-h-alt"></i> </div> <div class="content mt-4"> <h5 class="title">作者信息</h5> </div> </a> </div> </div><!--end col--> </div><!--end row--> </div><!--end container--> </section><!--end section--> <footer class="bg-dark footer-bar py-4"> <div class="container"> <div class="row justify-content-center"> <div class="col-12 text-center"> <p class="foot-color mb-0">persistenthuang@163.com</p> </div> </div> </div> </footer> <!-- javascript --> <script src="static/js/jquery.min.js"></script> <script src="static/js/bootstrap.bundle.min.js"></script> <script src="static/js/jquery.easing.min.js"></script> <script src="static/js/scrollspy.min.js"></script> <!-- Magnific popup --> <script src="static/js/jquery.magnific-popup.min.js"></script> <script src="static/js/magnific.init.js"></script> <!-- Parallax --> <script src="static/js/parallax.js"></script> <!-- 图标库 --> <script src="static/js/bundle.js"></script> <script src="static/js/feather.min.js"></script> <!-- Contact --> <script src="static/js/contact.js"></script> <!-- Main Js --> <script src="static/js/app.js"></script> </body> </html> move.html <!-- Start 评分 --> <section class="section" id="price" style="background: #E4E7ED;min-height: 600px"> <div class="container"> <div class="row justify-content-center"> <div class="col-12"> <div class="section-title text-center mb-4 pb-2"> <h4 class="title mb-4">豆瓣电影Top250电影</h4> </div> </div><!--end col--> </div><!--end row--> <!--表格--> <table class="table table-hover table-light"> <tr> <td>排名</td> <td>中文名称</td> <td>外文名称</td> <td>评分</td> <td>人数</td> <td>一句话描述</td> <td>其他信息</td> </tr> {%for movie in movies%} <tr> <td>{{movie[0]}}</td> <td> <a href="{{movie[1]}}" target="_blank"> {{movie[3]}} </a> </td> <td>{{movie[4]}}</td> <td>{{movie[5]}}</td> <td>{{movie[6]}}</td> <td>{{movie[7]}}</td> <td>{{movie[8]}}</td> </tr> {%endfor%} </table> </div><!--end container--> </section><!--end section--> <footer class="bg-dark footer-bar py-4"> <div class="container"> <div class="row justify-content-center"> <div class="col-12 text-center"> <p class="foot-color mb-0">persistenthuang@163.com</p> </div> </div> </div> </footer> score.html <!-- Start 评分 --> <section class="section" id="price" style="background: #E4E7ED;min-height: 600px"> <div class="container"> <div class="row justify-content-center"> <div class="col-12"> <div class="section-title text-center mb-4 pb-2"> <h4 class="title mb-4">豆瓣电影Top250评分分布图</h4> </div> </div><!--end col--> </div><!--end row--> <!-- 为 ECharts 准备一个具备大小（宽高）的 DOM --> <div id="main" style="width: 100%;height:450px;margin: 0 auto;"></div> </div><!--end container--> </section><!--end section--> <footer class="bg-dark footer-bar py-4"> <div class="container"> <div class="row justify-content-center"> <div class="col-12 text-center"> <p class="foot-color mb-0">persistenthuang@163.com</p> </div> </div> </div> </footer> <script type="text/javascript"> // 基于准备好的dom，初始化echarts实例 var myChart = echarts.init( document.getElementById('main')); var dataAxis = {{ score }}; var data = {{ number }}; var yMax = 50; var dataShadow = []; for (var i = 0; i < data.length; i++) { dataShadow.push(yMax); } option = { xAxis: { type: 'category', data: dataAxis, axisLabel: { inside: true, textStyle: { color: '#1c1b1b' } }, axisTick: { show: false }, axisLine: { show: false }, z: 10 }, yAxis: { axisLine: { show: false }, axisTick: { show: false }, axisLabel: { textStyle: { color: '#999' } } }, dataZoom: [ { type: 'inside' } ], series: [ { // For shadow type: 'bar', itemStyle: { color: 'rgba(0,0,0,0.05)' }, barGap: '-100%', barCategoryGap: '40%', data: dataShadow, animation: false }, { type: 'bar', itemStyle: { color: new echarts.graphic.LinearGradient( 0, 0, 0, 1, [ {offset: 0, color: '#83bff6'}, {offset: 0.5, color: '#188df0'}, {offset: 1, color: '#188df0'} ] ) }, emphasis: { itemStyle: { color: new echarts.graphic.LinearGradient( 0, 0, 0, 1, [ {offset: 0, color: '#2378f7'}, {offset: 0.7, color: '#2378f7'}, {offset: 1, color: '#83bff6'} ] ) } }, data: data } ] }; // Enable data zoom when user click bar. var zoomSize = 6; myChart.on('click', function (params) { console.log(dataAxis[Math.max(params.dataIndex - zoomSize / 2, 0)]); myChart.dispatchAction({ type: 'dataZoom', startValue: dataAxis[Math.max(params.dataIndex - zoomSize / 2, 0)], endValue: dataAxis[Math.min(params.dataIndex + zoomSize / 2, data.length - 1)] }); }); // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option); </script> word.html <section class="section" id="service" style="background: #E4E7ED;min-height: 600px"> <div class="container mt-60 mt-5"> <div class="row align-items-center"> <div class="col-lg-8 col-md-8"> <div class="mr-lg-5"> <img src="./../static/word.jpg" class="img-fluid" alt=""> </div> </div><!--end col--> <div class="col-lg-4 col-md-4 mt-4 mt-sm-0 pt-2 pt-sm-0"> <div class="section-title"> <h2 class="text-primary"> <i class="uim uim-google-play"></i> </h2> <h4 class="title mt-3 mb-4">词频统计</h4> <p class="text-blue para-desc">根据250部电影的一句话概述，提取的词云树，让我们了解一下经典电影都有什么相同点</p> </div> </div><!--end col--> </div><!--end row--> </div><!--end container--> </section><!--end section--> <!-- Services End --> <footer class="bg-dark footer-bar py-4"> <div class="container"> <div class="row justify-content-center"> <div class="col-12 text-center"> <p class="foot-color mb-0">persistenthuang@163.com</p> </div> </div> </div> </footer>