Python爬虫抓取B站热榜

环境

Python 3.7.4 bs4==0.0.1 pandas==1.0.1 urllib3==1.24.2 re

实现代码

# -*- coding:utf-8 -*- import re from bs4 import BeautifulSoup import urllib import pandas as pd class Spider(): ''' Description: Spider program to crawl data from bilibili.com hot search rank list Attributes: None ''' def __init__(self): self.url = 'https://www.bilibili.com/ranking?spm_id_from=333.851.b_7072696d61727950616765546162.3' # regular matching pattern self.pattern = re.compile(r'<a class="title" href="(.*?)" target="_blank">(.*?)</a>') self.headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"} ''' Description: crawl page from the given URL Args: url: the URL of page need to get Returns: page of the given URL ''' def crawl(self): self.request = urllib.request.Request(headers = self.headers, url = self.url) self.response = urllib.request.urlopen(self.request) page = self.response.read().decode('utf-8') return page ''' Description: extract data from the given page by bs4 and re library, return the list of data Args: None Returns: list of data extract from given page ''' def extract(self): page = self.crawl() beautifulSoup = BeautifulSoup(page, 'html.parser') results = [] for frame in beautifulSoup.find_all('div', class_ = 'content'): frame = str(frame) result = re.findall(self.pattern, frame)[0] results.append(result) return results if __name__ == "__main__": spider = Spider() results = spider.extract() print(pd.DataFrame(results))

测试结果

0 1 0 https://www.bilibili.com/video/BV1554y1k7DN 【原神】外 乡 人 1 https://www.bilibili.com/video/BV1JZ4y157R1 使命怎么又在召唤 2 https://www.bilibili.com/video/BV1Ta4y1777b 这动画也太沙雕了！2020十月新番吐槽大盘点！（第一弹） 3 https://www.bilibili.com/video/BV1DK4y187xe 每天一遍，防止........ 4 https://www.bilibili.com/video/BV1YT4y1A74y 【绘画教程合集】100P【日韩漫画】干货教程~（人体+造型+结构+眼睛+透视+手 脚+局部++... .. ... ... 95 https://www.bilibili.com/video/BV1WZ4y1577e 立意远大于内容，极其可惜的《姜子牙》【国动万象】 96 https://www.bilibili.com/video/BV1Wh41197Yx 【看片师】看完你就升华了！全方位暴锤三观的纪录片TOP10 97 https://www.bilibili.com/video/BV17v411y73z 帅小伙耗时七天自制糯米酒，香甜可口，比外面卖的还 好喝！ 98 https://www.bilibili.com/video/BV12t4y1q71T 【时代少年团】《少年ON FIRE》首场公演《做我的猫》纯享版 99 https://www.bilibili.com/video/BV1E5411j7oL 王老菊教你学习暴雪 [100 rows x 2 columns]

最后

由于博主水平有限，不免有疏漏之处，欢迎读者随时批评指正，以免造成不必要的误解！