1、环境

pycharm+selenium+pyquery+openpyxl+tkinter+tkinter.filedialog

2、功能说明

通过openpyxl读取关键词，爬取亚马逊指定关键词商品 的信息，并将获取到的信息通过openpyxl写入到excel中用户可自己选择文件打开和存储的路径用户可选择增加或删减查找的信息用户可选择配送地址

3、新增文件对话框模块

import tkinter as tk from tkinter import filedialog from tkinter import messagebox root = tk.Tk()# 创建一个Tkinter.Tk()实例 root.withdraw()# 将Tkinter.Tk()实例隐藏 # 参数相关： # defaultextension: 默认文件的扩展名,该后缀会自动添加 # filetypes: 指定筛选文件类型的下拉菜单，如filetypes=[(‘xlsx’,‘xlsx’)] # initialdir: 对话框中默认的路径（默认路径是当前文件夹） # initialfile: 对话框中初始化显示的文件名 # parent: 父对话框(由哪个窗口弹出就在哪个上端) # title: 弹出对话框的标题 # 选择文件后，会返回显示文件的完整路径，取消的话，会返回空字符串 file_path = filedialog.askopenfilename(initialdir='C:/Users/45906/Desktop', title='选择关键词存在文件', filetypes=[('Excel', '.xls .xlsx')]) print("askopenfilename = ", file_path) save_file = filedialog.asksaveasfilename(title='请选择生成文件存储路径', initialdir='C:/Users/45906/Desktop', filetypes=[('Excel', '.xls .xlsx')], defaultextension='.xlsx') print("asksaveasfilename = ", save_file) #提示框 messagebox.showinfo("提示","我是一个提示框") #警告框 messagebox.showwarning('警告','我是一个警告框') #错误框 messagebox.showerror('错误','我是一个错误框') #对话框,返回True/Fasle messagebox.askokcancel('提示', '我是一个对话框吗') #问题框,返回Yes/No messagebox.askquestion('提示', '我是一个问题框吗')

4、完整代码

import time from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.common.by import By from pyquery import PyQuery as Pq import openpyxl import tkinter as tk from tkinter import filedialog from tkinter import messagebox def change_address(value): count = 0 while True: count += 1 if count > 3: ok = messagebox.askokcancel('提示', '超时, 是否退出???') if ok: exit() try: driver.find_element_by_id('glow-ingress-line1').click() time.sleep(2) except Exception as e: print("get glow-ingress-line1 id fail!!!,retry") driver.refresh() time.sleep(5) continue try: driver.find_element_by_id("GLUXChangePostalCodeLink").click() time.sleep(2) except: print("get GLUXChangePostalCodeLink id fail!!!") pass try: driver.find_element_by_id('GLUXZipUpdateInput').send_keys(value) time.sleep(1) break except : driver.refresh() time.sleep(10) continue driver.find_element_by_id('GLUXZipUpdate').click() time.sleep(1) driver.refresh() time.sleep(3) def parse_detail(page_source, page_index, keyword): global write_row_index finish = False doc = Pq(page_source) # 判断是否为最后一页，最后一页就停止往下获取 last = doc('.a-disabled.a-last').text() if last.strip() != "": finish = True print("last text = ", last, " finish = ", finish) # 获取商品所有信息 main_list = doc('.s-main-slot.s-result-list.s-search-results.sg-row') par_list = main_list.children().items() normal_count = 0 for child in par_list: # flag_text为空的表示非正常商品链接 flag_text = child.attr('data-asin') if flag_text.strip() == "": continue normal_count += 1 # 是否为广告 sponsor = child('.s-label-popover-default .a-size-mini.a-color-secondary').text() # 标题 title = child('.a-size-base-plus.a-color-base.a-text-normal').text() # 价格，但是会有打折价格，所以多个价格只选第一个 price_list = child('.a-price .a-offscreen').text().split(" ") price = price_list[0] # 当前商品位置 cur_pos = str(page_index) + "-" + str(normal_count) # 评论数 review_num = child('.a-section.a-spacing-none.a-spacing-top-micro .a-row.a-size-small .a-size-base').text() # 评分 level = child('.a-icon-alt').text() # 配送地址 adds = child('.a-row.a-size-base.a-color-secondary.s-align-children-center .a-size-small.a-color-secondary').text() data_list = [] for val in global_split_list: if val == "标题": data_list.append(title) elif val == "价格": data_list.append(price) elif val == "自然位置": data_list.append(cur_pos) elif val == "评论数": data_list.append(review_num) elif val == "评分": data_list.append(level) elif val == "配送地址": data_list.append(adds) print(keyword, title, ", ", price, ", ", sponsor, ", ", cur_pos, ", ", level, ", ", review_num, ", ", adds) # 写入到excel write_row_index += 1 try: write_row(worksheet, write_row_index, data_list) except: pass print(normal_count) return finish def open_excel(): global global_save_path filepath = filedialog.askopenfilename(initialdir='C:/Users/45906/Desktop', title='选择关键词存在文件', filetypes=[('Excel', '.xls .xlsx')]) try: # 需要文件存在 open_wb = openpyxl.load_workbook(filepath) open_names = open_wb.sheetnames open_ws = open_wb[open_names[0]] # 获取第一行的所有列 max_rows = open_ws.max_row max_cols = open_ws.max_column rows = open_ws.iter_rows(1, 1, 1, max_cols) # 查找目标的列 key_col = -1 adds_col = -1 data_col = -1 for key_list in rows: for key in key_list: print(key.value, "============") if key.value == "关键词": key_col = key.column if key.value == "配送地址": adds_col = key.column if key.value == "筛选信息": data_col = key.column break print(max_rows, max_cols, key_col, adds_col, data_col) # 获取关键词列表 if key_col != -1: rows = open_ws.iter_rows(2, max_rows, key_col, key_col) for key_list in rows: for keyValue in key_list: if keyValue.value != None: global_key_list.append(keyValue.value) if len(global_key_list) <= 0: messagebox.showinfo("提示", "当前文件关键词为空！！！自动退出") exit() # 获取配送地址 if adds_col != -1: rows = open_ws.iter_rows(2, max_rows, adds_col, adds_col) for key_list in rows: for keyValue in key_list: if keyValue.value != None: global_adds_list.append(keyValue.value) # 筛选信息 if data_col != -1: rows = open_ws.iter_rows(2, max_rows, data_col, data_col) for key_list in rows: for keyValue in key_list: if keyValue.value != None: global_split_list.append(keyValue.value) except: messagebox.showinfo("提示", "当前文件读取内容时发生异常！！！自动退出") exit() try: while True: save_file = filedialog.asksaveasfilename(title='请选择生成文件存储路径', initialdir='C:/Users/45906/Desktop', filetypes=[('Excel', '.xls .xlsx')], defaultextension='.xlsx') if save_file == filepath: isOk = messagebox.askokcancel('提示', '当前选择会覆盖关键词文件,确定执行???') if (isOk): global_save_path = save_file break else: continue else: global_save_path = save_file break except: messagebox.showinfo("提示", "当前选取文件保存路径时发生异常！！！自动退出") exit() def write_row(ws, row, data_list): cnt = len(data_list) for col in range(0, cnt): ws.cell(row, col + 1, data_list[col]) if __name__ == '__main__': # 创建一个Tkinter.Tk()实例 root = tk.Tk() # 将Tkinter.Tk()实例隐藏 root.withdraw() # 获取关键词 global_key_list = [] # 配送地址 global_adds_list = [] # 筛选信息 global_split_list = [] # 保存的文件地址 global_save_path = "" write_row_index = 1 open_excel() print("global_key_list = ", global_key_list) print("global_adds_list = ", global_adds_list) print("global_split_list = ", global_split_list) print("global_save_path = ", global_save_path) if len(global_split_list) <= 0: global_split_list = ['标题', '价格', '模式', '自然位置'] workbook = openpyxl.Workbook() sheet_names = workbook.sheetnames worksheet = workbook[sheet_names[0]] write_row(worksheet, write_row_index, global_split_list) # 设置get直接返回，不再等待界面加载完成 desired_capabilities = DesiredCapabilities.CHROME desired_capabilities["pageLoadStrategy"] = "none" chrome_options = webdriver.ChromeOptions() # 无窗口模式 # chrome_options.add_argument('--headless') # 禁止硬件加速，避免严重占用cpu chrome_options.add_argument('--disable-gpu') # 关闭安全策略 chrome_options.add_argument("disable-web-security") # 禁止图片加载 chrome_options.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2}) # 隐藏Chrome正在受到自动软件的控制 chrome_options.add_argument('disable-infobars') # 设置开发者模式启动，该模式下webdriver属性为正常值 chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 模拟移动设备 chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"') driver = webdriver.Chrome(options=chrome_options) # 返回驱动等待的变量 wait = WebDriverWait(driver, 20) # driver.maximize_window() print(time.strftime("start %Y-%m-%d %H:%M:%S", time.localtime())) search_url = 'https://www.amazon.com/' if len(global_adds_list) >= 1: driver.get(search_url) time.sleep(3) change_address(global_adds_list[0]) # 固定搜索内容，变化的只有页面 search_page_url = 'https://www.amazon.com/s?k={}&page={}' for key in global_key_list: for i in range(1, 10): # 最大往下爬取10页 print("正在爬取", search_page_url.format(key, i)) driver.get(search_page_url.format(key, i)) time.sleep(3) # css选择器，返回结果存在跳出，异常报错 try: wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, "div.s-result-list"))) isEnd = parse_detail(driver.page_source, i, key) if isEnd: break except: print("url: " + search_page_url.format(i) + "获取失败") pass print(time.strftime("end %Y-%m-%d %H:%M:%S", time.localtime())) driver.quit() try: # 注意 excel被手动打开后，保存会失败 workbook.save(global_save_path) workbook.close() except: pass

5、结果

我的可供选择的筛选信息只有：标题、价格、自然位置、评论数、评分、配送地址，所以只要在这个设置下，可以获取到内容

简单的亚马逊商品获取就到此为止了，完整代码就在上面，测试复制粘贴就可使用，因为所有的资源，教程都来自网上，所以我也将自己一步一步学的过程都贴了出来

抄袭：

python3爬虫实战之selenium爬取亚马逊商品

以及抄袭了分享了例如excel操作，文件对话框，xpath，pyquery，bs4，selenium等等的博主