一、安装
1、通过requests 对响应内容进行处理,requests.get()方法会返回一个Response对象
pip install requests
2、beautifulSoup对网页解析不仅灵活、高效而且非常方便，支持多种解析器
pip install beautifulsoup4
3、pymongo是python操作mongo的工具包
pip install pymongo
4、安装mongo
二、分析网页&源代码
1、确定目标：首先要知道要抓取哪个页面的哪个版块
2、分析目标：确定抓取目标之后要分析URL链接格式以及拼接参数的含义其次还要分析页面源代码确定数据格式
3、编写爬虫代码 并 执行
三、编写代码
# -*- coding: utf-8 -*-# __author__ : "初一丶" 公众号:程序员共成长# __time__ : 2018/8/22 18:51# __file__ : spider_mayun.py# 导入相关库import requestsfrom bs4 import BeautifulSoupimport pymongo"""通过分析页面url 查询不同语言的热门信息是有language这个参数决定的"""# language = 'java'language = 'python'domain = 'https://gitee.com'uri = '/explore/starred?lang=%s' % languageurl = domain + uri# 用户代理user_agent = 'Mozilla/5.0 (Macintosh;Intel Mac OS X 10_12_6) ' \ 'AppleWebKit/537.36(KHTML, like Gecko) ' \ 'Chrome/67.0.3396.99Safari/537.36'# 构建headerheader = { 'User_Agent': user_agent}# 获取页面源代码html = requests.get(url, headers=header).text# 获取Beautiful对象soup = BeautifulSoup(html)# 热门类型分类 今日热门 本周热门 data-tab标签来区分当日热门和本周热门hot_type = ['today-trending', 'week-trending']# divs = soup.find_all('div', class_='ui tab active')# 创建热门列表hot_gitee = []for i in hot_type: # 通过热门标签 查询该热门下的数据 divs = soup.find_all('div', attrs={ 'data-tab': i}) divs = divs[0].select('div.row') for div in divs: gitee = {} a_content = div.select('div.sixteen > h3 > a') div_content = div.select('div.project-desc') # 项目描述 script = div_content[0].string # title属性 title = a_content[0]['title'] arr = title.split('/') # 作者名字 author_name = arr[0] # 项目名字 project_name = arr[1] # 项目url href = domain + a_content[0]['href'] # 进入热门项目子页面 child_page = requests.get(href, headers=header).text child_soup = BeautifulSoup(child_page) child_div = child_soup.find('div', class_='ui small secondary pointing menu') """ """ a_arr = child_div.findAll('a') # git http下载链接 http_url = a_arr[0]['data-url'] # git ssh下载链接 ssh_url = a_arr[1]['data-url'] gitee['project_name'] = project_name gitee['author_name'] = author_name gitee['href'] = href gitee['script'] = script gitee['http_url'] = http_url gitee['ssh_url'] = ssh_url gitee['hot_type'] = i # 连接mongo hot_gitee.append(gitee)print(hot_gitee)# 链接mongo参数HOST, PORT, DB, TABLE = '127.0.0.1', 27017, 'spider', 'gitee'# 创建链接client = pymongo.MongoClient(host=HOST, port=PORT)# 选定库db = client[DB]tables = db[TABLE]# 插入mongo库tables.insert_many(hot_gitee)
四、执行结果
[{ 'project_name': 'IncetOps', 'author_name': 'staugur', 'href': 'https://gitee.com/staugur/IncetOps', 'script': '基于Inception，一个审计、执行、回滚、统计sql的开源系统', 'http_url': 'https://gitee.com/staugur/IncetOps.git', 'ssh_url': 'git@gitee.com:staugur/IncetOps.git', 'hot_type': 'today-trending'}, { 'project_name': 'cms', 'author_name': 'dlg_center', 'href': 'https://gitee.com/dlg_center/cms', 'script': None, 'http_url': 'https://gitee.com/dlg_center/cms.git', 'ssh_url': 'git@gitee.com:dlg_center/cms.git', 'hot_type': 'today-trending'}, { 'project_name': 'WebsiteAccount', 'author_name': '张聪', 'href': 'https://gitee.com/crazy_zhangcong/WebsiteAccount', 'script': '各种问答平台账号注册', 'http_url': 'https://gitee.com/crazy_zhangcong/WebsiteAccount.git', 'ssh_url': 'git@gitee.com:crazy_zhangcong/WebsiteAccount.git', 'hot_type': 'today-trending'}, { 'project_name': 'chain', 'author_name': '何全', 'href': 'https://gitee.com/hequan2020/chain', 'script': 'linux 云主机 管理系统，包含 CMDB,webssh登录、命令执行、异步执行shell/python/yml等。持续更...', 'http_url': 'https://gitee.com/hequan2020/chain.git', 'ssh_url': 'git@gitee.com:hequan2020/chain.git', 'hot_type': 'today-trending'}, { 'project_name': 'Lepus', 'author_name': '茹憶。', 'href': 'https://gitee.com/ruzuojun/Lepus', 'script': '简洁、直观、强大的开源企业级数据库监控系统，MySQL/Oracle/MongoDB/Redis一站式监控，让数据库监控更简...', 'http_url': 'https://gitee.com/ruzuojun/Lepus.git', 'ssh_url': 'git@gitee.com:ruzuojun/Lepus.git', 'hot_type': 'today-trending'}, { 'project_name': 'AutoLink', 'author_name': '苦叶子', 'href': 'https://gitee.com/lym51/AutoLink', 'script': 'AutoLink是一个开源Web IDE自动化测试集成解决方案', 'http_url': 'https://gitee.com/lym51/AutoLink.git', 'ssh_url': 'git@gitee.com:lym51/AutoLink.git', 'hot_type': 'week-trending'}, { 'project_name': 'PornHubBot', 'author_name': 'xiyouMc', 'href': 'https://gitee.com/xiyouMc/pornhubbot', 'script': '全球最大成人网站PornHub爬虫 （Scrapy、MongoDB） 一天500w的数据', 'http_url': 'https://gitee.com/xiyouMc/pornhubbot.git', 'ssh_url': 'git@gitee.com:xiyouMc/pornhubbot.git', 'hot_type': 'week-trending'}, { 'project_name': 'wph_opc', 'author_name': '万屏汇', 'href': 'https://gitee.com/wph_it/wph_opc', 'script': None, 'http_url': 'https://gitee.com/wph_it/wph_opc.git', 'ssh_url': 'git@gitee.com:wph_it/wph_opc.git', 'hot_type': 'week-trending'}, { 'project_name': 'WebsiteAccount', 'author_name': '张聪', 'href': 'https://gitee.com/crazy_zhangcong/WebsiteAccount', 'script': '各种问答平台账号注册', 'http_url': 'https://gitee.com/crazy_zhangcong/WebsiteAccount.git', 'ssh_url': 'git@gitee.com:crazy_zhangcong/WebsiteAccount.git', 'hot_type': 'week-trending'}, { 'project_name': 'information27', 'author_name': '印妈妈', 'href': 'https://gitee.com/itcastyinqiaoyin/information27', 'script': None, 'http_url': 'https://gitee.com/itcastyinqiaoyin/information27.git', 'ssh_url': 'git@gitee.com:itcastyinqiaoyin/information27.git', 'hot_type': 'week-trending'}] 