2022年 11月 3日

十一个爆火的Python爬虫实战项目源码【不用谢】

目录

Python爬虫:爬取豆瓣电影中速度与激情8演员图片

Python爬虫:斗鱼弹幕相关信息保存到mongodb

Python爬虫：抓取喜马拉雅电台音频

Python爬虫—抓包分析爬取实习僧全部招聘信息

Python爬虫：批量抓取花瓣网高清美图并保存

Python爬虫：爬取v2ex数据用csv保存

Python爬虫：豌豆荚设计奖三种爬取方法速度对比

Python爬虫：使用lxml解析HTML，输出对应值

Python爬虫：使用Selenium爬取一点资讯动态数据

Python爬虫：Selenium+xpath+bs4爬取亚马逊数据保存到mongodb

Python爬虫：获取黑大验证码并登录

Python爬虫:爬取豆瓣电影中速度与激情8演员图片

  1. import urllib.request
  2. import os
  3. import re
  4.  
  5.  
  6. def douban(url):
  7.     r = urllib.request.urlopen(url)
  8.     html = r.read().decode('utf-8')
  9.     result = re.findall(r'https://img\d.doubanio.com/img/celebrity/medium/.*.jpg', html)
  10.     result2 = re.findall(r'(?<=title=").\S+', html)
  11.     result2.pop()
  12.     result3 = sorted(set(result2), key=result2.index)
  13.     result3.pop(-3)
  14.     if not os.path.exists('douban'):
  15.         os.makedirs('douban')
  16.     i = 0
  17.     for link in result:
  18.         filename = 'douban\\' + str(result3[i]) + '.jpg'
  19.         i += 1
  20.         with open(filename, 'w') as file:
  21.             urllib.request.urlretrieve(link, filename)
  22.  
  23.  
  24. url = 'https://movie.douban.com/subject/26260853/celebrities'
  25. if __name__ == '__main__':
  26.     douban(url)

Python爬虫:斗鱼弹幕相关信息保存到mongodb

  1. # 这个抓取弹幕,然后把用户的uid，昵称，等级，弹幕内容都保存到mongodb中
  2. __author__ = '布咯咯_rieuse'
  3. __time__ = '2017.6.2'
  4. __github__ = 'https://github.com/rieuse'
  5.  
  6. import multiprocessing
  7. import re
  8. import socket
  9. import time
  10.  
  11. import pymongo
  12. import requests
  13. from bs4 import BeautifulSoup
  14.  
  15. clients = pymongo.MongoClient('localhost')
  16. db = clients["DouyuTV_danmu"]
  17. col = db["info"]
  18.  
  19. client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  20. host = socket.gethostbyname("openbarrage.douyutv.com")
  21. port = 8601
  22. client.connect((host, port))
  23.  
  24. danmu_path = re.compile(b'txt@=(.+?)/cid@')
  25. uid_path = re.compile(b'uid@=(.+?)/nn@')
  26. nickname_path = re.compile(b'nn@=(.+?)/txt@')
  27. level_path = re.compile(b'level@=([1-9][0-9]?)/sahf')
  28.  
  29.  
  30. def sendmsg(msgstr):
  31.     msg = msgstr.encode('utf-8')
  32.     data_length = len(msg) + 8
  33.     code = 689
  34.     msgHead = int.to_bytes(data_length, 4, 'little') \
  35.               + int.to_bytes(data_length, 4, 'little') + int.to_bytes(code, 4, 'little')
  36.     client.send(msgHead)
  37.     sent = 0
  38.     while sent < len(msg):
  39.         tn = client.send(msg[sent:])
  40.         sent = sent + tn
  41.  
  42.  
  43. def start(roomid):
  44.     msg = 'type@=loginreq/username@=rieuse/password@=douyu/roomid@={}/\0'.format(roomid)
  45.     sendmsg(msg)
  46.     msg_more = 'type@=joingroup/rid@={}/gid@=-9999/\0'.format(roomid)
  47.     sendmsg(msg_more)
  48.  
  49.     print('---------------欢迎连接到{}的直播间---------------'.format(get_name(roomid)))
  50.     while True:
  51.         data = client.recv(1024)
  52.         uid_more = uid_path.findall(data)
  53.         nickname_more = nickname_path.findall(data)
  54.         level_more = level_path.findall(data)
  55.         danmu_more = danmu_path.findall(data)
  56.         if not level_more:
  57.             level_more = b'0'
  58.         if not data:
  59.             break
  60.         else:
  61.             for i in range(0, len(danmu_more)):
  62.                 try:
  63.                     product = {
  64.                         'uid': uid_more[0].decode(encoding='utf-8'),
  65.                         'nickname': nickname_more[0].decode(encoding='utf-8'),
  66.                         'level': level_more[0].decode(encoding='utf-8'),
  67.                         'danmu': danmu_more[0].decode(encoding='utf-8')
  68.                     }
  69.                     print(product)
  70.                     col.insert(product)
  71.                     print('成功导入mongodb')
  72.                 except Exception as e:
  73.                     print(e)
  74.  
  75.  
  76. def keeplive():
  77.     while True:
  78.         msg = 'type@=keeplive/tick@=' + str(int(time.time())) + '/\0'
  79.         sendmsg(msg)
  80.         time.sleep(15)
  81.  
  82.  
  83. def get_name(roomid):
  84.     r = requests.get("http://www.douyu.com/" + roomid)
  85.     soup = BeautifulSoup(r.text, 'lxml')
  86.     return soup.find('a', {'class', 'zb-name'}).string
  87.  
  88.  
  89. if __name__ == '__main__':
  90.     room_id = input('请出入房间ID： ')
  91.     p1 = multiprocessing.Process(target=start, args=(room_id,))
  92.     p2 = multiprocessing.Process(target=keeplive)
  93.     p1.start()
  94.     p2.start()

Python爬虫：抓取喜马拉雅电台音频

  1. _author__ = '布咯咯_rieuse'
  2.  
  3. import json
  4. import random
  5. import time
  6. import pymongo
  7. import requests
  8. from bs4 import BeautifulSoup
  9. from lxml import etree
  10.  
  11. clients = pymongo.MongoClient('localhost')
  12. db = clients["XiMaLaYa"]
  13. col1 = db["album2"]
  14. col2 = db["detaile2"]
  15.  
  16. UA_LIST = [
  17.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
  18.     "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
  19.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
  20.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
  21.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
  22.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
  23.     "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
  24.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  25.     "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  26.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
  27.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  28.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  29.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  30.     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  31.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
  32.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  33.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  34.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
  35.     "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
  36.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
  37.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
  38.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
  39.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
  40.     "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
  41.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  42.     "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  43.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  44.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  45.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  46.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  47.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  48.     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  49.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
  50.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  51.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
  52. ]
  53. headers1 = {
  54.     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
  55.     'Accept-Encoding': 'gzip, deflate, sdch',
  56.     'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
  57.     'Cache-Control': 'max-age=0',
  58.     'Proxy-Connection': 'keep-alive',
  59.     'Upgrade-Insecure-Requests': '1',
  60.     'User-Agent': random.choice(UA_LIST)
  61. }
  62. headers2 = {
  63.     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
  64.     'Accept-Encoding': 'gzip, deflate, sdch',
  65.     'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
  66.     'Cache-Control': 'max-age=0',
  67.     'Proxy-Connection': 'keep-alive',
  68.     'Referer': 'http://www.ximalaya.com/dq/all/2',
  69.     'Upgrade-Insecure-Requests': '1',
  70.     'User-Agent': random.choice(UA_LIST)
  71. }
  72.  
  73.  
  74. def get_url():
  75.     start_urls = ['http://www.ximalaya.com/dq/all/{}'.format(num) for num in range(1, 85)]
  76.     for start_url in start_urls:
  77.         html = requests.get(start_url, headers=headers1).text
  78.         soup = BeautifulSoup(html, 'lxml')
  79.         for item in soup.find_all(class_="albumfaceOutter"):
  80.             content = {
  81.                 'href': item.a['href'],
  82.                 'title': item.img['alt'],
  83.                 'img_url': item.img['src']
  84.             }
  85.             col1.insert(content)
  86.             print('写入一个频道' + item.a['href'])
  87.             print(content)
  88.             another(item.a['href'])
  89.         time.sleep(1)
  90.  
  91.  
  92. def another(url):
  93.     html = requests.get(url, headers=headers2).text
  94.     ifanother = etree.HTML(html).xpath('//div[@class="pagingBar_wrapper"]/a[last()-1]/@data-page')
  95.     if len(ifanother):
  96.         num = ifanother[0]
  97.         print('本频道资源存在' + num + '个页面')
  98.         for n in range(1, int(num)):
  99.             print('开始解析{}个中的第{}个页面'.format(num, n))
  100.             url2 = url + '?page={}'.format(n)
  101.             get_m4a(url2)
  102.     get_m4a(url)
  103.  
  104.  
  105. def get_m4a(url):
  106.     time.sleep(1)
  107.     html = requests.get(url, headers=headers2).text
  108.     numlist = etree.HTML(html).xpath('//div[@class="personal_body"]/@sound_ids')[0].split(',')
  109.     for i in numlist:
  110.         murl = 'http://www.ximalaya.com/tracks/{}.json'.format(i)
  111.         html = requests.get(murl, headers=headers1).text
  112.         dic = json.loads(html)
  113.         col2.insert(dic)
  114.         print(murl + '中的数据已被成功插入mongodb')
  115.  
  116.  
  117. if __name__ == '__main__':
  118.     get_url()

Python爬虫—抓包分析爬取实习僧全部招聘信息

  1. import json
  2. import requests
  3. import pymongo
  4. import time
  5.  
  6. clients = pymongo.MongoClient('localhost')
  7. db = clients["Shixiseng"]
  8. col = db["detail_info"]
  9.  
  10. urls = ['http://www.shixiseng.com/app/internsvt?c=%E5%85%A8%E5%9B%BD&p={}&t=hot'.format(n) for n in range(1, 3487)]
  11. for url in urls:
  12.     print(url)
  13.     r = requests.get(url)
  14.     html = r.content.decode('utf-8')
  15.     content = json.loads(html)['msg']['b']
  16.     for i in content:
  17.         print('插入一条数据：')
  18.         print(i)
  19.         col.insert(i)
  20.     time.sleep(0.01)

Python爬虫：批量抓取花瓣网高清美图并保存

  1. __author__ = '布咯咯_rieuse'
  2.  
  3. import os
  4. import lxml.html
  5. import requests
  6. from selenium import webdriver
  7. from selenium.webdriver.common.by import By
  8. from selenium.webdriver.support import expected_conditions as EC
  9. from selenium.webdriver.support.ui import WebDriverWait
  10.  
  11. SERVICE_ARGS = ['--load-images=false', '--disk-cache=true']
  12. browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)
  13. # browser = webdriver.Firefox()
  14. wait = WebDriverWait(browser, 5)
  15. browser.set_window_size(1400, 900)
  16.  
  17.  
  18. def parser(url, param):
  19.     # 解析模块
  20.     browser.get(url)
  21.     wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, param)))
  22.     html = browser.page_source
  23.     doc = lxml.html.fromstring(html)
  24.     return doc
  25.  
  26.  
  27. def get_main_url():
  28.     print('打开主页搜寻链接中...')
  29.     try:
  30.         doc = parser('http://huaban.com/boards/favorite/beauty/', '#waterfall')
  31.         name = doc.xpath('//*[@id="waterfall"]/div/a[1]/div[2]/h3/text()')
  32.         u = doc.xpath('//*[@id="waterfall"]/div/a[1]/@href')
  33.         for item, fileName in zip(u, name):
  34.             main_url = 'http://huaban.com' + item
  35.             print('主链接已找到' + main_url)
  36.             if '*' in fileName:
  37.                 fileName = fileName.replace('*', '')
  38.             download(main_url, fileName)
  39.     except Exception as e:
  40.         print(e)
  41.  
  42.  
  43. def download(main_url, fileName):
  44.     print('-------准备下载中-------')
  45.     try:
  46.         doc = parser(main_url, '#waterfall')
  47.         if not os.path.exists('image\\' + fileName):
  48.             print('创建文件夹...')
  49.             os.makedirs('image\\' + fileName)
  50.         link = doc.xpath('//*[@id="waterfall"]/div/a/@href')
  51.         # print(link)
  52.         i = 0
  53.         for item in link:
  54.             i += 1
  55.             minor_url = 'http://huaban.com' + item
  56.             doc = parser(minor_url, '#pin_view_page')
  57.             img_url = doc.xpath('//*[@id="baidu_image_holder"]/a/img/@src')
  58.             img_url2 = doc.xpath('//*[@id="baidu_image_holder"]/img/@src')
  59.             img_url += img_url2
  60.             try:
  61.                 url = 'http:' + str(img_url[0])
  62.                 print('正在下载第' + str(i) + '张图片，地址：' + url)
  63.                 r = requests.get(url)
  64.                 filename = 'image\\{}\\'.format(fileName) + str(i) + '.jpg'
  65.                 with open(filename, 'wb') as fo:
  66.                     fo.write(r.content)
  67.             except Exception:
  68.                 print('出错了！')
  69.     except Exception:
  70.         print('出错啦!')
  71.  
  72.  
  73. if __name__ == '__main__':
  74.     get_main_url()

Python爬虫：爬取v2ex数据用csv保存

  1. import csv, requests, re
  2. from bs4 import BeautifulSoup
  3.  
  4. url = 'https://www.v2ex.com/?tab=all'
  5. html = requests.get(url).text
  6. soup = BeautifulSoup(html, 'html.parser')
  7. articles = []
  8. for article in soup.find_all(class_='cell item'):
  9.     title = article.find(class_='item_title').get_text()
  10.     category = article.find(class_='node').get_text()
  11.     author = re.findall(r'(?<=<a href="/member/).+(?="><img)', str(article))[0]
  12.     u = article.select('.item_title > a')
  13.     link = 'https://www.v2ex.com' + re.findall(r'(?<=href=").+(?=")', str(u))[0]
  14.     articles.append([title, category, author, link])
  15.  
  16. with open(r'document\v2ex.csv', 'w') as f:
  17.     writer = csv.writer(f)
  18.     writer.writerow(['文章标题', '分类', '作者', '文章地址'])
  19.     for row in articles:
  20.         writer.writerow(row)

Python爬虫：豌豆荚设计奖三种爬取方法速度对比

  1. __author__ = '布咯咯_rieuse'
  2.  
  3. import asyncio
  4. import random
  5. import time
  6. import aiohttp
  7. import pymongo
  8. import requests
  9. import multiprocessing
  10. from bs4 import BeautifulSoup
  11.  
  12. # 共用部分
  13. clients = pymongo.MongoClient('localhost')
  14. db = clients["wandoujia"]
  15. col = db["info"]
  16.  
  17. urls = ['http://www.wandoujia.com/award?page={}'.format(num) for num in range(1, 46)]
  18. UA_LIST = [
  19.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
  20.     "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
  21.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
  22.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
  23.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
  24.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
  25.     "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
  26.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  27.     "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  28.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
  29.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  30.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  31.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  32.     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  33.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
  34.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  35.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  36.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
  37.     "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
  38.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
  39.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
  40.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
  41.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
  42.     "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
  43.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  44.     "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  45.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
  46.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  47.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
  48.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  49.     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  50.     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
  51.     "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
  52.     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
  53.     "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
  54. ]
  55. headers = {
  56.     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
  57.     'Accept-Encoding': 'gzip, deflate, sdch',
  58.     'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
  59.     'Connection': 'keep-alive',
  60.     'Host': 'www.wandoujia.com',
  61.     'User-Agent': random.choice(UA_LIST)
  62. }
  63.  
  64. proxies = {
  65.     'http': 'http://123.206.6.17:3128',
  66.     'https': 'http://123.206.6.17:3128'
  67. }
  68.  
  69.  
  70. # 方式一：使用常见的requests
  71. def method_1():
  72.     start = time.time()
  73.     for url in urls:
  74.         html = requests.get(url, headers=headers, proxies=proxies).text
  75.         soup = BeautifulSoup(html, 'lxml')
  76.         title = soup.find_all(class_='title')
  77.         app_title = soup.find_all(class_='app-title')
  78.         item_cover = soup.find_all(class_='item-cover')
  79.         icon_cover = soup.select('div.list-wrap > ul > li > div.icon > img')
  80.         for title_i, app_title_i, item_cover_i, icon_cover_i in zip(title, app_title, item_cover, icon_cover):
  81.             content = {
  82.                 'title': title_i.get_text(),
  83.                 'app_title': app_title_i.get_text(),
  84.                 'item_cover': item_cover_i['data-original'],
  85.                 'icon_cover': icon_cover_i['data-original']
  86.             }
  87.             col.insert(content)
  88.             print('成功插入一组数据' + str(content))
  89.     print('一共用时：' + str(time.time() - start))
  90.  
  91.  
  92. # if __name__ == '__main__':
  93. #     method_1()
  94.  
  95.  
  96.  
  97.  
  98.  
  99. # 方式二：使用Requests + Pool
  100. def method_2(url):
  101.     html = requests.get(url, headers=headers, proxies=proxies).text
  102.     soup = BeautifulSoup(html, 'lxml')
  103.     title = soup.find_all(class_='title')
  104.     app_title = soup.find_all(class_='app-title')
  105.     item_cover = soup.find_all(class_='item-cover')
  106.     icon_cover = soup.select('div.list-wrap > ul > li > div.icon > img')
  107.     for title_i, app_title_i, item_cover_i, icon_cover_i in zip(title, app_title, item_cover, icon_cover):
  108.         content = {
  109.             'title': title_i.get_text(),
  110.             'app_title': app_title_i.get_text(),
  111.             'item_cover': item_cover_i['data-original'],
  112.             'icon_cover': icon_cover_i['data-original']
  113.         }
  114.         # time.sleep(1)
  115.         col.insert(content)
  116.         print('成功插入一组数据' + str(content))
  117.  
  118.  
  119. # if __name__ == '__main__':
  120. #     start = time.time()
  121. #     pool = multiprocessing.Pool(4)
  122. #     pool.map(method_2, urls)
  123. #     pool.close()
  124. #     pool.join()
  125. #     print('一共用时：' + str(time.time() - start))
  126.  
  127.  
  128. # 方式三：使用Asyncio + Aiohttp python3.4之后出的异步io模块
  129.  
  130. def method_3():
  131.     async def get_url(url):
  132.         async with aiohttp.ClientSession() as session:  # async关键字将一个函数声明为协程函数，函数执行时返回一个协程对象。
  133.             async with session.get(url) as html:
  134.                 response = await html.text(encoding="utf-8")  # await关键字将暂停协程函数的执行，等待异步IO返回结果。
  135.                 return response
  136.  
  137.     async def parser(url):
  138.         html = await get_url(url)
  139.         soup = BeautifulSoup(html, 'lxml')
  140.         title = soup.find_all(class_='title')
  141.         app_title = soup.find_all(class_='app-title')
  142.         item_cover = soup.find_all(class_='item-cover')
  143.         icon_cover = soup.select('div.list-wrap > ul > li > div.icon > img')
  144.         for title_i, app_title_i, item_cover_i, icon_cover_i in zip(title, app_title, item_cover, icon_cover):
  145.             content = {
  146.                 'title': title_i.get_text(),
  147.                 'app_title': app_title_i.get_text(),
  148.                 'item_cover': item_cover_i['data-original'],
  149.                 'icon_cover': icon_cover_i['data-original']
  150.             }
  151.             col.insert(content)
  152.             print('成功插入一组数据' + str(content))
  153.  
  154.     start = time.time()
  155.     loop = asyncio.get_event_loop()
  156.     tasks = [parser(url) for url in urls]
  157.     loop.run_until_complete(asyncio.gather(*tasks))
  158.     print(time.time() - start)
  159.  
  160. if __name__ == '__main__':
  161.     method_3()
  162. © 2021 GitHub, Inc.

Python爬虫：使用lxml解析HTML，输出对应值

  1. import requests
  2. import lxml.html
  3.  
  4. url = 'http://news.ifeng.com/listpage/11502/0/1/rtlist.shtml'
  5. html = requests.get(url).text
  6. doc = lxml.html.fromstring(html)
  7. titles = doc.xpath('//div[@class="newsList"]/ul/li/a/text()')
  8. href = doc.xpath('//div[@class="newsList"]/ul/li/a/@href')
  9. i = 0
  10. for content in titles:
  11.     results = {
  12.         '标题': titles[i],
  13.         '链接': href[i]
  14.     }
  15.     i += 1
  16.     print(results)

Python爬虫：使用Selenium爬取一点资讯动态数据

  1. from selenium.webdriver.common.keys import Keys
  2. from selenium import webdriver
  3. from bs4 import BeautifulSoup
  4. import csv
  5.  
  6. driver = webdriver.Firefox()
  7. driver.implicitly_wait(3)
  8. first_url = 'http://www.yidianzixun.com/channel/c6'
  9. driver.get(first_url)
  10. driver.find_element_by_class_name('icon-refresh').click()
  11. for i in range(1, 90):
  12.     driver.find_element_by_class_name('icon-refresh').send_keys(Keys.DOWN)
  13. soup = BeautifulSoup(driver.page_source, 'lxml')
  14. articles = []
  15. for article in soup.find_all(class_='item doc style-small-image style-content-middle'):
  16.     title = article.find(class_='doc-title').get_text()
  17.     source = article.find(class_='source').get_text()
  18.     comment = article.find(class_='comment-count').get_text()
  19.     link = 'http://www.yidianzixun.com' + article.get('href')
  20.     articles.append([title, source, comment, link])
  21. driver.quit()
  22. with open(r'document\yidian.csv', 'w') as f:
  23.     writer = csv.writer(f)
  24.     writer.writerow(['文章标题', '作者', '评论数', '文章地址'])
  25.     for row in articles:
  26.         writer.writerow(row)

Python爬虫：Selenium+xpath+bs4爬取亚马逊数据保存到mongodb

  1. from selenium.common.exceptions import TimeoutException
  2. from selenium.webdriver.common.by import By
  3. from selenium.webdriver.support import expected_conditions as EC
  4. from selenium.webdriver.support.ui import WebDriverWait
  5. from selenium import webdriver
  6. from bs4 import BeautifulSoup
  7. import lxml.html
  8. import pymongo
  9. import re
  10.  
  11. MONGO_URL = 'localhost'
  12. MONGO_DB = 'amazon'
  13. MONGO_TABLE = 'amazon-python'
  14. SERVICE_ARGS = ['--load-images=false', '--disk-cache=true']
  15. KEYWORD = 'python'
  16. client = pymongo.MongoClient(MONGO_URL)
  17. db = client[MONGO_DB]
  18.  
  19. browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)
  20. # browser = webdriver.Firefox()
  21. wait = WebDriverWait(browser, 10)
  22. browser.set_window_size(1400, 900)
  23.  
  24.  
  25. def search():
  26.     print('正在搜索')
  27.     try:
  28.         browser.get('https://www.amazon.cn/')
  29.         input = wait.until(
  30.             EC.presence_of_element_located((By.CSS_SELECTOR, '#twotabsearchtextbox'))
  31.         )
  32.         submit = wait.until(
  33.             EC.element_to_be_clickable((By.CSS_SELECTOR, '#nav-search > form > div.nav-right > div > input')))
  34.         input.send_keys(KEYWORD)
  35.         submit.click()
  36.         total = wait.until(
  37.             EC.presence_of_element_located((By.CSS_SELECTOR, '#pagn > span.pagnDisabled')))
  38.         get_products()
  39.         print('一共' + total.text + '页')
  40.         return total.text
  41.     except TimeoutException:
  42.         return search()
  43.  
  44.  
  45. def next_page(number):
  46.     print('正在翻页', number)
  47.     try:
  48.         wait.until(EC.text_to_be_present_in_element(
  49.             (By.CSS_SELECTOR, '#pagnNextString'), '下一页'))
  50.         submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#pagnNextString')))
  51.         submit.click()
  52.         wait.until(EC.text_to_be_present_in_element(
  53.             (By.CSS_SELECTOR, '.pagnCur'), str(number)))
  54.         get_products()
  55.     except TimeoutException:
  56.         next_page(number)
  57.  
  58.  
  59. def get_products():
  60.     try:
  61.         wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#s-results-list-atf')))
  62.         html = browser.page_source
  63.         soup = BeautifulSoup(html, 'lxml')
  64.         doc = lxml.html.fromstring(html)
  65.         date = doc.xpath('//*[@class="s-result-item  celwidget "]/div/div[2]/div[1]/span[2]/text()')
  66.         content = soup.find_all(attrs={"id": re.compile(r'result_\d+')})
  67.         for item, time in zip(content, date):
  68.             product = {
  69.                 'title': item.find(class_='s-access-title').get_text(),
  70.                 'image': item.find(class_='s-access-image cfMarker').get('src'),
  71.                 'price': item.find(class_='a-size-base a-color-price s-price a-text-bold').get_text(),
  72.                 'date': time
  73.             }
  74.             # save_to_mongo(product)
  75.             print(product)
  76.     except Exception as e:
  77.         print(e)
  78.  
  79.  
  80. def save_to_mongo(result):
  81.     try:
  82.         if db[MONGO_TABLE].insert(result):
  83.             print('存储到mongodb成功', result)
  84.     except Exception:
  85.         print('存储到mongodb失败', result)
  86.  
  87.  
  88. def main():
  89.     try:
  90.         total = int(search())
  91.         for i in range(2, total + 1):
  92.             next_page(i)
  93.     except Exception as e:
  94.         print('出错啦', e)
  95.     finally:
  96.         browser.close()
  97.  
  98.  
  99. if __name__ == '__main__':
  100.     main()

Python爬虫：获取黑大验证码并登录

  1. import requests
  2. from PIL import Image
  3. from bs4 import BeautifulSoup
  4.  
  5. url1 = 'http://my.hlju.edu.cn/captchaGenerate.portal?'
  6. url2 = 'http://my.hlju.edu.cn/userPasswordValidate.portal'
  7. url3 = 'http://my.hlju.edu.cn/index.portal'
  8. headers = {
  9.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
  10. }
  11. s = requests.session()
  12. response = s.get(url1, headers=headers)
  13. html = response.text
  14. soup = BeautifulSoup(html, 'html.parser')
  15. with open('img\code.jpg', 'wb') as f:
  16.     f.write(response.content)
  17. img = Image.open('img\code.jpg')
  18. img.show()
  19. data = {}
  20. data['Login.Token1'] = '20154433'
  21. data['Login.Token2'] = ''
  22. data['captcha'] = input('输入验证码：')
  23. data['goto'] = 'http://my.hlju.edu.cn/loginSuccess.portal'
  24. data['gotoOnFail'] = 'http://my.hlju.edu.cn/loginFailure.portal'
  25. response2 = s.post(url=url2, data=data, headers=headers)
  26. response3 = s.get(url3, headers=headers)
  27. print(response3.text)