首页 > python爬虫

python爬虫实现获取下一页代码

时间:2020-08-02 python爬虫 查看: 1026

我们首先来看下实例代码:

from time import sleep

import faker
import requests
from lxml import etree

fake = faker.Faker()

base_url = "http://angelimg.spbeen.com"

def get_next_link(url):
  content = downloadHtml(url)
  html = etree.HTML(content)
  next_url = html.xpath("//a[@class='ch next']/@href")
  if next_url:
    return base_url + next_url[0]
  else:
    return False

def downloadHtml(ur):
  user_agent = fake.user_agent()
  headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"}
  response = requests.get(url, headers=headers)
  return response.text

def getImgUrl(content):
  html = etree.HTML(content)
  img_url = html.xpath('//*[@id="content"]/a/img/@src')
  title = html.xpath(".//div['@class=article']/h2/text()")

  return img_url[0],title[0]

def saveImg(title,img_url):
  if img_url is not None and title is not None:
    with open("txt/"+str(title)+".jpg",'wb') as f:
      user_agent = fake.user_agent()
      headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"}
      content = requests.get(img_url, headers=headers)
      #request_view(content)
      f.write(content.content)
      f.close()

def request_view(response):
  import webbrowser
  request_url = response.url
  base_url = '<head><base href="%s" rel="external nofollow" >' %(request_url)
  base_url = base_url.encode()
  content = response.content.replace(b"<head>",base_url)
  tem_html = open('tmp.html','wb')
  tem_html.write(content)
  tem_html.close()
  webbrowser.open_new_tab('tmp.html')

def crawl_img(url):
  content = downloadHtml(url)
  res = getImgUrl(content)
  title = res[1]
  img_url = res[0]
  saveImg(title,img_url)

if __name__ == "__main__":
  url = "http://angelimg.spbeen.com/ang/4968/1"

  while url:
    print(url)
    crawl_img(url)
    url = get_next_link(url)

python 爬虫如何执行自动下一页循环加载文字

from bs4 import BeautifulSoup
import requests
import time
from lxml import etree
import os
# 该demo执行的为如何利用bs去爬一些文字
def start():
  # 发起网络请求
  html=requests.get('http://www.baidu.com')
  #编码
  html.encoding=html.apparent_encoding
  #创建sp
  soup=BeautifulSoup(html.text,'html.parser')
  print(type(soup))
  print('打印元素')
  print(soup.prettify())
  #存储一下title 该方法没有提示直接展示
  title=soup.head.title.string
  print(title)
#   写入文本
  with open(r'C:/Users/a/Desktop/a.txt','w') as f:
    f.write(title)
  print(time.localtime())

url_2 = 'http://news.gdzjdaily.com.cn/zjxw/politics/sz_4.shtml'
def get_html_from_bs4(url):

  # response = requests.get(url,headers=data,proxies=ip).content.decode('utf-8')
  response = requests.get(url).content.decode('utf-8')
  soup = BeautifulSoup(response, 'html.parser')
  next_page = soup.select('#displaypagenum a:nth-of-type(9)')[0].get('href')
  # for i in nett
  print(next_page)
  next2='http://news.gdzjdaily.com.cn/zjxw/politics/'+next_page


def get_html_from_etree(url):

  response = requests.get(url).content.decode('utf-8')
  html= etree.HTML(response)

  next_page = html.xpath('.//a[@class="PageNum"][8]/@href')[0]
  print(next_page)
  # next2='http://news.gdzjdaily.com.cn/zjxw/politics/'+next_page


get_html_from_etree(url_2)

if __name__ == '__main__':
  start()

到此这篇关于python爬虫实现获取下一页代码的文章就介绍到这了,更多相关python爬虫获取下一页内容请搜索python博客以前的文章或继续浏览下面的相关文章希望大家以后多多支持python博客!

展开全文
上一篇:python numpy实现多次循环读取文件 等间隔过滤数据示例
下一篇:使用Python爬取弹出窗口信息的实例
输入字:
相关知识
Python爬虫基础之爬虫的分类知识总结

来给大家讲python爬虫的基础啦,首先我们从爬虫的分类开始讲起,下文有非常详细的知识总结,对正在学习python的小伙伴们很有帮助,需要的朋友可以参考下

Python爬虫基础讲解之请求

今天带大家了解一下python爬虫的基础知识,文中有非常详细的解释说明,对正在学习python爬虫的小伙伴们有很好地帮助,需要的朋友可以参考下

PyQt5爬取12306车票信息程序的实现

12306是学习爬虫的比较好的一个练手网站。本文主要实现了PyQt5爬取12306车票信息程序,具有一定的参考价值,感兴趣的小伙伴们可以参考一下

Python爬虫之m3u8文件里提取小视频的正确姿势

本文给大家分享如何正确提取m3u8文件里的.ts视频,并合成完整的.mp4格式视频,通过图文实例代码的形式给大家介绍的非常详细,对Python提取m3u8文件小视频感兴趣的朋友一起看看吧