101 lines
2.5 KiB
JavaScript
101 lines
2.5 KiB
JavaScript
|
||
const request = require('request');
|
||
const iconv = require('iconv-lite');
|
||
const cheerio = require('cheerio');
|
||
const fs = require('fs');
|
||
|
||
//网页基地址
|
||
var baseUrl = `https://pic.netbian.com`
|
||
|
||
//网页地址共1034页
|
||
//用于存放页面地址(一页一页的下载)
|
||
var pageArr = []
|
||
var pageIndex = 0
|
||
|
||
//用于存放详情页地址
|
||
var hrefArr = []
|
||
var hrefIndex = 0
|
||
|
||
|
||
//循环先爬3页
|
||
for (var i = 1; i <= 2; i++) {
|
||
if (i == 1) {
|
||
pageArr.push(`${baseUrl}/index.html`)
|
||
} else {
|
||
pageArr.push(`${baseUrl}/index_${i}.html`)
|
||
}
|
||
}
|
||
|
||
// console.log("pageArr:", pageArr);
|
||
|
||
|
||
function requestPage() {
|
||
|
||
//调用getHtml方法请求
|
||
getHtml(pageArr[pageIndex], ($) => {
|
||
//each遍历（下标，元素）
|
||
$(".slist li").each(function (index, element) {
|
||
|
||
//去除下一页图片（21下一页）总共20张图片
|
||
if (!$(this).hasClass("nextpage")) {
|
||
// console.log( $(this).find("a").attr("title"));
|
||
hrefArr.push(
|
||
//attr打印标签属性
|
||
//基地址 + 图片地址
|
||
`${baseUrl}${$(this).find("a").attr("href")}`
|
||
)
|
||
}
|
||
})
|
||
|
||
|
||
requestInfo()
|
||
// console.log("hrefArr:",hrefArr);
|
||
})
|
||
}
|
||
|
||
|
||
function requestInfo() {
|
||
getHtml(hrefArr[hrefIndex], ($) => {
|
||
|
||
//A 利用request去请求图片地址
|
||
request(`${baseUrl}${$(".photo-pic img").attr("src")}`)
|
||
//管道符: 将A的数据流入B
|
||
.pipe(
|
||
//B fs创建写入数据流 //图片命名方式，随机数＋时间戳避免名字重复
|
||
fs.createWriteStream(`./img/${ Math.random() * 1000000 + `${Date.now()}` }.jpg`)
|
||
)
|
||
|
||
// console.log(`${baseUrl}${$(".photo-pic img").attr("src")}`);
|
||
|
||
hrefIndex += 1
|
||
if(hrefIndex < hrefArr.length){
|
||
requestInfo()
|
||
}else{
|
||
pageIndex += 1
|
||
if(pageIndex < pageArr.length){
|
||
requestPage()
|
||
}
|
||
|
||
}
|
||
|
||
|
||
})
|
||
|
||
}
|
||
|
||
//每一页和每一页的每个图片，都需要发送请求，所以统一封装一下
|
||
|
||
function getHtml(url, callback) {
|
||
request({
|
||
uri: url,
|
||
method: 'GET',
|
||
encoding: 'binary'
|
||
//转码
|
||
}, function (error, response, body) {
|
||
var str2 = iconv.decode(new Buffer(body, 'binary'), 'gbk');
|
||
const $ = cheerio.load(str2)
|
||
callback($)
|
||
// console.log(str2);
|
||
});
|
||
}
|
||
requestPage() |