81 lines
2.6 KiB
JavaScript
81 lines
2.6 KiB
JavaScript
import * as cheer from "cheerio";
|
||
import axios from "axios";
|
||
import { HttpsProxyAgent } from "https-proxy-agent";
|
||
import Config from "../config/config.js";
|
||
|
||
class utils {
|
||
constructor() {
|
||
this.ax = axios
|
||
// 让 ajax 请求 走flclash的翻墙代理
|
||
const httpsAgent = new HttpsProxyAgent(`http://127.0.0.1:${Config.proxyPort}`);
|
||
this.ax.defaults.httpsAgent = httpsAgent;
|
||
this.ax.defaults.proxy = false;
|
||
|
||
this.ResponseInterceptor()
|
||
}
|
||
|
||
/**
|
||
* 返回 cheerio 数据结构
|
||
* @param html
|
||
* @returns {*}
|
||
*/
|
||
cheerio(html) {
|
||
return cheer.load(html)
|
||
}
|
||
|
||
split(str, index = 1) {
|
||
return str.split(Config.baseUrl)[index]
|
||
}
|
||
|
||
/**
|
||
*
|
||
* @param url 请求的地址
|
||
* @returns {Promise<AxiosResponse<any>>}
|
||
*/
|
||
async get(url) {
|
||
return await this.ax.get(url, {
|
||
headers: {
|
||
'sec-ch-ua-mobile': '?1',
|
||
'sec-fetch-site': 'same-origin',
|
||
'upgrade-insecure-requests': '1',
|
||
'priority': 'u=0, i',
|
||
'sec-fetch-mode': 'navigate',
|
||
'cookie': `_ga=GA1.1.1571194344.1732275775; _lscache_vary=0e8aace2958628a1edb01710bbabbcab; showed_system_notice=showed; PHPSESSID=5jmev023fsh6i76huat60njio3; _ga_ZT9Q6FCC5R=GS1.1.1732668048.9.1.1732668791.0.0.0`,
|
||
'referer': 'https://www.06se.com/hot_tags',
|
||
'user-agent': `Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1`
|
||
}
|
||
});
|
||
}
|
||
|
||
maxPage(el, $) {
|
||
let pageArr = []
|
||
$(el).each(function() {
|
||
let page = Number($(this).text())
|
||
page ? pageArr.push(page) : null
|
||
})
|
||
return Math.max(...pageArr)
|
||
}
|
||
|
||
/**
|
||
*
|
||
* 响应拦截器
|
||
* @constructor
|
||
*/
|
||
async ResponseInterceptor() {
|
||
this.ax.interceptors.response.use((response) => {
|
||
return this.cheerio(response.data);
|
||
}, (error) => {
|
||
console.log(`
|
||
❌ 请求错误❕❕❕
|
||
1: 错误信息 ${error.message}
|
||
2: 原因：目标网站禁止国内大陆IP访问，爬虫会走系统代理进行请求访问
|
||
但检测到你未开启翻墙工具，请打开Clash或FlClash等代理工具，保证能正常访问 Google。
|
||
|
||
3: 自定义代理端口：修改代码'/config/config.js'中 proxyPort 实现自定义端口，端口从代理工具中查看
|
||
`);
|
||
return Promise.reject(error)
|
||
})
|
||
}
|
||
}
|
||
|
||
export default new utils(); |