增加爬虫

This commit is contained in:
编码猿
2024-11-23 12:21:29 +08:00
parent 12eca48220
commit b4297bf17c
4 changed files with 57 additions and 2 deletions

View File

@@ -1,10 +1,16 @@
import * as cheer from "cheerio";
import axios from "axios";
import { HttpsProxyAgent } from "https-proxy-agent";
import Config from "../config/config.js";
class utils {
constructor() {
this.ax = axios
// 让 ajax 请求 走flclash的翻墙代理
const httpsAgent = new HttpsProxyAgent(`http://127.0.0.1:${Config.proxyPort}`);
this.ax.defaults.httpsAgent = httpsAgent;
this.ax.defaults.proxy = false;
this.ResponseInterceptor()
}
@@ -17,7 +23,7 @@ class utils {
return cheer.load(html)
}
split(str,index = 1) {
split(str, index = 1) {
return str.split(Config.baseUrl)[index]
}
@@ -46,7 +52,14 @@ class utils {
this.ax.interceptors.response.use((response) => {
return this.cheerio(response.data);
}, (error) => {
console.log("请求错误...");
console.log(`
❌ 请求错误❕❕❕
1: 错误信息 ${error.message}
2: 原因目标网站禁止国内大陆IP访问爬虫会走系统代理进行请求访问
但检测到你未开启翻墙工具请打开Clash或FlClash等代理工具保证能正常访问 Google。
3: 自定义代理端口:修改代码'/config/config.js'中 proxyPort 实现自定义端口,端口从代理工具中查看
`);
return Promise.reject(error)
})
}