bmy/pte
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加爬虫

Browse Source
This commit is contained in:
编码猿
2024-11-22 22:59:41 +08:00
parent 8fdd65b9bb
commit 4bc03fc814
10 changed files with 502 additions and 1573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
newBackApi/app.js
View File

@@ -5,13 +5,15 @@ import { ipv6, ipv4 } from "./utils/ip.js";
import tagsRouter from "./router/tags.js";
import infoRouter from "./router/info.js";
import searchRouter from "./router/search.js";
import homeRouter from "./router/home.js";
const app = express()
app.use('/api/v1/tag', tagsRouter);
app.use('/api/v1/info', infoRouter);
app.use('/api/v1/search', searchRouter);
app.use('/v1/api/tag', tagsRouter);
app.use('/v1/api/info', infoRouter);
app.use('/v1/api/search', searchRouter);
app.use('/v1/api/home', homeRouter);
app.listen(config.port, () => {
console.log(`

2
newBackApi/config/config.js
View File

@@ -1,5 +1,5 @@
export default {
port: 3000,
port: 3030,
baseUrl: 'https://www.06se.com',
hot_tags: function () {
return `${this.baseUrl}/hot_tags`

1721
newBackApi/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
newBackApi/package.json
View File

@@ -5,14 +5,15 @@
"main": "index.js",
"type": "module",
"scripts": {
"dev": "node app.js"
"dev": "node-dev app.js"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^1.7.7",
"cheerio": "^1.0.0-rc.12",
"express": "^4.21.1",
"ip": "^2.0.1",
"x-crawl": "^10.0.2"
"ip": "^2.0.1"
}
}

75
newBackApi/router/home.js Normal file
View File

@@ -0,0 +1,75 @@
import express from "express";
import Util from '../utils/index.js'
import Config from '../config/config.js'
const router = express.Router();
// 获取首页banner图
// http://192.168.31.101:3030/v1/api/home/banner
router.get('/banner', async (req, res) => {
let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
let result = []
$(".new-swiper .swiper-wrapper .swiper-slide").each(function() {
let src = $(this).find("span img").attr("src")
if (src) {
result.push({ src })
}
})
res.json({
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})
// 获取热门推荐的写真
// http://192.168.31.101:3030/v1/api/home/hot
router.get('/hot', async (req, res) => {
let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
let result = []
$(".box-body .tab-content .posts-mini").each(function() {
result.push({
id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
title: $(this).find(".item-heading a").text(),
time: $(this).find(".item-meta .icon-circle").text(),
view: $(this).find(".item-meta .meta-right .meta-view").text()
})
})
res.json({
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})
// 获取首页 默认的随机数据
// http://192.168.31.101:3030/v1/api/home/list
router.get('/list', async (req, res) => {
let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
let result = []
$(".tab-content .posts-row .posts-item").each(function() {
result.push({
id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
title: $(this).find(".item-heading a").text(),
time: $(this).find(".item-meta .icon-circle").text(),
view: $(this).find(".item-meta .meta-right .meta-view").text()
})
})
res.json({
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})
export default router

43
newBackApi/router/info.js
View File

@@ -1,42 +1,41 @@
import express from "express";
// import app from "./utils/app.js"
import { Crawl } from '../utils/crawl.js'
import Util from '../utils/index.js'
import Config from '../config/config.js'
const router = express.Router();
const crawlApp = new Crawl()
// http://192.168.31.101:3000/api/v1/info/list?id=6808
// 进入详情的接口
// http://192.168.31.101:3030/v1/api/info/list?id=6808
router.get('/list', async (req, res) => {
let { id } = req.query
const { browser, page } = await crawlApp.page(`${Config.baseUrl}/${id}.html`);
await page.waitForSelector('.article')
let $ = await Util.get(`${Config.baseUrl}/${id}.html`)
let result = {
title: await page.$eval('.article-header h1 a', el => el.innerText),
src_list: await page.$$eval('.article-content .theme-box p img', el => el.map(x => { return { title: x.getAttribute("alt"), alt: x.getAttribute("data-src")} })),
recommend: []
title: $(".article .article-header .article-title a").text(),
src_list: [],
recommend: [],
}
const recommendList = await page.$$('.zib-widget .swiper-wrapper .swiper-slide')
$(".article-content .wp-posts-content p img").map(function(i, el) {
result.src_list.push($(el).attr("data-src"))
})
for (const handle of recommendList) {
$(".swiper-container .swiper-slide a").each(function(i, el) {
result.recommend.push({
id: await handle.$eval('a', el => el.pathname.replace(/[^\d]/g, "")),
thumbnail: await handle.$eval('a img', el => el.getAttribute("data-src")),
title: await handle.$eval('a img+div', el => el.innerText),
time: await handle.$eval('a .px12 item', el => el.innerText),
view: await handle.$eval('a .px12 .pull-right', el => el.innerText),
id: Util.split($(this).attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find("img").attr("data-src"),
title: $(this).find(".text-ellipsis").text(),
time: $(this).find(".px12 item").eq(0).text(),
view: $(this).find(".px12 item").eq(1).text(),
})
}
})
res.json({
code: 300,
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})

68
newBackApi/router/search.js
View File

@@ -1,20 +1,70 @@
import express from "express";
// import app from "./utils/app.js"
import { Crawl } from '../utils/crawl.js'
import Util from '../utils/index.js'
import Config from '../config/config.js'
const router = express.Router();
const crawlApp = new Crawl()
// http://192.168.31.101:3000/api/v1/search/hot
// 搜索页面的热门搜索接口
// http://192.168.31.101:3030/v1/api/search/hot
router.get('/hot', async (req, res) => {
const { browser, page } = await crawlApp.page(Config.baseUrl);
// await page.waitForSelector('.search-input')
let $ = await Util.get(`${Config.baseUrl}/wp-admin/admin-ajax.php?action=search_box`)
let result = {
hotKeyword: [],
recommend: []
}
$(".search-input .search-keywords").eq(0).find("div a").each(function() {
result.hotKeyword.push($(this).text())
})
$(".search-input .relates-thumb .swiper-slide a").each(function() {
result.recommend.push({
id: Util.split($(this).attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find("img").attr("data-src"),
title: $(this).find(".text-ellipsis").text(),
time: $(this).find(".px12 item").eq(0).text(),
view: $(this).find(".px12 item").eq(1).text(),
})
})
res.json({
code: 200,
data: await page.$$eval('.search-keywords', el => el.map(x => { return { title: x } }))
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})
// 搜索页面的热门搜索接口
// http://192.168.31.101:3030/v1/api/search?keyword=杨晨晨
router.get('/', async (req, res) => {
let { keyword, page } = req.query
page = page || 1
let $ = await Util.get(`${Config.baseUrl}/page/${page}?s=${encodeURI(keyword)}&type=post`)
let result = {
currentPage: page,
totalPage: $(".pagenav a").eq(2).text(),
message: $(".search-content .badg .focus-color").text(),
list: [],
}
$(".search-content .posts-item").each(function() {
result.list.push({
id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
title: $(this).find(".item-body .item-heading a").text(),
time: $(this).find(".item-body .item-meta .icon-circle").text(),
view: $(this).find(".item-body .item-meta .meta-right .meta-view").text()
})
})
res.json({
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
})
export default router

77
newBackApi/router/tags.js
View File

@@ -1,60 +1,61 @@
import express from "express";
// import app from "./utils/app.js"
import { Crawl } from '../utils/crawl.js'
import Util from '../utils/index.js'
import Config from '../config/config.js'
const router = express.Router();
const crawlApp = new Crawl()
// http://192.168.31.101:3000/api/v1/tag/list
// 获取 热门标签
// http://192.168.31.101:3030/v1/api/tag/list
router.get('/list', async (req, res) => {
const { browser, page } = await crawlApp.page(Config.hot_tags());
await page.waitForSelector('.categorieslist ul')
const categoriesList = await page.$$('.categorieslist ul li')
let $ = await Util.get(Config.hot_tags())
let result = []
for (const handle of categoriesList) {
$(".categorieslist ul li a").each(function (i) {
result.push({
title: await handle.$eval('h2', el => el.innerText),
count: await handle.$eval('small p', el => el.innerText.replace(/[^\d]/g, "")),
url: await handle.$eval('a', el => el.pathname)
})
}
title: $(this).find("h2").text(),
count: $(this).find("small p").text().replace(/[^\d]/g, ""),
url: Util.split($(this).attr("href")),
});
});
res.json({
code: 300,
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
// 关闭浏览器
// browser.close()
});
})
// http://192.168.31.101:3000/api/v1/tag/info?url=/xiuren&pages=1
// 进入对应标签的列表页面
// http://192.168.31.101:3030/v1/api/tag/info?url=/xiuren&page=1
router.get('/info', async (req, res) => {
let { url, pages } = req.query
pages = pages || 1
const { browser, page } = await crawlApp.page(`${Config.baseUrl}${url}/page/${pages}`);
await page.waitForSelector('.content-layout .posts-row')
const postsList = await page.$$('.posts-item')
let result = []
let { url, page } = req.query
page = page || 1
for (const handle of postsList) {
result.push({
id: await handle.$eval('.item-thumbnail a', el => el.pathname.replace(/[^\d]/g, "")),
thumbnail: await handle.$eval('.item-thumbnail a img', el => el.getAttribute("data-src")),
title: await handle.$eval('.item-body .item-heading a', el => el.innerText),
tags: await handle.$$eval('.item-body .item-tags a', links => links.map(x => { return { title: x.innerText, href: x.href } })),
time: await handle.$eval('.item-body .item-meta item', el => el.innerText),
view: await handle.$eval('.item-body .item-meta .meta-right item', el => el.innerText)
})
let $ = await Util.get(`${Config.baseUrl}${url}/page/${page}`)
let result = {
count: $(".content-layout .title-h-left .icon-spot").text().replace(/[^\d]/g, ""),
introduce: $(".content-layout .muted-2-color").text(),
totalPage: $(".pagenav a").eq(2).text(),
currentPage: page,
list: []
}
$(".content-layout .posts-row .posts-item").each(function (i) {
result.list.push({
id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
title: $(this).find(".item-body .item-heading a").text(),
time: $(this).find(".item-body .item-meta .icon-circle").text(),
view: $(this).find(".item-body .item-meta .meta-right .meta-view").text()
});
});
res.json({
code: 300,
status: 200,
copyright: `bmy ${new Date()}`,
data: result
})
});
})
export default router

19
newBackApi/utils/crawl.js
View File

@@ -1,19 +0,0 @@
import { createCrawl } from 'x-crawl'
export class Crawl {
constructor(option) {
this.app = createCrawl(option)
}
async page(url) {
let { data } = await this.app.crawlPage(url)
return data
}
async html(url) {
let { data } = await this.app.crawlHTML(url)
return data
}
}

55
newBackApi/utils/index.js Normal file
View File

@@ -0,0 +1,55 @@
import * as cheer from "cheerio";
import axios from "axios";
import Config from "../config/config.js";
class utils {
constructor() {
this.ax = axios
this.ResponseInterceptor()
}
/**
* 返回 cheerio 数据结构
* @param html
* @returns {*}
*/
cheerio(html) {
return cheer.load(html)
}
split(str,index = 1) {
return str.split(Config.baseUrl)[index]
}
/**
*
* @param url 请求的地址
* @returns {Promise<AxiosResponse<any>>}
*/
async get(url) {
return await this.ax.get(url, {
headers: {
'cookie': `_ga=GA1.1.2076335189.1731510827; history_search=%5B%22%5Cu6768%5Cu6668%5Cu6668%26type%3Dpost%22%2C%22%5Cu767d%5Cu4e1d%26type%3Dpost%22%2C%22jk%26type%3Dpost%22%5D; PHPSESSID=22oh5maqqirjstqs1p7gbtome7; _lscache_vary=0e8aace2958628a1edb01710bbabbcab; showed_system_notice=showed; _ga_ZT9Q6FCC5R=GS1.1.1732278002.10.1.1732278777.0.0.0`,
'referer': 'https://www.06se.com/',
'user-agent': `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36`,
'accept': `text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7`
}
});
}
/**
*
* 响应拦截器
* @constructor
*/
async ResponseInterceptor() {
this.ax.interceptors.response.use((response) => {
return this.cheerio(response.data);
}, (error) => {
console.log("请求错误...");
return Promise.reject(error)
})
}
}
export default new utils();