增加爬虫

newBackApi/app.js

@@ -5,13 +5,15 @@ import { ipv6, ipv4 } from "./utils/ip.js";
 import tagsRouter from "./router/tags.js";
 import infoRouter from "./router/info.js";
 import searchRouter from "./router/search.js";
+import homeRouter from "./router/home.js";
 
 
 const app = express()
 
-app.use('/api/v1/tag', tagsRouter);
+app.use('/v1/api/tag', tagsRouter);
-app.use('/api/v1/info', infoRouter);
+app.use('/v1/api/info', infoRouter);
-app.use('/api/v1/search', searchRouter);
+app.use('/v1/api/search', searchRouter);
+app.use('/v1/api/home', homeRouter);
 
 app.listen(config.port, () => {
   console.log(`

newBackApi/config/config.js

@@ -1,5 +1,5 @@
 export default {
-    port: 3000,
+    port: 3030,
     baseUrl: 'https://www.06se.com',
     hot_tags: function () {
         return `${this.baseUrl}/hot_tags`

newBackApi/package.json

@@ -5,14 +5,15 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "dev": "node app.js"
+    "dev": "node-dev app.js"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "axios": "^1.7.7",
+    "cheerio": "^1.0.0-rc.12",
     "express": "^4.21.1",
-    "ip": "^2.0.1",
+    "ip": "^2.0.1"
-    "x-crawl": "^10.0.2"
   }
 }

newBackApi/router/home.js

@@ -0,0 +1,75 @@
+import express from "express";
+import Util from '../utils/index.js'
+import Config from '../config/config.js'
+
+const router = express.Router();
+
+// 获取首页banner图
+// http://192.168.31.101:3030/v1/api/home/banner
+router.get('/banner', async (req, res) => {
+    let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
+    let result = []
+
+    $(".new-swiper .swiper-wrapper .swiper-slide").each(function() {
+        let src = $(this).find("span img").attr("src")
+        if (src) {
+            result.push({ src })
+        }
+    })
+
+    res.json({
+        status: 200,
+        copyright: `bmy ${new Date()}`,
+        data: result
+    })
+})
+
+// 获取热门推荐的写真
+// http://192.168.31.101:3030/v1/api/home/hot
+router.get('/hot', async (req, res) => {
+    let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
+    let result = []
+
+    $(".box-body .tab-content .posts-mini").each(function() {
+        result.push({
+            id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
+            thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
+            title: $(this).find(".item-heading a").text(),
+            time: $(this).find(".item-meta .icon-circle").text(),
+            view: $(this).find(".item-meta .meta-right .meta-view").text()
+        })
+    })
+
+    res.json({
+        status: 200,
+        copyright: `bmy ${new Date()}`,
+        data: result
+    })
+})
+
+
+// 获取首页 默认的随机数据
+// http://192.168.31.101:3030/v1/api/home/list
+router.get('/list', async (req, res) => {
+    let $ = await Util.get(`${Config.baseUrl}/?orderby=rand`)
+    let result = []
+
+    $(".tab-content .posts-row .posts-item").each(function() {
+        result.push({
+            id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
+            thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
+            title: $(this).find(".item-heading a").text(),
+            time: $(this).find(".item-meta .icon-circle").text(),
+            view: $(this).find(".item-meta .meta-right .meta-view").text()
+        })
+    })
+
+    res.json({
+        status: 200,
+        copyright: `bmy ${new Date()}`,
+        data: result
+    })
+})
+
+
+export default router

newBackApi/router/info.js

@@ -1,38 +1,37 @@
 import express from "express";
-// import app from "./utils/app.js"
+import Util from '../utils/index.js'
-import { Crawl } from '../utils/crawl.js'
 import Config from '../config/config.js'
 
 const router = express.Router();
-const crawlApp = new Crawl()
 
-
+// 进入详情的接口
-// http://192.168.31.101:3000/api/v1/info/list?id=6808
+// http://192.168.31.101:3030/v1/api/info/list?id=6808
 router.get('/list', async (req, res) => {
     let { id } = req.query
-    const { browser, page } = await crawlApp.page(`${Config.baseUrl}/${id}.html`);
+    let $ = await Util.get(`${Config.baseUrl}/${id}.html`)
-    await page.waitForSelector('.article')
-
     let result = {
-        title: await page.$eval('.article-header h1 a', el => el.innerText),
+        title: $(".article .article-header .article-title a").text(),
-        src_list: await page.$$eval('.article-content .theme-box p img', el => el.map(x => { return { title: x.getAttribute("alt"), alt: x.getAttribute("data-src")} })),
+        src_list: [],
-        recommend: []
+        recommend: [],
     }
 
-    const recommendList = await page.$$('.zib-widget .swiper-wrapper .swiper-slide')
+    $(".article-content .wp-posts-content p img").map(function(i, el) {
-
+        result.src_list.push($(el).attr("data-src"))
-    for (const handle of recommendList) {
+    })
-        result.recommend.push({
+
-            id: await handle.$eval('a', el => el.pathname.replace(/[^\d]/g, "")),
+    $(".swiper-container .swiper-slide a").each(function(i, el) {
-            thumbnail: await handle.$eval('a img', el => el.getAttribute("data-src")),
+        result.recommend.push({
-            title: await handle.$eval('a img+div', el => el.innerText),
+            id: Util.split($(this).attr("href")).replace(/[^\d]/g, ""),
-            time: await handle.$eval('a .px12 item', el => el.innerText),
+            thumbnail: $(this).find("img").attr("data-src"),
-            view: await handle.$eval('a .px12 .pull-right', el => el.innerText),
+            title: $(this).find(".text-ellipsis").text(),
+            time: $(this).find(".px12 item").eq(0).text(),
+            view: $(this).find(".px12 item").eq(1).text(),
+        })
     })
-    }
 
     res.json({
-        code: 300,
+        status: 200,
+        copyright: `bmy ${new Date()}`,
         data: result
     })
 

newBackApi/router/search.js

@@ -1,20 +1,70 @@
 import express from "express";
-// import app from "./utils/app.js"
+import Util from '../utils/index.js'
-import { Crawl } from '../utils/crawl.js'
 import Config from '../config/config.js'
 
 const router = express.Router();
-const crawlApp = new Crawl()
 
-
+// 搜索页面的热门搜索接口
-// http://192.168.31.101:3000/api/v1/search/hot
+// http://192.168.31.101:3030/v1/api/search/hot
 router.get('/hot', async (req, res) => {
-    const { browser, page } = await crawlApp.page(Config.baseUrl);
+    let $ = await Util.get(`${Config.baseUrl}/wp-admin/admin-ajax.php?action=search_box`)
-    // await page.waitForSelector('.search-input')
+    let result = {
+        hotKeyword: [],
+        recommend: []
+    }
+    
+    $(".search-input .search-keywords").eq(0).find("div a").each(function() {
+        result.hotKeyword.push($(this).text())
+    })
+
+    $(".search-input .relates-thumb .swiper-slide a").each(function() {
+        result.recommend.push({
+            id: Util.split($(this).attr("href")).replace(/[^\d]/g, ""),
+            thumbnail: $(this).find("img").attr("data-src"),
+            title: $(this).find(".text-ellipsis").text(),
+            time: $(this).find(".px12 item").eq(0).text(),
+            view: $(this).find(".px12 item").eq(1).text(),
+        })
+    })
+
     res.json({
-        code: 200,
+        status: 200,
-        data: await page.$$eval('.search-keywords', el => el.map(x => { return { title: x } }))
+        copyright: `bmy ${new Date()}`,
+        data: result
     })
 })
 
+
+// 搜索页面的热门搜索接口
+// http://192.168.31.101:3030/v1/api/search?keyword=杨晨晨
+router.get('/', async (req, res) => {
+    let { keyword, page } = req.query
+    page = page || 1
+
+    let $ = await Util.get(`${Config.baseUrl}/page/${page}?s=${encodeURI(keyword)}&type=post`)
+    let result = {
+        currentPage: page,
+        totalPage: $(".pagenav a").eq(2).text(),
+        message: $(".search-content .badg .focus-color").text(),
+        list: [],
+    }
+ 
+    $(".search-content .posts-item").each(function() {
+        result.list.push({
+            id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
+            thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
+            title: $(this).find(".item-body .item-heading a").text(),
+            time: $(this).find(".item-body .item-meta .icon-circle").text(),
+            view: $(this).find(".item-body .item-meta .meta-right .meta-view").text()
+        })
+    })
+
+    res.json({
+        status: 200,
+        copyright: `bmy ${new Date()}`,
+        data: result
+    })
+})
+
+
 export default router

newBackApi/router/tags.js

@@ -1,60 +1,61 @@
 import express from "express";
-// import app from "./utils/app.js"
+import Util from '../utils/index.js'
-import { Crawl } from '../utils/crawl.js'
 import Config from '../config/config.js'
 
 const router = express.Router();
-const crawlApp = new Crawl()
 
-// http://192.168.31.101:3000/api/v1/tag/list
+// 获取 热门标签
+// http://192.168.31.101:3030/v1/api/tag/list
 router.get('/list', async (req, res) => {
-    const { browser, page } = await crawlApp.page(Config.hot_tags());
+    let $ = await Util.get(Config.hot_tags())
-    await page.waitForSelector('.categorieslist ul')
-    const categoriesList = await page.$$('.categorieslist ul li')
     let result = []
-
+    $(".categorieslist ul li a").each(function (i) {
-    for (const handle of categoriesList) {    
         result.push({
-            title: await handle.$eval('h2', el => el.innerText),
+            title: $(this).find("h2").text(),
-            count: await handle.$eval('small p', el => el.innerText.replace(/[^\d]/g, "")),
+            count: $(this).find("small p").text().replace(/[^\d]/g, ""),
-            url: await handle.$eval('a', el => el.pathname)
+            url: Util.split($(this).attr("href")),
-        })
+        });
-    }
+    });
+
 
     res.json({
-        code: 300,
+        status: 200,
+        copyright: `bmy ${new Date()}`,
         data: result
+    });
 })
 
-    // 关闭浏览器
+// 进入对应标签的列表页面
-    // browser.close()
+// http://192.168.31.101:3030/v1/api/tag/info?url=/xiuren&page=1
-})
-
-
-// http://192.168.31.101:3000/api/v1/tag/info?url=/xiuren&pages=1
 router.get('/info', async (req, res) => {
-    let { url, pages } = req.query
+    let { url, page } = req.query
-    pages = pages || 1
+    page = page || 1
-    const { browser, page } = await crawlApp.page(`${Config.baseUrl}${url}/page/${pages}`);
-    await page.waitForSelector('.content-layout .posts-row')
-    const postsList = await page.$$('.posts-item')
-    let result = []
 
-    for (const handle of postsList) {
+    let $ = await Util.get(`${Config.baseUrl}${url}/page/${page}`)
-        result.push({
+    let result = {
-            id:  await handle.$eval('.item-thumbnail a', el => el.pathname.replace(/[^\d]/g, "")),
+        count: $(".content-layout .title-h-left .icon-spot").text().replace(/[^\d]/g, ""),
-            thumbnail: await handle.$eval('.item-thumbnail a img', el => el.getAttribute("data-src")),
+        introduce: $(".content-layout .muted-2-color").text(),
-            title: await handle.$eval('.item-body .item-heading a', el => el.innerText),
+        totalPage: $(".pagenav a").eq(2).text(),
-            tags: await handle.$$eval('.item-body .item-tags a', links => links.map(x => { return { title: x.innerText, href: x.href } })),
+        currentPage: page,
-            time: await handle.$eval('.item-body .item-meta item', el => el.innerText),
+        list: []
-            view: await handle.$eval('.item-body .item-meta .meta-right item', el => el.innerText)
-        })
     }
 
+    $(".content-layout .posts-row .posts-item").each(function (i) {
+        result.list.push({
+            id: Util.split($(this).find(".item-thumbnail a").attr("href")).replace(/[^\d]/g, ""),
+            thumbnail: $(this).find(".item-thumbnail a img").attr("data-src"),
+            title: $(this).find(".item-body .item-heading a").text(),
+            time: $(this).find(".item-body .item-meta .icon-circle").text(),
+            view: $(this).find(".item-body .item-meta .meta-right .meta-view").text()
+        });
+    });
+
+
     res.json({
-        code: 300,
+        status: 200,
+        copyright: `bmy ${new Date()}`,
         data: result
-    })
+    });
 })
 
 export default router

newBackApi/utils/crawl.js

@@ -1,19 +0,0 @@
-import { createCrawl } from 'x-crawl'
-
-export class Crawl {
-
-    constructor(option) {
-        this.app = createCrawl(option)
-    }
-
-
-    async page(url) {
-       let { data } = await this.app.crawlPage(url)
-       return data
-    }
-
-    async html(url) {
-        let { data } = await this.app.crawlHTML(url)
-        return data
-     }
-}

newBackApi/utils/index.js

@@ -0,0 +1,55 @@
+import * as cheer from "cheerio";
+import axios from "axios";
+import Config from "../config/config.js";
+
+class utils {
+    constructor() {
+        this.ax = axios
+        this.ResponseInterceptor()
+    }
+
+    /**
+     * 返回 cheerio 数据结构
+     * @param html
+     * @returns {*}
+     */
+    cheerio(html) {
+        return cheer.load(html)
+    }
+
+    split(str,index = 1) {
+        return str.split(Config.baseUrl)[index]
+    }
+
+    /**
+     *
+     * @param url 请求的地址
+     * @returns {Promise<AxiosResponse<any>>}
+     */
+    async get(url) {
+        return await this.ax.get(url, {
+            headers: {
+                'cookie': `_ga=GA1.1.2076335189.1731510827; history_search=%5B%22%5Cu6768%5Cu6668%5Cu6668%26type%3Dpost%22%2C%22%5Cu767d%5Cu4e1d%26type%3Dpost%22%2C%22jk%26type%3Dpost%22%5D; PHPSESSID=22oh5maqqirjstqs1p7gbtome7; _lscache_vary=0e8aace2958628a1edb01710bbabbcab; showed_system_notice=showed; _ga_ZT9Q6FCC5R=GS1.1.1732278002.10.1.1732278777.0.0.0`,
+                'referer': 'https://www.06se.com/',
+                'user-agent': `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36`,
+                'accept': `text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7`
+            }
+        });
+    }
+
+    /**
+     * 
+     * 响应拦截器
+     * @constructor
+     */
+    async ResponseInterceptor() {
+        this.ax.interceptors.response.use((response) => {
+            return this.cheerio(response.data);
+        }, (error) => {
+            console.log("请求错误...");
+            return Promise.reject(error)
+        })
+    }
+}
+
+export default new utils();