修改爬虫

h5_webview/.project

@@ -14,4 +14,15 @@
 	<natures>
 		<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
 	</natures>
+	<filteredResources>
+		<filter>
+			<id>1731511152229</id>
+			<name></name>
+			<type>30</type>
+			<matcher>
+				<id>org.eclipse.core.resources.regexFilterMatcher</id>
+				<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
+			</matcher>
+		</filter>
+	</filteredResources>
 </projectDescription>

h5_webview/.settings/org.eclipse.buildship.core.prefs

@@ -1,11 +1,11 @@
-arguments=
+arguments=--init-script /var/folders/jn/6t23s_gx66z2j_ng_bn8_83m0000gn/T/db3b08fc4a9ef609cb16b96b200fa13e563f396e9bb1ed0905fdab7bc3bc513b.gradle --init-script /var/folders/jn/6t23s_gx66z2j_ng_bn8_83m0000gn/T/52cde0cfcf3e28b8b7510e992210d9614505e0911af0c190bd590d7158574963.gradle
 auto.sync=false
 build.scans.enabled=false
 connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER)
 connection.project.dir=
 eclipse.preferences.version=1
 gradle.user.home=
-java.home=/usr
+java.home=/Library/Java/JavaVirtualMachines/jdk1.8.0_311.jdk/Contents/Home
 jvm.arguments=
 offline.mode=false
 override.workspace.settings=true

newBackApi/app.js

@@ -0,0 +1,27 @@
+import express from "express";
+import config from "./config/config.js";
+import { ipv6, ipv4 } from "./utils/ip.js";
+
+import tagsRouter from "./router/tags.js";
+import infoRouter from "./router/info.js";
+import searchRouter from "./router/search.js";
+
+
+const app = express()
+
+app.use('/api/v1/tag', tagsRouter);
+app.use('/api/v1/info', infoRouter);
+app.use('/api/v1/search', searchRouter);
+
+app.listen(config.port, () => {
+  console.log(`
+爬虫运行在下面网址：
+
+  1️⃣ 公网IPV6: http://[${ipv6()}]:${config.port}
+  2️⃣ 局域网IPV4: http://${ipv4()}:${config.port}
+    
+  `)
+})
+
+
+export default app

newBackApi/config/config.js

@@ -0,0 +1,7 @@
+export default {
+    port: 3000,
+    baseUrl: 'https://www.06se.com',
+    hot_tags: function () {
+        return `${this.baseUrl}/hot_tags`
+    }
+}

newBackApi/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "newbackapi",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "dev": "node app.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "express": "^4.21.1",
+    "ip": "^2.0.1",
+    "x-crawl": "^10.0.2"
+  }
+}

newBackApi/router/info.js

@@ -0,0 +1,43 @@
+import express from "express";
+// import app from "./utils/app.js"
+import { Crawl } from '../utils/crawl.js'
+import Config from '../config/config.js'
+
+const router = express.Router();
+const crawlApp = new Crawl()
+
+
+// http://192.168.31.101:3000/api/v1/info/list?id=6808
+router.get('/list', async (req, res) => {
+    let { id } = req.query
+    const { browser, page } = await crawlApp.page(`${Config.baseUrl}/${id}.html`);
+    await page.waitForSelector('.article')
+
+    let result = {
+        title: await page.$eval('.article-header h1 a', el => el.innerText),
+        src_list: await page.$$eval('.article-content .theme-box p img', el => el.map(x => { return { title: x.getAttribute("alt"), alt: x.getAttribute("data-src")} })),
+        recommend: []
+    }
+
+    const recommendList = await page.$$('.zib-widget .swiper-wrapper .swiper-slide')
+
+    for (const handle of recommendList) {
+        result.recommend.push({
+            id: await handle.$eval('a', el => el.pathname.replace(/[^\d]/g, "")),
+            thumbnail: await handle.$eval('a img', el => el.getAttribute("data-src")),
+            title: await handle.$eval('a img+div', el => el.innerText),
+            time: await handle.$eval('a .px12 item', el => el.innerText),
+            view: await handle.$eval('a .px12 .pull-right', el => el.innerText),
+        })
+    }
+    
+    res.json({
+        code: 300,
+        data: result
+    })
+
+  
+
+})
+
+export default router

newBackApi/router/search.js

@@ -0,0 +1,20 @@
+import express from "express";
+// import app from "./utils/app.js"
+import { Crawl } from '../utils/crawl.js'
+import Config from '../config/config.js'
+
+const router = express.Router();
+const crawlApp = new Crawl()
+
+
+// http://192.168.31.101:3000/api/v1/search/hot
+router.get('/hot', async (req, res) => {
+    const { browser, page } = await crawlApp.page(Config.baseUrl);
+    // await page.waitForSelector('.search-input')
+    res.json({
+        code: 200,
+        data: await page.$$eval('.search-keywords', el => el.map(x => { return { title: x } }))
+    })
+})
+
+export default router

newBackApi/router/tags.js

@@ -0,0 +1,60 @@
+import express from "express";
+// import app from "./utils/app.js"
+import { Crawl } from '../utils/crawl.js'
+import Config from '../config/config.js'
+
+const router = express.Router();
+const crawlApp = new Crawl()
+
+// http://192.168.31.101:3000/api/v1/tag/list
+router.get('/list', async (req, res) => {
+    const { browser, page } = await crawlApp.page(Config.hot_tags());
+    await page.waitForSelector('.categorieslist ul')
+    const categoriesList = await page.$$('.categorieslist ul li')
+    let result = []
+
+    for (const handle of categoriesList) {    
+        result.push({
+            title: await handle.$eval('h2', el => el.innerText),
+            count: await handle.$eval('small p', el => el.innerText.replace(/[^\d]/g, "")),
+            url: await handle.$eval('a', el => el.pathname)
+        })
+    }
+
+    res.json({
+        code: 300,
+        data: result
+    })
+
+    // 关闭浏览器
+    // browser.close()
+})
+
+
+// http://192.168.31.101:3000/api/v1/tag/info?url=/xiuren&pages=1
+router.get('/info', async (req, res) => {
+    let { url, pages } = req.query
+    pages = pages || 1
+    const { browser, page } = await crawlApp.page(`${Config.baseUrl}${url}/page/${pages}`);
+    await page.waitForSelector('.content-layout .posts-row')
+    const postsList = await page.$$('.posts-item')
+    let result = []
+
+    for (const handle of postsList) {
+        result.push({
+            id:  await handle.$eval('.item-thumbnail a', el => el.pathname.replace(/[^\d]/g, "")),
+            thumbnail: await handle.$eval('.item-thumbnail a img', el => el.getAttribute("data-src")),
+            title: await handle.$eval('.item-body .item-heading a', el => el.innerText),
+            tags: await handle.$$eval('.item-body .item-tags a', links => links.map(x => { return { title: x.innerText, href: x.href } })),
+            time: await handle.$eval('.item-body .item-meta item', el => el.innerText),
+            view: await handle.$eval('.item-body .item-meta .meta-right item', el => el.innerText)
+        })
+    }
+
+    res.json({
+        code: 300,
+        data: result
+    })
+})
+
+export default router

newBackApi/utils/crawl.js

@@ -0,0 +1,19 @@
+import { createCrawl } from 'x-crawl'
+
+export class Crawl {
+
+    constructor(option) {
+        this.app = createCrawl(option)
+    }
+
+
+    async page(url) {
+       let { data } = await this.app.crawlPage(url)
+       return data
+    }
+
+    async html(url) {
+        let { data } = await this.app.crawlHTML(url)
+        return data
+     }
+}

newBackApi/utils/ip.js

@@ -0,0 +1,12 @@
+import ip from 'ip'
+
+class IP {
+    ipv6() {
+        return ip.address('private','ipv6')
+    }
+    ipv4() {
+        return ip.address('public','ipv4')
+    }
+}
+
+export const {ipv6,ipv4} = new IP()