first commit

历届学生/王凯/18-11-23/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["@babel/preset-env"]
+}

历届学生/王凯/18-11-23/gulpfile.js

@@ -0,0 +1,13 @@
+var gulp = require('gulp');
+var babel = require("gulp-babel");
+var watch = require('gulp-watch');
+
+
+gulp.task('babel', function () {
+  return watch('src/**/*.js', function () {
+      gulp
+      .src('src/**/*.js')
+      .pipe(babel())
+      .pipe(gulp.dest('dist/'));
+  });
+});

历届学生/王凯/18-11-23/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "18-11-23",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "dev": " concurrently \"gulp babel\" \"pm2 start -i max ./src/bin/www.js\" ",
+    "stop": "pm2 stop www",
+    "start": "pm2 start -i max ./src/bin/www.js",
+    "dep": " concurrently \"gulp babel\" \"node-dev ./src/bin/www.js\" \" node-dev ./src/timer.js  \" "
+  },
+  "dependencies": {
+    "cookie-parser": "~1.4.3",
+    "debug": "~2.6.9",
+    "express": "~4.16.0",
+    "express-load-router": "^2.1.4",
+    "http-errors": "~1.6.2",
+    "mysql": "^2.16.0",
+    "node-schedule": "^1.3.0",
+    "nodemailer": "^4.7.0",
+    "pug": "^2.0.3"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.1.6",
+    "@babel/preset-env": "^7.1.6",
+    "cheerio": "^1.0.0-rc.2",
+    "concurrently": "^4.1.0",
+    "gulp": "^3.9.1",
+    "gulp-babel": "^8.0.0-beta.2",
+    "gulp-watch": "^5.0.1",
+    "request": "^2.88.0"
+  }
+}

历届学生/王凯/18-11-23/src/app.js

@@ -0,0 +1,41 @@
+var createError = require('http-errors');
+var express = require('express');
+var loadRouter = require('express-load-router');
+var path = require('path');
+var cookieParser = require('cookie-parser'); 
+
+var app = express();
+
+app.all('*', (req, res, next) => {
+  res.header("Access-Control-Allow-Origin", "*");
+  res.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS');
+  res.header("Access-Control-Allow-Headers", "X-Requested-With");
+  res.header('Access-Control-Allow-Headers', 'Content-Type');
+  next();
+});
+
+
+loadRouter(app, path.join(__dirname, 'controllers'));
+
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+
+
+
+// catch 404 and forward to error handler
+// app.use(function(req, res, next) {
+//   next(createError(404));
+// }); 
+
+// error handler
+// app.use(function(err, req, res, next) {
+//   // set locals, only providing error in development
+//   res.locals.message = err.message;
+//   res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+//   // render the error page
+//   res.status(err.status || 500);
+// });
+
+module.exports = app;

历届学生/王凯/18-11-23/src/bin/www.js

@@ -0,0 +1,59 @@
+var app = require('../app');
+var debug = require('debug')('18-11-23:server');
+var http = require('http');
+var server;
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = 3000;
+app.set('port', port);
+
+server = http.createServer(app);
+server.listen(port);
+
+
+
+
+
+server.on('error', onError);
+server.on('listening', onListening);
+
+
+
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  console.log('Listening on ' + bind);
+}

历届学生/王凯/18-11-23/src/config/index.js

@@ -0,0 +1,55 @@
+module.exports = {
+  BaseUrl: 'http://www.xbiquge.la',
+  BookUrl: '/15/15409/index.html',
+  ApiStatusCode: {
+    success: {
+      status: 200,
+      data: null
+    },
+    errors: {
+      status: 404,
+      data: []
+    }
+  },
+  MysqlConfig: {
+    connectionLimit : 10,
+    host            : 'localhost',
+    user            : 'root',
+    password        : 'lb714500',
+    database        : 'chongzi'
+  },
+  Sql: {
+    index: {
+      insert: "INSERT chongzi_text (text_title, text_url) VALUES (?,?)",
+      select: "SELECT * FROM chongzi_text"
+    }
+  },
+  mailOption: {
+    send: {
+      // 服务器地址
+      host: 'smtp.163.com',
+      // 使用了内置传输发送邮件 查看支持列表：https://nodemailer.com/smtp/well-known/
+      service: '163', 
+      // SMTP 端口
+      port: 465,
+      // 使用SSL
+      secureConnection: true, 
+      auth: {
+        user: 'lb2271608011@163.com',
+        // 这里密码不是qq密码，是smtp授权码
+        pass: 'lb714500',
+      }
+    },
+    accept: {
+      // "发件人名称" <发件人邮箱>
+      from: '"小说更新程序" <lb2271608011@163.com>',
+      // 接收人邮箱
+      to: '2271608011@qq.com',
+      // 邮件标题
+      subject: null,
+      // 发送text或者html格式
+      // text: 'Hello world?',
+      html: null // html body
+    }
+  }
+}

历届学生/王凯/18-11-23/src/controllers/home/index.js

@@ -0,0 +1,40 @@
+const db = require("./../../model");
+const request = require('request');
+const config = require('./../../config');
+const cheerio = require('cheerio')
+const utils = require('./../../utils')
+
+exports.index = {
+  method: 'GET',
+  params: [':id'],
+  async handler(req, res) {
+    
+       // 爬取网页并解析dom
+       const $ = utils.Dom(await utils.http(utils.ReturnUrl()))
+       var data = [], webSitelengths = $('#list dl dd').length; // 从网页上爬取到的数据长度(这个数据长度会大于数据库的数据[小说更新])
+
+       // 数据库中已存在的小说数据长度
+       var MysqlLength =  (await db.query({ sql: config.Sql.index.select,  par: [] })).data.length
+
+      // 对比爬到的数据和数据库中的数据，如果爬到的数据有更新，
+      // 那么将新章节存到数据库，没有就从数据库读数据显示到页面上
+      if(webSitelengths > MysqlLength){
+        for(var i = MysqlLength;i<webSitelengths;i++) {
+          // 暂存数据到数组
+          data.push({ title: $('#list dl dd').eq(i).find('a').text(), url: config.BaseUrl+$('#list dl dd').eq(i).find('a').attr("href") })
+          
+          // 数据库存数据
+          await db.query({ 
+            sql: config.Sql.index.insert, 
+            par: [ $('#list dl dd').eq(i).find('a').text(), config.BaseUrl+$('#list dl dd').eq(i).find('a').attr("href")] 
+          })
+        }
+        console.log(utils.SendMail(data))
+      }
+
+    
+      res.json(await db.query({ sql: config.Sql.index.select,  par: [] }))
+
+
+  }
+};

历届学生/王凯/18-11-23/src/controllers/info/index.js

@@ -0,0 +1,5 @@
+exports.info = (req, res) => {
+  res.json({
+    title: "user"
+  });
+};

历届学生/王凯/18-11-23/src/email.pug

@@ -0,0 +1,4 @@
+ul
+  each obj in data
+    li
+      a(href='' +obj.url+ '')= obj.title

历届学生/王凯/18-11-23/src/model/index.js

@@ -0,0 +1,24 @@
+const mysql = require('mysql');
+const config = require('../config');
+class Mysql {
+
+  constructor() {
+    this.pool = mysql.createPool(config.MysqlConfig)
+  }
+
+  query (params) {
+    return new Promise((resolve,reject) => {
+      this.pool.query(params.sql, params.par ? params.par : [], function (error, results, fields) {
+        if (error) {
+          throw error;
+          reject(error)
+        } else {
+          config.ApiStatusCode.success.data = results
+          results.length == 0 ? resolve(config.ApiStatusCode.errors) : resolve(config.ApiStatusCode.success);
+        }
+      })
+    })
+  }
+}
+
+module.exports = new Mysql()

历届学生/王凯/18-11-23/src/timer.js

@@ -0,0 +1,11 @@
+var schedule = require("node-schedule");  
+const request = require('request');
+
+
+var rule = new schedule.RecurrenceRule();
+rule.minute = 42; 
+var j = schedule.scheduleJob(rule, function(){
+  request('http://localhost:3000/home/api?id=2', function (error, response, body) {
+    console.log(body);
+  });
+});

历届学生/王凯/18-11-23/src/utils/index.js

@@ -0,0 +1,52 @@
+const config = require("../config")
+const request = require('request');
+const cheerio = require('cheerio');
+const nodemailer = require('nodemailer');
+const fs = require('fs');
+const path = require('path');
+const pug = require('pug');
+
+module.exports = {
+
+  ReturnUrl() {
+    return config.BaseUrl+config.BookUrl
+  },
+
+  http (url) {
+    return new Promise((resolve,reject) => {
+      request(url, function (error, response, body) {
+        if(error) throw error;
+        
+        resolve(body)
+      })
+    })
+  },
+
+  Dom(html) {
+    return cheerio.load(html)
+  },
+
+  SendMail(data) {
+    
+    const compiledFunction = pug.compileFile(path.join(__dirname, '../email.pug'));
+
+  
+
+    config.mailOption.accept.subject = "您关注的小说 <牧神记> 有新更新啦？"
+    config.mailOption.accept.html =  compiledFunction({ data: data })
+
+    let transporter = nodemailer.createTransport(config.mailOption.send);
+
+    transporter.sendMail(config.mailOption.accept, (error, info) => {
+      if (error) {
+        return console.log(error);
+      }
+      return info;
+
+    });
+
+
+  }
+
+
+}

历届学生/王凯/18-11-23/课程列表.txt

@@ -0,0 +1,21 @@
+第一天：
+
+  1：使用nodejs的web开发框架express 生成了一个基本的nodejs web项目
+  2：使用了 express的自动路由加载模块：express-load-router，实现对路由的自动加载，无需手动进行繁琐的配置
+  3：使用 mysql 模块，负责与数据库进行链接，并封装数据库链接 使用es6 语法进行。
+
+第二天：
+
+  1：建立mysql数据库(用来存放爬虫爬到的小说数据)，继续完善mysql类库文件的封装
+  2：request 发送 http请求获取网页源码
+  3: 使用 cheerio 模块解析网页源码拿到我们想要的数据
+  4: 返回数据给前端
+
+第三天：
+
+  1：配置好babel + gulp + watch 对代码进行转换
+  2：完成数据库的Promise封装，实现小说数据入库
+  3：node-schedule实现定时爬取小说网站，对比新旧章节后将最新章节存入数据库，然后将最新章节邮件通知给用户
+  4：通过两种方法pm2，cluster 实现接口负载均衡
+  5：手工配置pug，渲染邮件模板，可自己美化pug代码达到精美邮件模板
+  6：concurrently 实现并发执行多个任务