first commit

downKsVideo/src/.pydio

@@ -0,0 +1 @@
+528077e4-c137-4dbf-9206-1909e22cda73

downKsVideo/src/config/.pydio

@@ -0,0 +1 @@
+4357a39c-dc52-4180-a387-00d44344f293

downKsVideo/src/config/config.js

@@ -0,0 +1,49 @@
+const { gql } = require("graphql-request");
+
+module.exports = {
+    url: "https://www.kuaishou.com/graphql",
+    // 获取收藏的视频
+    visionProfileLikePhotoList: gql`
+    fragment photoContent on PhotoEntity { 
+        __typename 
+        id duration 
+        caption originCaption 
+        likeCount viewCount commentCount realLikeCount 
+        coverUrl photoUrl photoH265Url manifest 
+        manifestH265 videoResource coverUrls 
+        { 
+            url __typename 
+        } 
+        timestamp expTag animatedCoverUrl distance 
+        videoRatio liked stereoType profileUserTopPhoto 
+        musicBlocked riskTagContent riskTagUrl 
+    } 
+    fragment recoPhotoFragment on recoPhotoEntity { 
+        __typename id duration caption originCaption likeCount 
+        viewCount commentCount realLikeCount coverUrl photoUrl photoH265Url manifest
+        manifestH265 videoResource coverUrls 
+        { url __typename } 
+        timestamp expTag animatedCoverUrl distance videoRatio liked stereoType 
+        profileUserTopPhoto musicBlocked riskTagContent riskTagUrl 
+    } 
+    fragment feedContent on Feed { 
+        type author 
+        { 
+            id name headerUrl following headerUrls 
+            { 
+                url __typename 
+            } __typename 
+        } photo { 
+            ...photoContent ...recoPhotoFragment __typename 
+        } canAddComment llsid status currentPcursor tags 
+        { type name __typename } 
+        __typename 
+    } 
+    query visionProfileLikePhotoList($pcursor: String, $page: String, $webPageArea: String) { 
+        visionProfileLikePhotoList(pcursor: $pcursor, page: $page, webPageArea: $webPageArea) { 
+            result llsid webPageArea feeds {
+                ...feedContent __typename 
+                } hostName pcursor __typename 
+        } 
+    }`,
+};

downKsVideo/src/http/.pydio

@@ -0,0 +1 @@
+9aaa7146-0671-40e8-8c0b-15fb571f35a8

downKsVideo/src/http/http.js

@@ -0,0 +1,24 @@
+const { GraphQLClient } = require("graphql-request");
+const config = require("../config/config");
+
+class GraphqlRequest {
+  constructor() {
+    this.client = new GraphQLClient(config.url, {
+      headers: {
+        Cookie: 'context=1|MS43NjQ1ODM2OTgyODY2OTgyLjYxODQzOTUyLjE3MjQ2MTEwOTkyMjkuNDcyNDA=|MS43NjQ1ODM2OTgyODY2OTgyLjE5ODE4Mjk4LjE3MjQ2MTEwOTkyMjkuNDcyNDE=|0|graphql-server|webservice|false|NA; kpf=PC_WEB; clientid=3; did=web_017a1f82461c8b4dc575f8df3fee78ff; userId=58344290; kpn=KUAISHOU_VISION; kuaishou.server.web_st=ChZrdWFpc2hvdS5zZXJ2ZXIud2ViLnN0EqABVncUlGSoudMExPNRzEXf8OSAig_Gst7K3MuSob5dchyTwVVPrkPebW5B_ZMl9MFrNPRDjNhSpmGW_PABiZxH3bKuG7Cj-IaMhtEaaQiqs8ajQZfJ5CNjHKAmVizdwjF8qRnULuxZbrDwBE3BdzuUEDB-xE1ybAhhF0zQGz0LqzVDIT4ohfCbbzXfEZWCygEQCArtAgQB8sEooFEfLPXiwBoS5Uf6LooXNTrpm4StA9DDXaoVIiDqFSEmUkB66mi_cuqecbrHKQ5fBC5x74q1-WPiNNwCMSgFMAE;'
+      },
+    })
+  }
+  async post(document, variables) {
+      try {
+        let res = await this.client.request(document, variables);
+        return res.visionProfileLikePhotoList;
+      } catch (error) {
+          console.log("===========================请求出错===========================");
+          console.log(error.message);
+          console.log("===========================请求出错===========================");
+      }
+  }
+}
+
+module.exports = new GraphqlRequest();

downKsVideo/src/index.js

@@ -0,0 +1,126 @@
+const request = require("./http/http");
+const config = require("./config/config");
+const process = require('child_process');
+const https = require("https");
+const fs = require("fs");
+const pr = require('node:process');
+
+let TotalFeeds = [], TotalCount = 0, principalId = "空";
+
+// 爬虫下载视频的速度过快（已做20秒延迟），快手服务器会突然断开连接，
+// 从而http请求失效引发错误导致程序直接退出
+// 这边从进程级捕获错误，避免程序中断，
+// 下面的http error 部分会重新再次下载刚被拒绝的视频
+// 同时数据太大，会导致超出nodejs的默认堆内存导致程序崩溃
+// 所以启动时需要加上参数 --max-old-space-size=20240
+pr.on('uncaughtException', err => {
+  console.log(' pr NOT exit...');
+  console.log(err.stack);
+});
+
+/**
+ * 自动下载快手主播主页所有视频和图片
+ */
+const main = async () => {
+  console.log("开始 ==============================================================");
+  console.log(`当前页码为：${principalId}`);
+  let { pcursor, feeds } = await request.post(config.visionProfileLikePhotoList, {
+    page: "profile",
+    pcursor: principalId
+  });
+  principalId = pcursor;
+  console.log(`本单次请求到：${feeds.length} 条视频`);
+
+  // console.log(`下一页页码为：${principalId}`);
+  TotalFeeds = [...TotalFeeds, ...feeds];
+  console.log("当前总视频数为：", TotalFeeds.length);
+  console.log("");
+  feeds.forEach(async item => {
+    await getVideoData(item.photo.photoUrl, 'binary', item.photo.id);
+  });
+}
+
+/**
+ * 发送get请求获取视频流数据
+ * @param {*} url  视频地址 
+ * @param {*} encoding  获取的类型
+ * @param {*} filename  文件名
+ * @returns 
+ */
+const getVideoData = (url, encoding, filename) => {
+  return new Promise((resolve, reject) => {
+    let req = https.get(url, function (res) {
+      let result = ''
+      encoding && res.setEncoding(encoding)
+      res.on('data', function (d) {
+        result += d
+      })
+      res.on('end', function () {
+        savefileToPath(result, filename)
+      })
+      res.on('error', function (e) {
+        console.log("1 res.on 出错了：", e);
+        console.log("重新请求数据中：", url);
+        getVideoData(url, encoding, filename)
+        // reject(e)
+      })
+    })
+    req.end()
+  })
+}
+
+/**
+ * 保存视频到本地
+ * @param {*} fileData 视频流数据
+ * @param {*} fileName 文件名
+ * @returns 
+ */
+const savefileToPath = async (fileData, fileName) => {
+  let fileFullName = `dist/${fileName}.mp4`;
+  return new Promise((resolve, reject) => {
+    fs.writeFile(fileFullName, fileData, 'binary', function (err) {
+      if (err) {
+        console.log('savefileToPath error:', err)
+      } else {
+        TotalCount += 1;
+        console.log(`第 ${TotalCount} 条视频：${fileName}.mp4 已下载！`);
+        Screenshot(fileName)
+        if (TotalCount >= TotalFeeds.length) {
+          console.log("");
+          console.log(`已下载：${TotalFeeds.length} 条视频，20秒后将开始继续下载..`);
+          console.log("结束 ==============================================================");
+          setTimeout(() => {
+            main();
+          }, 10000)
+        }
+        resolve('已下载')
+      }
+    })
+  })
+}
+
+/**
+ * 对下载的视频进行截图
+ * @param {*} fileName 视频文件名称
+ */
+const Screenshot = async (fileName) => {
+  var shell = `ffmpeg -ss 00:00:01 -i dist/${fileName}.mp4 -y -f image2 -frames:v 1 screenshot/${fileName}.jpg`;
+  console.log(`正在为视频: ${fileName}.mp4 进行截图：`);
+  process.execSync(shell);
+}
+
+main()
+
+
+// ffmpeg -ss 00:00:06 -i dist/video_1.mp4 -y -f image2 -frames:v 1 screenshot/sample.jpg
+
+// 实际下载：1648
+// 缺少1个视频
+
+// 真实被收藏视频：1649
+// 原因：计数器从1开始的，需要改成0
+
+// document.querySelectorAll(".feed-list .feed-list-item").length  1680
+// document.querySelectorAll(".feed-list .feed-list-item .multiple").length 3
+// document.querySelectorAll(".feed-list .feed-list-item .vertical").length 26
+// document.querySelectorAll(".feed-list .feed-list-item .single").length 2