bmy/ksapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
编码猿
2024-09-27 01:15:04 +08:00
commit 89004b22d2
47 changed files with 4871 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
downKsVideo/.pydio Normal file
View File

@@ -0,0 +1 @@
c91bfd8e-55d1-40b8-85d4-fdec0878e61b

151
downKsVideo/package-lock.json generated Normal file
View File

@@ -0,0 +1,151 @@
{
"name": "autoKuaiShou",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "autoKuaiShou",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"graphql": "^15.5.0",
"graphql-request": "^3.4.0"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/cross-fetch": {
"version": "3.1.8",
"resolved": "https://registry.npmmirror.com/cross-fetch/-/cross-fetch-3.1.8.tgz",
"integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
"dependencies": {
"node-fetch": "^2.6.12"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/extract-files": {
"version": "9.0.0",
"resolved": "https://registry.npmmirror.com/extract-files/-/extract-files-9.0.0.tgz",
"integrity": "sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==",
"engines": {
"node": "^10.17.0 || ^12.0.0 || >= 13.7.0"
},
"funding": {
"url": "https://github.com/sponsors/jaydenseric"
}
},
"node_modules/form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/graphql": {
"version": "15.9.0",
"resolved": "https://registry.npmmirror.com/graphql/-/graphql-15.9.0.tgz",
"integrity": "sha512-GCOQdvm7XxV1S4U4CGrsdlEN37245eC8P9zaYCMr6K1BG0IPGy5lUwmJsEOGyl1GD6HXjOtl2keCP9asRBwNvA==",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/graphql-request": {
"version": "3.7.0",
"resolved": "https://registry.npmmirror.com/graphql-request/-/graphql-request-3.7.0.tgz",
"integrity": "sha512-dw5PxHCgBneN2DDNqpWu8QkbbJ07oOziy8z+bK/TAXufsOLaETuVO4GkXrbs0WjhdKhBMN3BkpN/RIvUHkmNUQ==",
"dependencies": {
"cross-fetch": "^3.0.6",
"extract-files": "^9.0.0",
"form-data": "^3.0.0"
},
"peerDependencies": {
"graphql": "14 - 16"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
}
}
}

16
downKsVideo/package.json Normal file
View File

@@ -0,0 +1,16 @@
{
"name": "autoKuaiShou",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"dev": "node --max-old-space-size=20240 src/index.js"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"graphql": "^15.5.0",
"graphql-request": "^3.4.0"
}
}

1
downKsVideo/src/.pydio Normal file
View File

@@ -0,0 +1 @@
528077e4-c137-4dbf-9206-1909e22cda73

1
downKsVideo/src/config/.pydio Normal file
View File

@@ -0,0 +1 @@
4357a39c-dc52-4180-a387-00d44344f293

49
downKsVideo/src/config/config.js Normal file
View File

@@ -0,0 +1,49 @@
const { gql } = require("graphql-request");
module.exports = {
url: "https://www.kuaishou.com/graphql",
// 获取收藏的视频
visionProfileLikePhotoList: gql`
fragment photoContent on PhotoEntity {
__typename
id duration
caption originCaption
likeCount viewCount commentCount realLikeCount
coverUrl photoUrl photoH265Url manifest
manifestH265 videoResource coverUrls
{
url __typename
}
timestamp expTag animatedCoverUrl distance
videoRatio liked stereoType profileUserTopPhoto
musicBlocked riskTagContent riskTagUrl
}
fragment recoPhotoFragment on recoPhotoEntity {
__typename id duration caption originCaption likeCount
viewCount commentCount realLikeCount coverUrl photoUrl photoH265Url manifest
manifestH265 videoResource coverUrls
{ url __typename }
timestamp expTag animatedCoverUrl distance videoRatio liked stereoType
profileUserTopPhoto musicBlocked riskTagContent riskTagUrl
}
fragment feedContent on Feed {
type author
{
id name headerUrl following headerUrls
{
url __typename
} __typename
} photo {
...photoContent ...recoPhotoFragment __typename
} canAddComment llsid status currentPcursor tags
{ type name __typename }
__typename
}
query visionProfileLikePhotoList($pcursor: String, $page: String, $webPageArea: String) {
visionProfileLikePhotoList(pcursor: $pcursor, page: $page, webPageArea: $webPageArea) {
result llsid webPageArea feeds {
...feedContent __typename
} hostName pcursor __typename
}
}`,
};

1
downKsVideo/src/http/.pydio Normal file
View File

@@ -0,0 +1 @@
9aaa7146-0671-40e8-8c0b-15fb571f35a8

24
downKsVideo/src/http/http.js Normal file
View File

@@ -0,0 +1,24 @@
const { GraphQLClient } = require("graphql-request");
const config = require("../config/config");
class GraphqlRequest {
constructor() {
this.client = new GraphQLClient(config.url, {
headers: {
Cookie: 'context=1|MS43NjQ1ODM2OTgyODY2OTgyLjYxODQzOTUyLjE3MjQ2MTEwOTkyMjkuNDcyNDA=|MS43NjQ1ODM2OTgyODY2OTgyLjE5ODE4Mjk4LjE3MjQ2MTEwOTkyMjkuNDcyNDE=|0|graphql-server|webservice|false|NA; kpf=PC_WEB; clientid=3; did=web_017a1f82461c8b4dc575f8df3fee78ff; userId=58344290; kpn=KUAISHOU_VISION; kuaishou.server.web_st=ChZrdWFpc2hvdS5zZXJ2ZXIud2ViLnN0EqABVncUlGSoudMExPNRzEXf8OSAig_Gst7K3MuSob5dchyTwVVPrkPebW5B_ZMl9MFrNPRDjNhSpmGW_PABiZxH3bKuG7Cj-IaMhtEaaQiqs8ajQZfJ5CNjHKAmVizdwjF8qRnULuxZbrDwBE3BdzuUEDB-xE1ybAhhF0zQGz0LqzVDIT4ohfCbbzXfEZWCygEQCArtAgQB8sEooFEfLPXiwBoS5Uf6LooXNTrpm4StA9DDXaoVIiDqFSEmUkB66mi_cuqecbrHKQ5fBC5x74q1-WPiNNwCMSgFMAE;'
},
})
}
async post(document, variables) {
try {
let res = await this.client.request(document, variables);
return res.visionProfileLikePhotoList;
} catch (error) {
console.log("===========================请求出错===========================");
console.log(error.message);
console.log("===========================请求出错===========================");
}
}
}
module.exports = new GraphqlRequest();

126
downKsVideo/src/index.js Normal file
View File

@@ -0,0 +1,126 @@
const request = require("./http/http");
const config = require("./config/config");
const process = require('child_process');
const https = require("https");
const fs = require("fs");
const pr = require('node:process');
let TotalFeeds = [], TotalCount = 0, principalId = "空";
// 爬虫下载视频的速度过快（已做20秒延迟），快手服务器会突然断开连接，
// 从而http请求失效引发错误导致程序直接退出
// 这边从进程级捕获错误，避免程序中断，
// 下面的http error 部分会重新再次下载刚被拒绝的视频
// 同时数据太大，会导致超出nodejs的默认堆内存导致程序崩溃
// 所以启动时需要加上参数 --max-old-space-size=20240
pr.on('uncaughtException', err => {
console.log(' pr NOT exit...');
console.log(err.stack);
});
/**
* 自动下载快手主播主页所有视频和图片
*/
const main = async () => {
console.log("开始 ==============================================================");
console.log(`当前页码为：${principalId}`);
let { pcursor, feeds } = await request.post(config.visionProfileLikePhotoList, {
page: "profile",
pcursor: principalId
});
principalId = pcursor;
console.log(`本单次请求到：${feeds.length} 条视频`);
// console.log(`下一页页码为：${principalId}`);
TotalFeeds = [...TotalFeeds, ...feeds];
console.log("当前总视频数为：", TotalFeeds.length);
console.log("");
feeds.forEach(async item => {
await getVideoData(item.photo.photoUrl, 'binary', item.photo.id);
});
}
/**
* 发送get请求获取视频流数据
* @param {*} url 视频地址
* @param {*} encoding 获取的类型
* @param {*} filename 文件名
* @returns
*/
const getVideoData = (url, encoding, filename) => {
return new Promise((resolve, reject) => {
let req = https.get(url, function (res) {
let result = ''
encoding && res.setEncoding(encoding)
res.on('data', function (d) {
result += d
})
res.on('end', function () {
savefileToPath(result, filename)
})
res.on('error', function (e) {
console.log("1 res.on 出错了：", e);
console.log("重新请求数据中：", url);
getVideoData(url, encoding, filename)
// reject(e)
})
})
req.end()
})
}
/**
* 保存视频到本地
* @param {*} fileData 视频流数据
* @param {*} fileName 文件名
* @returns
*/
const savefileToPath = async (fileData, fileName) => {
let fileFullName = `dist/${fileName}.mp4`;
return new Promise((resolve, reject) => {
fs.writeFile(fileFullName, fileData, 'binary', function (err) {
if (err) {
console.log('savefileToPath error:', err)
} else {
TotalCount += 1;
console.log(`第 ${TotalCount} 条视频：${fileName}.mp4 已下载！`);
Screenshot(fileName)
if (TotalCount >= TotalFeeds.length) {
console.log("");
console.log(`已下载：${TotalFeeds.length} 条视频，20秒后将开始继续下载..`);
console.log("结束 ==============================================================");
setTimeout(() => {
main();
}, 10000)
}
resolve('已下载')
}
})
})
}
/**
* 对下载的视频进行截图
* @param {*} fileName 视频文件名称
*/
const Screenshot = async (fileName) => {
var shell = `ffmpeg -ss 00:00:01 -i dist/${fileName}.mp4 -y -f image2 -frames:v 1 screenshot/${fileName}.jpg`;
console.log(`正在为视频: ${fileName}.mp4 进行截图：`);
process.execSync(shell);
}
main()
// ffmpeg -ss 00:00:06 -i dist/video_1.mp4 -y -f image2 -frames:v 1 screenshot/sample.jpg
// 实际下载：1648
// 缺少1个视频
// 真实被收藏视频：1649
// 原因：计数器从1开始的，需要改成0
// document.querySelectorAll(".feed-list .feed-list-item").length 1680
// document.querySelectorAll(".feed-list .feed-list-item .multiple").length 3
// document.querySelectorAll(".feed-list .feed-list-item .vertical").length 26
// document.querySelectorAll(".feed-list .feed-list-item .single").length 2