bmy/ksapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加下载数据到json

Browse Source
This commit is contained in:
编码猿
2025-07-23 23:52:30 +08:00
parent 9f998fc3e7
commit fe9c4137d5
10 changed files with 460 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
downKsVideo/app.js
View File

@@ -21,13 +21,31 @@ app.use((req, res, next) => {
})
app.get('/', async (req, res) => {
let { page } = req.query
let data = await request.post(config.visionProfileLikePhotoList, {
page: "profile",
pcursor: "空"
pcursor: page
});
res.json(data)
})
app.listen(3000, () => {
console.log(`http://127.0.0.1:3000`)
app.get('/tj', async (req, res) => {
let { feeds } = await request.post(config.visionNewRecoFeed, {
dailyFirstPage: false
});
let result = []
feeds.forEach(v => {
result.push({
video_introduce: v.photo.caption,
video_url: v.photo.photoH265Url
})
});
res.json(result)
})
app.listen(3030, () => {
console.log(`http://127.0.0.1:3030`)
})

55
downKsVideo/data.js Normal file
View File

@@ -0,0 +1,55 @@
const request = require("./src/http/http");
const config = require("./src/config/config");
const fs = require('fs');
let result = []
// 延迟函数，单位为毫秒
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const main = async () => {
for (let index = 1; index <= 50; index++) {
try {
let { feeds } = await request.post(config.visionNewRecoFeed, { dailyFirstPage: false });
console.log(`第: ${index} 次爬`);
console.log(`爬到的数据长度: ${feeds.length}`);
console.log(``);
feeds.forEach(v => {
result.push({
video_introduce: v.photo.caption,
video_url: v.photo.photoUrl,
video_img: v.photo.coverUrl
});
});
// 设置延迟时间（例如3000毫秒=3秒）
await delay(1000); // 你可以调整这个值
} catch (error) {
console.error(`第 ${index} 次请求出错:`, error);
// 出错时也等待一段时间再继续
await delay(3000);
}
}
// 先读取现有数据，然后追加新数据
fs.readFile('data.json', 'utf8', (err, data) => {
if (err) {
console.error('读取文件出错:', err);
return;
}
const existingData = JSON.parse(data);
console.log(`data.json 原文件数据长度: ${existingData.length}`);
console.log(`爬到的新数据长度: ${result.length}`);
const updatedData = [...existingData, ...result];
console.log(`合并后总数据长度: ${updatedData.length}`);
fs.writeFile('data.json', JSON.stringify(updatedData, null, 2), (err) => {
if (err) console.error('写入文件出错:', err);
else console.log('数据已成功追加到data.json');
});
});
}
main();

147
downKsVideo/index.html
View File

@@ -4,6 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Document</title>
<script src="./lazyload.min.js"></script>
<style>
* {
margin: 0;
@@ -24,8 +25,10 @@
justify-content: center;
cursor: pointer;
}
ul li img {
ul li .lazy {
display: block;
width: 100%;
height: 100%;
}
</style>
</head>
@@ -33,29 +36,129 @@
<ul></ul>
</body>
<script>
fetch("http://127.0.0.1:3000/")
.then((response) => response.json())
.then((data) => {
console.log(data);
let { feeds, pcursor } = data;
let html = feeds
.map((e) => {
return `
<li>
<img src="${e.photo.coverUrl}" alt="">
</li>
`;
})
.toString()
.replaceAll(",", "");
let page = "";
// 获取当前页面的滚动高度
let currentTop = document.documentElement.scrollTop,
lload = null,
scrollLock = false,
handlerId = null,
ul = document.querySelector("ul");
document.querySelector("ul").innerHTML += html;
let getHttpData = async () => {
let res = await fetch(`http://127.0.0.1:3030?page=${page}`);
let data = await res.json();
scrollLock = false;
render(data);
};
document.querySelectorAll("ul li").forEach((e) => {
e.onmouseover = () => {
console.log("e: ", e);
};
});
// 渲染页面
let render = (data) => {
let { feeds, pcursor } = data;
console.log(feeds, pcursor);
page = pcursor;
let index = 0;
function add() {
const li = document.createElement("li");
const video = document.createElement("video");
video.className = "lazy";
video.preload = "metadata";
video.poster = feeds[index].photo.coverUrl;
video.dataset.src = feeds[index].photo.photoUrl;
video.muted = true;
video.autoplay = true;
video.loop = true;
li.appendChild(video);
ul.appendChild(li);
index += 1; //修改图像的位置
if (index < feeds.length) {
//在动画没有结束前，递归渲染
requestAnimationFrame(add);
} else {
initLazyLoad();
if (currentTop == 0) {
autoLoad();
}
}
}
requestAnimationFrame(add);
};
let initLazyLoad = () => {
lload = new LazyLoad({
// use_native: true,
threshold: 0,
elements_selector: ".lazy",
});
};
let autoLoad = () => {
var setScroll = () => {
scroll(0, (currentTop += 2));
handlerId = requestAnimationFrame(setScroll);
};
handlerId = requestAnimationFrame(setScroll);
// // 设置一个计时器，每隔200ms执行一次滚动操作
// var time = setInterval(() => {
// // 使用 window.scroll 方法滚动页面，currentTop 增加 2 实现向下滚动
// window.scroll(0, (currentTop += 1));
// }, 10);
};
// 当用户点击页面时，停止自动滚动
document.onclick = function () {
cancelAnimationFrame(handlerId);
handlerId = null
};
document.onkeydown = function (e) {
//对整个页面监听
var keyNum = window.event ? e.keyCode : e.which; //获取被按下的键值
//判断如果用户按下了回车键（keycody=13）
if (keyNum == 13 && handlerId == null) {
autoLoad();
}
};
window.onscroll = function () {
//文档内容实际高度（包括超出视窗的溢出部分）
var scrollHeight = Math.max(
document.documentElement.scrollHeight,
document.body.scrollHeight
);
//滚动条滚动距离
var scrollTop =
window.pageYOffset ||
document.documentElement.scrollTop ||
document.body.scrollTop;
//窗口可视范围高度
var clientHeight =
window.innerHeight ||
Math.min(
document.documentElement.clientHeight,
document.body.clientHeight
);
if (clientHeight + scrollTop >= scrollHeight - 150) {
if (scrollLock == false) {
console.log("满足");
scrollLock = true;
getHttpData();
}
}
};
getHttpData();
</script>
</html>

1
downKsVideo/lazyload.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
downKsVideo/package.json
View File

@@ -5,7 +5,8 @@
"main": "index.js",
"scripts": {
"dev": "node --max-old-space-size=20240 src/index.js",
"http": "node-dev app.js"
"http": "node-dev app.js",
"tj": "node-dev src/tuijian.js"
},
"keywords": [],
"author": "",
@@ -15,4 +16,4 @@
"graphql": "^15.5.0",
"graphql-request": "^3.4.0"
}
}
}

16
downKsVideo/src/config/config.js
View File

@@ -81,5 +81,21 @@ module.exports = {
feeds { ...feedContentWithLiveInfo __typename }
hostName pcursor __typename }
}
`,
visionNewRecoFeed: gql`
fragment photoContent on PhotoEntity { __typename id duration caption originCaption likeCount viewCount commentCount realLikeCount coverUrl photoUrl photoH265Url manifest manifestH265 videoResource coverUrls { url __typename } timestamp expTag animatedCoverUrl distance videoRatio liked stereoType profileUserTopPhoto musicBlocked riskTagContent riskTagUrl
}
fragment recoPhotoFragment on recoPhotoEntity { __typename id duration caption originCaption likeCount viewCount commentCount realLikeCount coverUrl photoUrl photoH265Url manifest manifestH265 videoResource coverUrls { url __typename } timestamp expTag animatedCoverUrl distance videoRatio liked stereoType profileUserTopPhoto musicBlocked riskTagContent riskTagUrl
}
fragment feedContentWithLiveInfo on Feed { type author { id name headerUrl following livingInfo headerUrls { url __typename } __typename } photo { ...photoContent ...recoPhotoFragment __typename } canAddComment llsid status currentPcursor tags { type name __typename } __typename
}
fragment photoResult on PhotoResult { result llsid expTag serverExpTag pcursor feeds { ...feedContentWithLiveInfo __typename } webPageArea __typename
}
query visionNewRecoFeed($semKeyword: String, $semCrowd: String, $utmSource: String, $utmMedium: String, $utmCampaign: String, $dailyFirstPage: Boolean) { visionNewRecoFeed(semKeyword: $semKeyword, semCrowd: $semCrowd, utmSource: $utmSource, utmMedium: $utmMedium, utmCampaign: $utmCampaign, dailyFirstPage: $dailyFirstPage) { ...photoResult __typename }
}
`
};

18
downKsVideo/src/http/http.js
View File

@@ -5,19 +5,19 @@ class GraphqlRequest {
constructor() {
this.client = new GraphQLClient(config.url, {
headers: {
Cookie: 'kpf=PC_WEB; clientid=3; did=web_4abc14d7790d18335fedc13249834c42; userId=58344290; kuaishou.server.webday7_st=ChprdWFpc2hvdS5zZXJ2ZXIud2ViZGF5Ny5zdBKwARn6brG-fMigWK_p8OkNIz3auEEilOc5O0XadgJ_dkmu41zuO2zSfxcmYBuyqph0KnFS-cVu-YbgPb2HqxwbgaznjeTKtJzieVkfoYClBCJy2Hojx7euihIStw4izkYtP7ztIaIAOK5_kiGFvS2_6V4jAXupKegEjQeM9nd2vxpkSuw63dHwZDzfN1EmTr5__nCExUlhCcxi-vyBl446ReOLlywiTjMe5AGI2_I1uz4dGhLcWLZU0wMn7F5ONAeuFgyQkCwiIFEFfJL9dH1pCQZANFR6CCZP1SaZ_xw7ZpeNre2m31gjKAUwAQ; kuaishou.server.webday7_ph=07c02e409d3bb524d0531c7d3e2d8b539eb6; kpn=KUAISHOU_VISION'
Cookie: 'did=web_8e01e1d7a6199e8ad21de2e5f486f2db; clientid=3; kpf=PC_WEB; kpn=KUAISHOU_VISION'
},
})
}
async post(document, variables) {
try {
let res = await this.client.request(document, variables);
return res.visionProfileLikePhotoList || res.visionProfilePhotoList;
} catch (error) {
console.log("===========================请求出错===========================");
console.log(error.message);
console.log("===========================请求出错===========================");
}
try {
let res = await this.client.request(document, variables);
return res.visionProfileLikePhotoList || res.visionProfilePhotoList || res.visionNewRecoFeed;
} catch (error) {
console.log("===========================请求出错===========================");
console.log(error.message);
console.log("===========================请求出错===========================");
}
}
}

132
downKsVideo/src/utils/utils.js
View File

@@ -14,81 +14,81 @@ pr.on('uncaughtException', err => {
class Utils {
TotalFeeds = []
TotalCount = 0
principalId = ""
type = ""
userId
TotalFeeds = []
TotalCount = 0
principalId = ""
type = ""
userId
init (type = "visionProfileLikePhotoList", userId) {
this.type = type
this.userId = userId
}
init(type = "visionProfileLikePhotoList", userId) {
this.type = type
this.userId = userId
}
async start() {
let params = { page: "profile", pcursor: this.principalId }
async start(params) {
let params = { page: "profile", pcursor: this.principalId }
this.userId ? params.userId = this.userId : null
let { pcursor, feeds } = await request.post(config[this.type], params);
this.principalId = pcursor;
this.userId ? params.userId = this.userId : null
let { pcursor, feeds } = await request.post(config[this.type], params);
console.log("");
console.log(`1：本单次请求到：${feeds.length} 条视频`);
// console.log(`下一页页码为：${principalId}`);
this.TotalFeeds = [...this.TotalFeeds, ...feeds];
console.log("2：当前总视频数为：", this.TotalFeeds.length);
console.log("");
console.log("-----------视频下载进度-------------");
feeds.forEach(async item => {
await this.getVideoData(item.photo.photoUrl, 'binary', item.photo.id);
});
}
this.principalId = pcursor;
async getVideoData(url, encoding, filename) {
return new Promise((resolve, reject) => {
let req = https.get(url, res => {
let result = ''
encoding && res.setEncoding(encoding)
res.on('data', d => result += d)
res.on('end', () => this.savefileToPath(result, filename))
res.on('error', e => {
console.log("1 res.on 出错了：", e);
console.log("重新请求数据中：", url);
this.getVideoData(url, encoding, filename)
})
})
req.end()
console.log("");
console.log(`1：本单次请求到：${feeds.length} 条视频`);
// console.log(`下一页页码为：${principalId}`);
this.TotalFeeds = [...this.TotalFeeds, ...feeds];
console.log("2：当前总视频数为：", this.TotalFeeds.length);
console.log("");
console.log("-----------视频下载进度-------------");
feeds.forEach(async item => {
await this.getVideoData(item.photo.photoUrl, 'binary', item.photo.id);
});
}
async getVideoData(url, encoding, filename) {
return new Promise((resolve, reject) => {
let req = https.get(url, res => {
let result = ''
encoding && res.setEncoding(encoding)
res.on('data', d => result += d)
res.on('end', () => this.savefileToPath(result, filename))
res.on('error', e => {
console.log("1 res.on 出错了：", e);
console.log("重新请求数据中：", url);
this.getVideoData(url, encoding, filename)
})
}
})
req.end()
})
}
async savefileToPath(fileData, fileName) {
let fileFullName = path.join(__dirname, `../../dist/${fileName}.mp4`)
return new Promise((resolve, reject) => {
fs.writeFile(fileFullName, fileData, 'binary', err => {
if (err) {
console.log('savefileToPath error:', err)
} else {
this.TotalCount += 1;
console.log(`第 ${this.TotalCount} 条视频：${fileName}.mp4 已保存到本地！`);
// Screenshot(fileName)
if (this.TotalCount >= this.TotalFeeds.length) {
console.log("-----------视频下载进度-------------");
console.log("");
console.log(`3：已下载：${this.TotalFeeds.length} 条视频，5秒 后将开始下载 下一页..`);
console.log("")
console.log("")
setTimeout(() => this.start(), 5000)
}
resolve('已下载')
}
})
})
}
async savefileToPath(fileData, fileName) {
let fileFullName = path.join(__dirname, `../../dist/${fileName}.mp4`)
return new Promise((resolve, reject) => {
fs.writeFile(fileFullName, fileData, 'binary', err => {
if (err) {
console.log('savefileToPath error:', err)
} else {
this.TotalCount += 1;
console.log(`第 ${this.TotalCount} 条视频：${fileName}.mp4 已保存到本地！`);
// Screenshot(fileName)
if (this.TotalCount >= this.TotalFeeds.length) {
console.log("-----------视频下载进度-------------");
console.log("");
console.log(`3：已下载：${this.TotalFeeds.length} 条视频，5秒 后将开始下载 下一页..`);
console.log("")
console.log("")
setTimeout(() => this.start(), 5000)
}
resolve('已下载')
}
})
})
}
}

2
downKsVideo/src/visionProfileLikePhotoList.js
View File

@@ -3,4 +3,4 @@ const utils = require("./utils/utils");
let u = new utils()
// u.init("visionProfileLikePhotoList")
u.init("visionProfilePhotoList", "3xiem77vya42p4y")
u.start()
u.start({ page: "profile", pcursor: this.principalId })

163
downKsVideo/test.html Normal file
View File

@@ -0,0 +1,163 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Document</title>
<script src="./lazyload.min.js"></script>
<style>
* {
margin: 0;
padding: 0;
list-style: none;
font-size: 0;
line-height: 0;
}
ul {
display: flex;
flex-wrap: wrap;
}
ul li {
width: 20%;
background: #000;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
}
ul li .lazy {
display: block;
width: 100%;
height: 100%;
}
</style>
</head>
<body>
<ul></ul>
</body>
<script>
let page = "";
// 获取当前页面的滚动高度
let currentTop = document.documentElement.scrollTop,
lload = null,
scrollLock = false,
handlerId = null,
ul = document.querySelector("ul");
let getHttpData = async () => {
let res = await fetch(`http://127.0.0.1:3000?page=${page}`);
let data = await res.json();
scrollLock = false;
render(data);
};
// 渲染页面
let render = (data) => {
let { feeds, pcursor } = data;
console.log(feeds, pcursor);
page = pcursor;
let index = 0;
function add() {
const li = document.createElement("li");
const video = document.createElement("video");
video.className = "lazy";
video.preload = "metadata";
video.poster = feeds[index].photo.coverUrl;
video.dataset.src = feeds[index].photo.photoUrl;
video.muted = true;
video.autoplay = true;
li.appendChild(video);
ul.appendChild(li);
index += 1; //修改图像的位置
if (index < feeds.length) {
//在动画没有结束前，递归渲染
requestAnimationFrame(add);
} else {
initLazyLoad();
if (currentTop == 0) {
autoLoad();
}
}
}
requestAnimationFrame(add);
};
let initLazyLoad = () => {
lload = new LazyLoad({
// use_native: true,
threshold: 0,
elements_selector: ".lazy",
});
};
let autoLoad = () => {
var setScroll = () => {
scroll(0, (currentTop += 2));
handlerId = requestAnimationFrame(setScroll);
};
handlerId = requestAnimationFrame(setScroll);
// // 设置一个计时器，每隔200ms执行一次滚动操作
// var time = setInterval(() => {
// // 使用 window.scroll 方法滚动页面，currentTop 增加 2 实现向下滚动
// window.scroll(0, (currentTop += 1));
// }, 10);
};
// 当用户点击页面时，停止自动滚动
document.onclick = function () {
cancelAnimationFrame(handlerId);
handlerId = null
};
document.onkeydown = function (e) {
//对整个页面监听
var keyNum = window.event ? e.keyCode : e.which; //获取被按下的键值
//判断如果用户按下了回车键（keycody=13）
if (keyNum == 13 && handlerId == null) {
autoLoad();
}
};
window.onscroll = function () {
//文档内容实际高度（包括超出视窗的溢出部分）
var scrollHeight = Math.max(
document.documentElement.scrollHeight,
document.body.scrollHeight
);
//滚动条滚动距离
var scrollTop =
window.pageYOffset ||
document.documentElement.scrollTop ||
document.body.scrollTop;
//窗口可视范围高度
var clientHeight =
window.innerHeight ||
Math.min(
document.documentElement.clientHeight,
document.body.clientHeight
);
if (clientHeight + scrollTop >= scrollHeight - 150) {
if (scrollLock == false) {
console.log("满足");
scrollLock = true;
getHttpData();
}
}
};
getHttpData();
</script>
</html>