可以抓取个人主页的微博内容
// ==UserScript==
// @name 微博内容抓取2.0
// @namespace wwww
// @license MPL
// @version 2.1.0
// @description 可以抓取个人主页的微博内容
// @match https://www.weibo.com/u/*
// @match https://weibo.com/u/*
// @grant GM_xmlhttpRequest
// @grant GM_download
// @require https://code.jquery.com/jquery-2.1.4.min.js
// @require https://cdn.bootcss.com/blueimp-md5/2.12.0/js/md5.min.js
// ==/UserScript==
(function() {
'use strict';
console.log("textstr");
//https://weibo.com/ajax/statuses/mymblog?uid=1260134537&page=1&feature=0
var button = document.createElement("button"); //创建一个input对象（提示框按钮）
button.id = "id001";
button.textContent = "开始抓取";
button.style.width = "90px";
button.style.height = "30px";
button.style.align = "center";
button.style.backgroundColor = "#ea8011";
var sleep_times = 1000*10;
var sum = 0;
var is_can_get_commests = true;
//绑定按键点击功能
button.onclick = function (){
sum = 0;
var repeat = prompt("请输入你要抓取的条数:","10");
var doc =document.getElementsByTagName('html')[0]
var uid = doc.baseURI.replace(/[^0-9]/ig,"")
var page = 1
is_can_get_commests = true;
get_blog_request(uid,page,null,repeat);
return;
};
var box = document.getElementById('cniil_wza');
//在浏览器控制台可以查看所有函数，ctrl+shift+I 调出控制台，在Console窗口进行实验测试
box.parentNode.insertBefore(button, box)
function get_blog_request(uid,page,since_id,repeat) {
if(sum > parseInt(repeat)){
alert("结束爬取，爬取条数：" + sum+"\n查询数据网址：http://43.134.187.127:8080/#/")
console.log("结束爬取，爬取条数：" + sum)
return;
}
//拼接url
var url = "https://weibo.com/ajax/statuses/mymblog?uid="+uid+"&page="+page+"&feature=0";
if(since_id != null){
url+= ("&since_id="+since_id)
}
GM_xmlhttpRequest({
method: "get",
url: url,
headers: {
"Content-Type": "application/json",
"cookie": document.cookie
},
onload: function(response){
var data = analysisResponse(response);
//发送微博下面的评论
if(data.data.list.length == 0){
alert("请求不到更多的数据，结束爬取，爬取条数：" + sum+"\n 查询数据网址：http://43.134.187.127:8080/#/")
console.log("请求不到更多的数据，结束爬取，爬取条数：" + sum)
return;
}
sendBlogData(data);
sum += data.data.list.length;
console.log("已成功爬取条数：" + sum)
delay_get_commests_rquest(data.data.list,uid,0);
sleep(sleep_times).then(() => {
get_blog_request(uid,page+1,getSinceId(data),repeat);
}
);
},
onerror: function(response){
console.log(response);
console.log("请求失败");
}
});
}
function delay_get_commests_rquest( dataList,uid, i){
if(dataList.length<=i){
return;
}
sleep(sleep_times/20).then(() => {
try{
var id = dataList[i].id;
get_commests_rquest(id,null,uid,0);
delay_get_commests_rquest(dataList,uid, i+1)
}catch{
console.log("delay_get_commests_rquest error：" + dataList[i])
console.log(dataList[i])
}
})
}
function analysisResponse(response ){
try{
var data = JSON.parse(response.response)
return data;
}catch{
is_can_get_commests = false;
}
}
function sendBlogData(data) {
GM_xmlhttpRequest({
method: "POST",
url: "http://43.134.187.127:8080/weibo/saveBlogs",
headers: {
"Content-Type": "application/json"
},
data:JSON.stringify(data.data.list),
onload: function(response){
// console.log("sendBlogData 请求成功");
},
onerror: function(response){
console.log("sendBlogData 请求失败");
console.log(JSON.stringify(data.data.list));
}
});
}
function getSinceId(data) {
// body...
return data.data.since_id
}
function get_commests_rquest(id,max_id,uid,sum) {
if(sum > 200){
return;
}
var url = "https://weibo.com/ajax/statuses/buildComments?flow=0&is_reload=1&id="+id+"&is_show_bulletin=2&is_mix=0&count=20&uid="+uid;
if(max_id!=null){
url +=("&max_id="+max_id);
}
console.log("抓取评论:"+id+"\n");
GM_xmlhttpRequest({
method: "get",
url: url,
headers: {
"Content-Type": "application/json",
"cookie": document.cookie
},
onload: function(response){
//解析数据
var data = analysisResponse(response);
for (var i = 0; i < data.data.length; i++) {
data.data[i].blogId = id;
}
sendCommestsData(data);
//如果还有数据 继续请求
if(data.data.length>=20){
sleep(sleep_times/20 + sum *10).then(() => {
// 这里写sleep之后需要去做的事情
get_commests_rquest(id,getMaxId(data),uid, sum + 20);
})
}
},
onerror: function(response){
console.log("get_commests_rquest 请求失败");
console.log(response);
}
});
}
function getMaxId(data) {
return data.max_id
}
function sendCommestsData(data) {
// console.log("sendCommestsData ");
GM_xmlhttpRequest({
method: "POST",
url: "http://43.134.187.127:8080/weibo/saveComments",
headers: {
"Content-Type": "application/json"
},
data:JSON.stringify(data.data),
onload: function(response){
// console.log("sendCommestsData 请求成功");
},
onerror: function(response){
console.log("sendCommestsData 请求失败");
}
});
}
function sleep (time) {
return new Promise((resolve) => setTimeout(resolve, time));
}
})();