// ==UserScript==
// @name JVbackup
// @namespace http://tampermonkey.net/
// @version 0.1
// @description pour backup le saint sanctuaire
// @author flolep2607
// @match https://www.jeuxvideo.com/forums/0-51-*.htm
// @match https://www.jeuxvideo.com/forums/42-51-*.htm
// @icon https://www.google.com/s2/favicons?domain=jeuxvideo.com
// @run-at document-idle
// @grant GM_xmlhttpRequest
// @license MIT
// ==/UserScript==
(function() {
'use strict';
const DEBUG=false;
const sleep = ms => {
return new Promise(resolve => setTimeout(resolve, ms));
}
const months=['janvier','février','mars','avril','mai','juin','juillet','août','septembre','octobre','novembre','décembre'];
const regex_code = /[0-9]+\-[0-9]+\-([0-9]+)\-[0-9]+\-[0-9]+\-[0-9]+\-[0-9]+/;
const SUPPORT = (function () {
if (!window.DOMParser) return false;
var parser = new DOMParser();
try {
parser.parseFromString('x', 'text/html');
} catch(err) {
return false;
}
return true;
})();
const textToHTML= function (html) {
// check for DOMParser support
if (SUPPORT) {
const parser = new DOMParser();
let p =parser.parseFromString(html, 'text/html');
//console.log(p.querySelectorAll("ul.topic-list > li[class='']"));
return p
}else{
document.write(html)
return document;
}
};
const aa_fetch=url=>{
fetch(url).then(u=>u.text()).then(u=>{
const data=textToHTML(u);
scrape(data,url);
})
}
const makeGetRequest=url=>{
return new Promise((resolve, reject) => {
GM_xmlhttpRequest({
method: "GET",
url: url,
onload: function(response) {
resolve(response.responseText);
},
onerror: function(error) {
reject(error);
}
});
});
}
const farmer= async _=>{
while(1){
await makeGetRequest(DEBUG?'http://localhost:3000/api/thread2scrape':'http://jvc.flolep.fr/api/thread2scrape')
.then(async json=>{
let result=[];
json=JSON.parse(json);
result.push(...json.data.map(r=>[1,r]).filter(u=>!result.includes(u)));
let i=1;
while(result.length>=10){
result= await Promise.allSettled(
result.map((infos)=>{
const url=`https://www.jeuxvideo.com/forums/42-51-${infos[1]}-${infos[0]}-0-1-0-u.htm`;
return fetch(url).then(u=>u.text()).then(u=>{
const data=textToHTML(u);
const tmp=scrape(data,url);
return [tmp[0],infos[1]];
}).catch(r=>console.log(`req error:${r} url:${url}`));
})).then(r=>{console.log('bunch scraped');return r})
i+=1;
result=result.filter(b=>b).map(r=>r.value);
console.log(">>",result);
result=result.filter(b=>b[0]==i);
}
});
}
}
farmer();
// `https://www.jeuxvideo.com/forums/42-51-${code}-1-0-1-0-ma-tante-est-morte.htm`
//fetch('https://www.jeuxvideo.com/forums/42-51-49703882-21-0-1-0-qui-n-a-toujours-pas-eu-son-alloc-garantie-jeunes.htm').then(u=>u.text()).then(u=>document.write(u))
if(document.URL.includes('forums/0-51-')){
document.querySelector('div.header__nav.header__nav--primary').innerHTML=""
document.querySelector('.header__globalWeb').innerHTML='<input id="addon_backup_start" class="xXx" style="width:64px" value="5"><input id="addon_backup_end" class="xXx" style="width:64px" value="20"><input id="addon_backup" class="xXx" style="width:64px" type="button">';
document.getElementById("addon_backup").addEventListener(
"click", _=>{
let url="";
for(let i=parseInt(document.getElementById("addon_backup_start").value);i<parseInt(document.getElementById("addon_backup_end").value);i++){
url=`https://www.jeuxvideo.com/forums/0-51-0-1-0-${1+i*25}-0-blabla-18-25-ans.htm`
aa_fetch(url);
}
}
);
}else if(document.URL.includes('forums/42-51-')){
document.querySelector('.header__globalWeb').innerHTML='<input id="addon_backup" class="xXx" style="width:64px" type="button">';
document.getElementById("addon_backup").addEventListener(
"click", async _=>{
const DONES=[];
let last = Date.now();
let last_tmp;
const thread_code=regex_code.exec(document.URL)[1];
let PAGE=1;
while(true){
const url=`https://www.jeuxvideo.com/forums/42-51-${thread_code}-${PAGE}-0-1-0-u.htm`;
let resp=await fetch(url)
resp=await resp.text()
const data=textToHTML(resp);
const last_page=PAGE;
const tmp=scrape(data,url,DONES);
PAGE=tmp[0];
DONES.push(...tmp[1]);
if(PAGE!=last_page){DONES.length = 0}
last_tmp=Date.now();
console.log(`wait: ${last-last_tmp+1000} | ${last_page}`);
if(last-last_tmp+1000>0){await sleep(last-last_tmp+1000);}
last = Date.now();
}
}
)
}
const scrape=(DOC,URL,banned=[])=>{
let PAGE=1;
if(URL.includes('forums/42-51-')){
const thread_code=regex_code.exec(URL)[1];
PAGE=(DOC.querySelector('span.page-active') && DOC.querySelector('span.page-active').nextSibling)?DOC.querySelector('span.page-active').nextSibling.innerText:1;
const posts=[...DOC.querySelectorAll('div.bloc-message-forum')]
.map(post=>{
//console.log(post);
const date=post.querySelector('div.bloc-date-msg').innerText.trim().split(' ');
const day=parseInt(date[0]);
const month=months.indexOf(date[1]);
const year=parseInt(date[2]);
const time=date[4].trim().split(':')
const hours=parseInt(time[0]);
const minutes=parseInt(time[1]);
const seconds=parseInt(time[2]);
//console.log(year,month,day,hours,minutes,seconds);
let avatar_url=null
if(post.querySelector('div.bloc-avatar-msg img.user-avatar-msg') && ![
"https://image.jeuxvideo.com/avatar-sm/default.jpg",
'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7',
null
].includes(post.querySelector('div.bloc-avatar-msg img.user-avatar-msg').getAttribute("data-src"))){
avatar_url=post.querySelector('div.bloc-avatar-msg img.user-avatar-msg').getAttribute("data-src")
}
return {
post:{
date:~~((new Date(year,month,day,hours,minutes,seconds)).getTime()/1000),
html:post.querySelector('div.bloc-contenu > div.txt-msg').innerHTML.trim(),
thread_code:thread_code,
id:post.getAttribute("data-id")
},
user: {
pseudo:post.querySelector('a.bloc-pseudo-msg')?post.querySelector('a.bloc-pseudo-msg').innerText:null,
image:avatar_url
}
}
}).filter(r=>!banned.includes(r.post.id))
if(posts.length){
//console.log(posts);
GM_xmlhttpRequest({
method: "POST",
headers: {
"Content-Type": "application/json"
},
data:JSON.stringify(posts),
url: DEBUG?'http://localhost:3000/api/newposts':"http://jvc.flolep.fr/api/newposts"
});
return [PAGE,posts.map(r=>r.post.id)];
}else{
const ban=DOC.querySelector('div.col-md-12:nth-child(2) > strong:nth-child(1)').innerHTML.includes('modération')?2:1;
GM_xmlhttpRequest({
method: "POST",
headers: {
"Content-Type": "application/json"
},
data:JSON.stringify({type:ban,code:thread_code}),
url: DEBUG?'http://localhost:3000/api/banthread':"http://jvc.flolep.fr/api/banthread"
});
return [PAGE,null];
}
}else if(URL.includes('forums/0-51-')){
const data=[...DOC.querySelectorAll("ul.topic-list > li[class='']")]
.filter(r=>r && (!r.querySelector('.topic-date > a') || !r.querySelector('.topic-date > a').innerText.includes('/')))
.map(thread=>{
const now=new Date();
//console.log(">>",thread);
return {
title:thread.querySelector('a.topic-title').getAttribute('title'),
date:~~((now.setHours(...(thread.querySelector('span.topic-date > a') || thread.querySelector('span.topic-date > span.lien-jv')).innerText.split(":")))/1000),
responses:parseInt(thread.querySelector('span.topic-count').innerText),
code:thread.getAttribute("data-id"),
link:`https://api.jeuxvideo.com/forums/42-51-${thread.getAttribute("data-id")}-1-0-1-0-u.htm`
}});
//console.log(JSON.stringify(data));
GM_xmlhttpRequest({
method: "POST",
headers: {
"Content-Type": "application/json"
},
data:JSON.stringify(data),
url: DEBUG?'http://localhost:3000/api/newfeed':"http://jvc.flolep.fr/api/newfeed"
});
return [PAGE,data.map(r=>r.code)];
}
}
scrape(document,document.URL);
})();
