archive the notes
// ==UserScript==
// @name dA_archive_notes
// @namespace http://tampermonkey.net/
// @version 1.1
// @description archive the notes
// @author Dediggefedde
// @match http://*.deviantart.com/notifications/notes/*
// @match https://*.deviantart.com/notifications/notes/*
// @icon https://www.google.com/s2/favicons?sz=64&domain=deviantart.com
// @resource viewer https://phi.pf-control.de/userscripts/dA_archive_notes/Viewer.html
// @resource starter https://phi.pf-control.de/userscripts/dA_archive_notes/chrome_starter.bat
// @grant GM.addStyle
// @grant GM_getResourceText
// ==/UserScript==
/* global DiFi*/
/* global tiny_zip*/
/* global uint8array_from_binstr */
// @ts-check
(function() {
'use strict';
let viewer = GM_getResourceText("viewer")
let starter = GM_getResourceText("starter")
let debugLog = false;
GM.addStyle(`
#dA_AN_Menu{display:none;position: absolute;top: 0;left: 0;width: 200px;height: 170px;background: lightblue;border-radius: 15px;border: 2px ridge black;padding: 10px;box-shadow: -2px -2px 5px #0df inset;}
#dA_AN_Menu input[type=number]{line-height:1em;width:50px;}
#dA_AN_btnArchive{transform: translateX(-55%);left: 50%;position: relative;}
#dA_AN_Menu button{cursor:pointer;}
#dA_AN_Menu>*{margin:5px;}
#dA_AN_selectFolder{width:95%;}
#dA_AN_btnX{position:absolute;top:5px;right:5px;}
`);
/**
* Will download a file without user confirmation!
* Target is download-folder
* @param {*} content file content, here zip-file in Blob format
* @param {*} mimeType type of file, here octec-stream
* @param {*} filename file-name file is downloaded as
*/
function download(content, mimeType, filename) {
const a = document.createElement('a') // Create "a" element
const blob = new Blob([content], { type: mimeType }) // Create a blob (file-like object)
const url = URL.createObjectURL(blob) // Create an object URL from blob
a.setAttribute('href', url) // Set "a" element link
a.setAttribute('download', filename) // Set download filename
a.click() // Start downloading
}
/**
* dictionary folder_key for note information
* @type {Object.<string, {id:number,folder: string, sender: string,date:string,subject:string}>}
*/
let noteIds = {}; //list of note-ids with information to fetch, id=>{folder,sender,date,subject}
let noteMSGs = {}; //list of note contents fetched. id=>text
let pending = {}; //pages/notes still pending. Note: first all pages are scanned through, then all notes
let hiddenDom = document.createElement("div"); //parsing element to fetch note-information in noteIds
let folderNames = {}; //dictionary for folder names
let maxPrg = 1;
/** Interface to ensure noteIds structure */
function addNoteId(id, folder, sender, date, subject) {
noteIds[`${folder}_${id}`] = { id, folder, sender, date, subject };
}
/** Interface to ensure noteMSGs structure */
function addMsg(id, text) {
noteMSGs[id] = text;
}
/** request first page of folder to fetch max-page from botton.
* returns promise, resolve [folder,max page], reject error or data.response if server error
*/
function fetchMaxPage(folder) {
return new Promise((resolve, reject) => {
try {
DiFi.pushPost('Notes', 'display_folder', [folder, 0, false], (success, data) => {
try {
if (debugLog) console.log("dA_archive_notes fetchMaxPage ", folder, data.response);
if (data.response.status != "SUCCESS") {
return reject(data.response);
}
hiddenDom.innerHTML = data.response.content.body;
let pgs = hiddenDom.querySelectorAll("li.number a.away");
if (pgs.length < 2)
resolve([folder, 1]);
else
resolve([folder, parseInt(pgs[1].innerText)]);
} catch (ex) {
reject(ex);
}
});
DiFi.send();
} catch (ex) {
reject(ex);
}
})
}
/**
* fetches note IDs to fetch content later for.
* @param {string|number} folder "1" inbox, "0" draft, some named (unread), see list properties a["data-component"] or url ending #folder_offset
*/
function fetchIDs(folder, startPage, stopPage) {
startPage = parseInt(startPage);
stopPage = parseInt(stopPage);
return new Promise((resolve, reject) => {
try {
pending[folder] = 0;
let chunksize = 10;
for (let ch = 0; ch < Math.ceil((stopPage - startPage + 1) / chunksize); ++ch) {
for (let page = startPage + ch * chunksize; page < (startPage + (ch + 1) * chunksize) && (page <= stopPage); ++page) {
++pending[folder];
DiFi.pushPost('Notes', 'display_folder', [folder, (page - 1) * 10, false], function(success, data) {
if (debugLog) console.log("dA_archive_notes fetchIDs ", folder, page, data.response);
let id, name, date, subject;
if (data.response.status != "SUCCESS") {
//return reject(data.response);
} else {
hiddenDom.innerHTML = data.response.content.body;
Array.from(hiddenDom.querySelectorAll("li[data-noteid]")).forEach(el => {
id = el.getAttribute("data-noteid");
name = el.querySelector("span.sender").innerText.trim();
date = el.querySelector("span.ts").innerText.trim();
subject = el.querySelector("span.subject").innerText.trim();
addNoteId(id, folder, name, date, subject);
});
}
if (--pending[folder] == 0) {
resolve();
}
showProgress();
});
}
DiFi.send();
}
} catch (ex) {
reject(ex);
}
});
}
/**
* Fetches all Notes in noteIds and fills noteMSGs
* updates pending = remaining notes
* @returns Promise resolved when all notes fetched, rejected if one request failed or error occured
*/
function fetchNote() {
return new Promise((resolve, reject) => {
try {
pending["__notes"] = 0;
const chunkSize = 10;
for (let i = 0; i < Object.entries(noteIds).length; i += chunkSize) {
const chunk = Object.entries(noteIds).slice(i, i + chunkSize);
chunk.forEach(([id, note]) => {
++pending["__notes"];
DiFi.pushPost('Notes', 'display_note', [note.folder, note.id], function(success, data) {
if (debugLog) console.log("dA_archive_notes fetchNote ", note.folder, note.id, data.response);
if (data.response.status != "SUCCESS") {
return reject(data.response);
}
addMsg(id, data.response.content.body);
if (--pending["__notes"] == 0) {
resolve();
}
showProgress();
});
});
DiFi.send();
}
} catch (ex) {
reject(ex);
}
});
}
/**
* 1. makes a request in folder for each page in range to get list of notes
* 2. makes a bundled request for each note to get the content
* 3. calls zipResonse to create summary file and zip result for download
* @param {string} folder folder-ID
* @param {number} pageStart page to start downloading on
* @param {number} pageStop page to stop doanloding on
*/
function downloadFolder(folder, pageStart, pageStop) {
noteIds = {};
noteMSGs = {};
pending = {};
maxPrg = 1;
if (folder != "0") {
maxPrg = pageStop - pageStart + 1;
showProgress();
fetchIDs(folder, pageStart, pageStop).then((ret) => {
maxPrg = Object.keys(noteIds).length;
return fetchNote();
}).then((ret) => {
zipResponse();
}).catch(err => {
console.log("dA_archive_notes error2", err);
alert("Error downloading archive:\n" + err);
});
} else { //ALL folders
Promise.all(Object.keys(folderNames).map(fol => {
return fetchMaxPage(fol);
})).then((ret) => {
maxPrg = 0;
ret.forEach(([fol, maxP]) => { maxPrg += maxP; });
return Promise.all(ret.map(([fol, maxP]) => { return fetchIDs(fol, 1, maxP); }));
}).then((ret) => {
maxPrg = Object.keys(noteIds).length;
return fetchNote();
}).then((ret) => {
zipResponse();
}).catch(err => {
console.log("dA_archive_notes error3", err);
alert("Error downloading archive:\n" + err);
});
}
}
function showProgress() {
let sum = 0;
if (!Number.isInteger(maxPrg) || maxPrg == 0) maxPrg = 1;
Object.values(pending).forEach(pen => {
sum += pen;
});
let pr = Math.round((maxPrg - sum) / maxPrg * 100);
if (pr < 1) pr = 1;
if (pr > 100) pr = 100;
setTimeout(() => { document.getElementById("dA_AN_progress").value = pr; }, 500);
}
function getNoteFileName(id) {
let note = noteIds[id];
return `${folderNames[note.folder]}_${note.id}_${note.sender}_${note.date}.html`;
}
/**
* Creates summary file for noteIds, puts noteMSGs into html files, puts all into zip file, triggers download
*/
function zipResponse() {
let zip = new tiny_zip();
let contenttext = "ID\tFolder\tSender\tDate\tSubject\tFile\n";
contenttext += Object.entries(noteIds).map(([id, note]) => {
return `${note.id}\t${folderNames[note.folder]}\t` +
`${note.sender}\t${note.date}\t${note.subject}\t` +
`${getNoteFileName(id)}`
}).join("\n");
zip.add("content.tsv", uint8array_from_binstr(contenttext));
Object.entries(noteMSGs).forEach(([id, note]) => {
zip.add(getNoteFileName(id), uint8array_from_binstr(note));
});
const dt = (new Date()).toISOString().replace(/\D/gi, el => {
return { "-": "-", "T": "_", "Z": "", ":": "-", ".": "-" }[el];
}).slice(0, -5);
zip.add("_Viewer.html", uint8array_from_binstr(viewer));
zip.add("_chrome_starter.bat", uint8array_from_binstr(starter));
download(zip.generate(), "application/octet-stream", `dA_archive_notes_${dt}.zip`);
}
/** injects GUI and attaches event handlers */
function fillGUI() {
let pgs = document.querySelectorAll("li.number a.away");
let finalP = pgs[1].innerText;
let bar = document.createElement("div");
bar.id = "dA_AN_Menu";
let domEl = document.createElement("h3");
domEl.innerHTML = "Export Notes";
bar.appendChild(domEl);
let selFolder = document.createElement("select");
selFolder.id = "dA_AN_selectFolder";
let fragment = new DocumentFragment();
let opt = document.createElement('option');
opt.value = "0";
opt.innerHTML = "All Folders";
fragment.appendChild(opt);
Array.from(document.querySelectorAll("a.folder-link[data-folderid]")).forEach(el => {
let folder = el.getAttribute("data-folderid");
opt = document.createElement('option');
opt.value = folder;
opt.innerHTML = el.getAttribute("title");
fragment.appendChild(opt);
folderNames[folder] = el.getAttribute("title");
});
selFolder.appendChild(fragment);
bar.append(selFolder);
domEl = document.createElement("br");
bar.appendChild(domEl);
domEl = document.createElement("label");
domEl.for = "dA_AN_minP";
domEl.innerHTML = "Pages:";
bar.appendChild(domEl);
domEl = document.createElement("input");
domEl.type = "number";
domEl.id = "dA_AN_minP";
domEl.min = 1;
domEl.max = finalP;
domEl.value = 1;
bar.appendChild(domEl);
domEl = document.createElement("input");
domEl.type = "number";
domEl.id = "dA_AN_maxP";
domEl.min = 1;
domEl.max = finalP;
domEl.value = finalP;
bar.appendChild(domEl);
domEl = document.createElement("br");
bar.appendChild(domEl);
domEl = document.createElement("progress");
domEl.id = "dA_AN_progress";
domEl.value = 0;
domEl.max = 100;
bar.appendChild(domEl);
let btn = document.createElement("button");
btn.innerHTML = "Archive";
btn.id = "dA_AN_btnArchive";
btn.addEventListener("click", (ev) => {
let minP = document.getElementById("dA_AN_minP").value;
let maxP = document.getElementById("dA_AN_maxP").value;
let folder = document.getElementById("dA_AN_selectFolder").value;
downloadFolder(folder, minP, maxP);
}, false);
bar.append(btn);
btn = document.createElement("button");
btn.innerHTML = "X";
btn.id = "dA_AN_btnX";
btn.addEventListener("click", (ev) => {
bar.style.display = ""; //default none
}, false);
bar.append(btn);
let showBut = document.createElement("a");
showBut.innerHTML = "Archive";
showBut.id = "dA_AN_aShowDialog";
showBut.addEventListener("click", (ev) => {
bar.style.display = "block"; //default none
}, false);
bar.addEventListener("change", (ev) => {
ev.stopPropagation();
let minEl = document.getElementById("dA_AN_minP");
let maxEl = document.getElementById("dA_AN_maxP");
let maxLim = parseInt(maxEl.max);
let minLim = parseInt(maxEl.min);
let curVal = parseInt(ev.target.value);
if (ev.target.min && curVal < minLim) ev.target.value = minLim;
if (ev.target.max && curVal > maxLim) ev.target.value = maxLim;
if (ev.target.id == "dA_AN_minP" && curVal > parseInt(maxEl.value)) maxEl.value = minEl.value;
if (ev.target.id == "dA_AN_maxP" && curVal < parseInt(minEl.value)) {
ev.preventDefault();
minEl.value = maxEl.value;
return false;
}
if (ev.target.id == "dA_AN_selectFolder") {
fetchMaxPage(ev.target.value).then(([folder, maxP]) => {
minEl.max = maxP;
maxEl.max = maxP;
maxEl.value = maxP;
minEl.value = 1;
}).catch(err => {
console.log("dA_archive_notes error1", err);
});
}
}, true);
let ank = document.querySelector("div.note-controls.note-actions");
ank.append(showBut);
ank.append(bar);
}
/** Initial entry, called every second for javascript navigation, injects GUI */
function init() {
if (document.getElementById("dA_AN_Menu") != null) return; //already present
let pgs = document.querySelectorAll("li.number a.away");
if (pgs.length < 2) return; //Page not yet loaded
fillGUI();
}
//javascript navigation workaround
setInterval(init, 1000);
//ressource for creating zip:
/*
Copyright (C) 2013 https://github.com/vuplea
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
function uint8array_from_binstr(string) {
var binary = new Uint8Array(string.length);
for (var i = 0; i < string.length; ++i)
binary.set([string.charCodeAt(i)], i);
return binary;
}
function tiny_zip() {
var localHs = [];
var contents = [];
var local_offset = 0;
var centralHs = [];
var central_offset = 0;
this.add = function(nameStr, content) {
var utf8array_from_str = function(string) {
return uint8array_from_binstr(unescape(encodeURIComponent(string)));
};
var name = utf8array_from_str(nameStr.replace(/[\/\:*?"<>\\|]/g, "_").slice(0, 255));
var nlen = name.length;
var clen = content.length;
var crc = crc32(content);
var localH = new Uint8Array(30 + nlen);
localH.set([0x50, 0x4b, 0x03, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, crc, crc >> 8,
crc >> 16, crc >> 24, clen, clen >> 8, clen >> 16, clen >> 24, clen, clen >> 8, clen >> 16, clen >> 24,
nlen, nlen >> 8, 0x00, 0x00
]);
localH.set(name, 30);
//
var centralH = new Uint8Array(46 + nlen);
var loff = local_offset;
centralH.set([0x50, 0x4b, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
crc, crc >> 8, crc >> 16, crc >> 24, clen, clen >> 8, clen >> 16, clen >> 24, clen, clen >> 8, clen >> 16,
clen >> 24, nlen, nlen >> 8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, loff,
loff >> 8, loff >> 16, loff >> 24
]);
centralH.set(name, 46);
central_offset += centralH.length;
//
local_offset += localH.length + content.length;
localHs.push(localH);
contents.push(content);
centralHs.push(centralH);
};
this.generate = function() {
var n = localHs.length;
//
var endof = new Uint8Array(22);
var loff = local_offset;
var coff = central_offset;
endof.set([0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, n, n >> 8, n, n >> 8, coff, coff >> 8, coff >> 16,
coff >> 24, loff, loff >> 8, loff >> 16, loff >> 24, 0x00, 0x00
]);
//
var outQueue = [];
for (var i = 0; i < n; ++i) {
outQueue.push(localHs[i]);
outQueue.push(contents[i]);
}
for (var i = 0; i < n; ++i)
outQueue.push(centralHs[i]);
outQueue.push(endof);
//
return new Blob(outQueue, { type: "data:application/zip" });
};
var crcTable = function() {
var Table = [];
for (var i = 0; i < 256; ++i) {
var crc = i;
for (var j = 0; j < 8; ++j)
crc = -(crc & 1) & 0xEDB88320 ^ (crc >>> 1);
Table[i] = crc;
}
return Table;
}();
var crc32 = function(data) {
var crc = -1;
for (var i = 0; i < data.length; ++i)
crc = (crc >>> 8) ^ crcTable[(crc ^ data[i]) & 0xFF];
return ~crc;
};
}
})();