// ==UserScript==
// @name Mturk Qualification Database and Scraper
// @namespace https://greasyfork.org/en/users/1004048-elias041
// @version 0.84
// @description Scrape, display, sort and search your Mturk qualifications
// @author Elias041
// @match https://worker.mturk.com/qualifications/assigned*
// @match https://worker.mturk.com/qt
// @require https://code.jquery.com/jquery-3.6.3.js
// @require https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
// @require https://unpkg.com/[email protected]/dist/ag-grid-community.min.js
// @icon https://www.google.com/s2/favicons?sz=64&domain=mturk.com
// @icon data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==
// @grant none
// ==/UserScript==
const buttonStyle = {
color: "#fff",
padding: "10px",
boxShadow: "2px 2px 4px #888888",
opacity: "0.5",
cursor: "pointer"
};
const BUTTON_IDS = {
scrapeButton: "button",
cancelButton: "cancelButton",
progress: "progress",
dbButton: "dbButton"
};
let timeout = 1850;
let counter = " ";
let retry_count = 0;
let error_count = 0;
let scraping = false;
function createButton(parent, id, text, color, clickHandler) {
const btn = document.createElement("div");
Object.assign(btn.style, buttonStyle);
btn.style.background = color;
btn.id = id;
btn.innerHTML = text;
parent.insertBefore(btn, parent.firstChild);
btn.addEventListener("click", clickHandler);
}
const parentDiv = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0]?.parentNode;
if (parentDiv) {
createButton(parentDiv, BUTTON_IDS.scrapeButton, "Scrape Quals", "#33773A", function() {
localStorage.setItem('incompleteScrape', true);
scraping = true;
getAssignedQualifications();
$("#button").css('background', '#383c44');
$("#cancelButton").css('background', '#CE3132');
});
createButton(parentDiv, BUTTON_IDS.cancelButton, "Cancel", "#383c44", function() {
retry_count = 0;
scraping = false;
$("#cancelButton").css('background', '#383c44');
$("#button").css('background', '#33773A');
$("#progress").html("-");
});
createButton(parentDiv, BUTTON_IDS.dbButton, "Database", "#323552", function() {
window.open("https://worker.mturk.com/qt", "_blank");
});
createButton(parentDiv, BUTTON_IDS.progress, "-", "#323552", function() {
});
function initializeDatabase() {
const dbName = "qualifications_v2";
const storeName = "quals";
const version = 2;
const openRequest = indexedDB.open(dbName, version);
openRequest.onupgradeneeded = function(event) {
const db = event.target.result;
if (!db.objectStoreNames.contains(storeName)) {
const objectStore = db.createObjectStore(storeName, {
keyPath: "id"
});
objectStore.createIndex("id", "id", {
unique: true
});
objectStore.createIndex("requester", "requester", {
unique: false
});
objectStore.createIndex("description", "description", {
unique: false
});
objectStore.createIndex("score", "score", {
unique: false
});
objectStore.createIndex("date", "date", {
unique: false
});
objectStore.createIndex("qualName", "qualName", {
unique: false
});
objectStore.createIndex("reqURL", "reqURL", {
unique: false
});
objectStore.createIndex("reqQURL", "reqQURL", {
unique: false
});
objectStore.createIndex("retURL", "retURL", {
unique: false
});
objectStore.createIndex("canRetake", "canRetake", {
unique: false
});
objectStore.createIndex("hasTest", "hasTest", {
unique: false
});
objectStore.createIndex("canRequest", "canRequest", {
unique: false
});
objectStore.createIndex("isSystem", "isSystem", {
unique: false
});
}
};
}
initializeDatabase()
}
function openDatabase() {
return new Promise((resolve, reject) => {
const dbName = "qualifications_v2";
const openRequest = indexedDB.open(dbName);
openRequest.onsuccess = (event) => {
resolve(event.target.result);
};
openRequest.onerror = (event) => {
reject(event.target.errorCode);
};
});
}
function readDatabase() {
return openDatabase().then((db) => {
return new Promise((resolve, reject) => {
const storeName = "quals";
const transaction = db.transaction(storeName, "readonly");
const objectStore = transaction.objectStore(storeName);
const request = objectStore.getAll();
request.onsuccess = (event) => {
resolve(event.target.result);
};
request.onerror = (event) => {
reject(event.target.errorCode);
};
});
});
}
async function compareDatabases(oldDBPromise) {
const newDB = await readDatabase()
return oldDBPromise.then(oldDB => {
let changes = [];
for (let i = 0; i < newDB.length; i++) {
let newRecord = newDB[i];
let oldRecord = oldDB.find(r => r.id === newRecord.id);
if (oldRecord && oldRecord.score !== newRecord.score) {
changes.push({
id: newRecord.id,
field: "score",
requester: newRecord.requester,
qualName: newRecord.qualName,
oldValue: oldRecord.score,
newValue: newRecord.score
});
}
}
if (changes.length > 0) {
localStorage.setItem("changes", JSON.stringify(changes));
localStorage.setItem("hasChanges", true);
return changes;
}
})
}
function checkFirstRun() {
openDatabase()
.then((db) => {
const storeName = "quals";
const transaction = db.transaction(storeName, "readonly");
const objectStore = transaction.objectStore(storeName);
const request = objectStore.count();
request.onsuccess = (event) => {
const count = event.target.result;
if (count === 0) {
localStorage.setItem("firstRun", true);
} else {
localStorage.setItem("firstRun", false);
}
};
request.onerror = (event) => {
console.error("Error counting records:", event.target.errorCode);
};
})
.catch((error) => {
console.error("Error opening database:", error);
});
}
function addEntries(assigned_qualifications) {
const dbName = "qualifications_v2";
const storeName = "quals";
const openRequest = indexedDB.open(dbName);
openRequest.onsuccess = function(event) {
const db = event.target.result;
const transaction = db.transaction(storeName, "readwrite");
const objectStore = transaction.objectStore(storeName);
assigned_qualifications.forEach(function(t) {
const entry = {
id: t.request_qualification_url,
requester: t.creator_name,
description: t.description,
canRetake: t.can_retake_test_or_rerequest,
retry: t.earliest_retriable_time,
score: t.value,
date: t.grant_time,
qualName: t.name,
reqURL: t.creator_url,
retURL: t.retake_test_url,
isSystem: t.is_system_qualification,
canRequest: t.is_requestable,
hasTest: t.has_test,
};
objectStore.put(entry);
});
transaction.oncomplete = function() {
console.log("All entries added successfully");
};
transaction.onerror = function(event) {
console.error("Error adding entries:", event.target.errorCode);
};
};
openRequest.onerror = function(event) {
console.error("Error opening database:", event.target.errorCode);
};
}
checkFirstRun();
let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
let timeoutId;
let oldDBPromise;
let totalRetries = 0;
function getAssignedQualifications(nextPageToken = "") {
if (oldDBPromise === undefined) {
oldDBPromise = readDatabase();
}
if (!scraping) {
return;
}
$("#progress").html(counter);
$.getJSON(page)
.then(function(data) {
counter++
retry_count = 0
addEntries(data.assigned_qualifications);
if (data.next_page_token !== null) {
timeoutId = setTimeout(() => {
page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
getAssignedQualifications(data.next_page_token);
}, timeout);
} else if (data.next_page_token === null) {
console.log("Scraping completed");
console.log(counter + " pages");
console.log(totalRetries + " timeouts");
console.log("Clock was " + timeout);
if (localStorage.getItem("firstRun") === "false") {
compareDatabases(oldDBPromise)
}
localStorage.setItem('incompleteScrape', false);
$("#cancelButton").css('background', '#383c44');
$("#progress").css('background', '#25dc12');
$("#progress").html('✓');
$("#dbButton").css('background', '#57ab4f');
} else {
console.log("Timeout or abort. Clock was " + timeout);
$("#progress").css('background', '#FF0000');
$("#progress").html('X');
return;
}
})
.catch(function(error) {
if (error.status === 429 && retry_count < 20) {
retry_count++
totalRetries++
setTimeout(() => {
getAssignedQualifications(nextPageToken);
}, 3000);
} else if (error.status === 429 && retry_count > 20) {
console.log("error " + error_count)
error_count++;
timeout += 1000
setTimeout(() => {
getAssignedQualifications(nextPageToken);
}, 10000);
} else if (error.status === 429 && retry_count > 20 && error_count > 3) {
alert("There was a problem accessing the Mturk website. Scraping halted.")
scraping = false
return;
} else if (error.status === 503) {
$("#progress").css('background', '#FFFF00');
$("#progress").html('!');
if (confirm("Mturk responded with 503: Service Unavailable. Retry?")) {
$("#progress").css('background', '#33773A');
setTimeout(() => {
getAssignedQualifications(nextPageToken);
}, 10000);
} else {
$("#progress").css('background', '#FF0000');
$("#progress").html('X');
console.log("User declined retry.");
return;
}
}
})
}
if (location.href === "https://worker.mturk.com/qt") {
document.body.innerHTML = "";
let gridDiv = document.createElement("div");
gridDiv.setAttribute("id", "gridDiv");
document.body.appendChild(gridDiv);
document.title = "Qualifications";
window.closeModal = function() {
document.getElementById("changesModal").style.display = "none";
localStorage.setItem("hasChanges", false);
}
window.closeIModal = function() {
document.getElementById("incompleteModal").style.display = "none";
}
function getDataFromDatabase() {
return new Promise((resolve, reject) => {
const dbName = "qualifications_v2";
const storeName = "quals";
const openRequest = indexedDB.open(dbName);
openRequest.onsuccess = function(event) {
const db = event.target.result;
const transaction = db.transaction(storeName, "readonly");
const objectStore = transaction.objectStore(storeName);
const request = objectStore.getAll();
request.onsuccess = function() {
resolve(request.result);
};
request.onerror = function() {
reject(new Error("Error retrieving data from the database"));
};
};
openRequest.onerror = function(event) {
reject(new Error("Error opening the database"));
};
});
}
function displayChangeDetails() {
if (localStorage.getItem("firstRun") === "true") {
document.getElementById("changesModal").style.display = "none";
localStorage.setItem("hasChanges", false);
return;
}
if (localStorage.getItem("hasChanges") === "true") {
let storedData = localStorage.getItem("changes");
if (storedData) {
let changeDetails = JSON.parse(storedData);
let changesList = document.getElementById("changesList");
changeDetails.forEach(function(detail) {
let changeText = detail.requester + " - " + detail.qualName + " - " + detail.field + ": " + detail.oldValue + " -> " + detail.newValue;
let changeItem = document.createElement("div");
changeItem.textContent = changeText;
changesList.appendChild(changeItem);
});
document.getElementById("changesModal").style.display = "block";
}
}
}
function incompleteScrapeNotification() {
if (localStorage.getItem("incompleteScrape") === "true") {
document.getElementById("incompleteModal").style.display = "block";
}
}
gridDiv.innerHTML = `
<div id="myGrid" class="ag-theme-alpine">
<style>
.ag-theme-alpine {
--ag-grid-size: 3px;
width: 100%;
height: 100%;
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
.modal {
display: none;
position: fixed;
z-index: 1;
left: 0;
top: 0;
width: 100%;
height: 100%;
overflow: auto;
background-color: rgba(0, 0, 0, 0.4);
}
.modal-content {
background-color: #fefefe;
margin: auto;
margin-top: 10%;
padding: 20px;
border: 1px solid #888;
width: 80%;
max-width: 600px;
}
.modal-footer {
padding: 10px;
text-align: right;
}
.modal-close {
background-color: #4CAF50;
border: none;
color: white;
padding: 8px 16px;
text-align: center;
text-decoration: none;
display: inline-block;
font-size: 16px;
margin: 4px 2px;
cursor: pointer;
}
@media screen and (min-height: 600px) {
.modal-content {
margin-top: 15%;
}}}
</style>
<div id="changesModal" class="modal">
<div class="modal-content">
<h4>Changes Detected</h4>
<p id="changesList"></p>
</div>
<div class="modal-footer">
<button class="modal-close" ">Close</button>
</div>
</div>
</div>
<div id="incompleteModal" class="modal">
<div class="modal-content">
<h4>Incomplete Scrape Detected</h4>
<p>A scrape is in progress or the last scrape was incomplete.</p>
</div>
<div class="modal-footer">
<button class="modal-close" ">Close</button>
</div>
</div>
</div>
`
const gridOptions = {
columnDefs: [{
headerName: 'Mturk Qualification Database and Scraper',
children: [{
field: "qualName",
comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
}
},
{
headerName: "Requester",
field: "requester",
comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
}
}
]
},
{
headerName: ' ',
children: [{
field: "description",
width: 350,
cellRenderer: function(params) {
return '<span title="' + params.value + '">' + params.value + '</span>';
},
comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
}
},
{
headerName: "Value",
field: "score",
width: 100
},
{
headerName: "Date",
field: "date",
width: 100,
valueGetter: function(params) {
var date = new Date(params.data.date);
return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
},
comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
var dateA = new Date(valueA);
var dateB = new Date(valueB);
return dateA - dateB;
},
},
{
headerName: "Requester ID",
width: 150,
field: "reqURL",
valueFormatter: function(params) {
var parts = params.value.split("/");
return parts[2];
},
},
{
headerName: "Qual ID",
field: "id",
valueFormatter: function(params) {
if (!params.value || params.value === '') return '';
var parts = params.value.split("/");
return parts[2];
}
}
]
},
{
headerName: 'More',
children: [{
headerName: " ",
field: " ",
width: 100,
columnGroupShow: 'closed'
},
{
headerName: "Retake",
field: "canRetake",
width: 100,
columnGroupShow: 'open',
suppressMenu: true
},
{
headerName: "hasTest",
field: "hasTest",
width: 100,
columnGroupShow: 'open',
suppressMenu: true
},
{
headerName: "canReq",
field: "canRequest",
width: 100,
columnGroupShow: 'open',
suppressMenu: true
},
{
headerName: "System",
field: "isSystem",
width: 100,
columnGroupShow: 'open',
suppressMenu: true
},
]
}
],
defaultColDef: {
sortable: true,
filter: true,
editable: true,
resizable: true,
},
rowSelection: 'multiple',
animateRows: true,
rowData: []
};
const closeModalButtons = gridDiv.querySelectorAll(".modal-close");
closeModalButtons.forEach((button) => {
button.addEventListener("click", function() {
const modal = button.closest(".modal");
modal.style.display = "none";
if (modal.id === "changesModal") {
localStorage.setItem("hasChanges", false);
}
});
});
function addCSS(url, callback) {
const link = document.createElement('link');
link.rel = 'stylesheet';
link.href = url;
link.onload = callback;
document.head.appendChild(link);
}
addCSS('https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css', function() {
addCSS('https://cdn.jsdelivr.net/npm/[email protected]/styles/ag-theme-alpine.css', function() {
initializeAgGrid();
});
});
async function initializeAgGrid() {
window.addEventListener("load", function() {
displayChangeDetails();
incompleteScrapeNotification();
const gridDiv = document.querySelector("#myGrid");
getDataFromDatabase()
.then((data) => {
var filteredData = data.filter(function(row) {
return !row.qualName.includes("Exc: [");
});
gridOptions.rowData = filteredData;
new agGrid.Grid(gridDiv, gridOptions);
})
.catch((error) => {
console.error("Error loading data for ag-grid:", error);
});
});
};
}
