Removes duplicate posts from feeds and pages by hashing images and comparing URLs. Uses dHash compared in BK-Trees. Fast & Lightweight. Includes Control Panel.
// ==UserScript==
// @name Reddit - AntiDuplicate Content
// @namespace https://github.com/BD9Max/userscripts
// @version 2.8.0
// @description Removes duplicate posts from feeds and pages by hashing images and comparing URLs. Uses dHash compared in BK-Trees. Fast & Lightweight. Includes Control Panel.
// @icon https://raw.githubusercontent.com/BD9Max/userscripts/33ebed2e9a48b78f1324d8c1d4bf5d0d37b6489b/media/icons/Reddit%20AntiDup%20Icon%2064.png
// @author krbd9max
// @match https://www.reddit.com/*
// @grant none
// @run-at document-end
// @license MIT
// ==/UserScript==
(function() {
'use strict';
// Configuration
const HAMMING_THRESHOLD = 3; // Bit difference tolerance for image similarities
// --- SETTINGS MANAGEMENT (LocalStorage ensures persistence across script updates) ---
const defaultSettings = {
enableSubreddits: true,
enableProfiles: true,
excludedSubreddits: "",
excludedProfiles: ""
};
function getSettings() {
try {
const saved = localStorage.getItem('redditAntiDupSettings');
return saved ? { ...defaultSettings, ...JSON.parse(saved) } : defaultSettings;
} catch (e) {
return defaultSettings;
}
}
function saveSettings(settings) {
localStorage.setItem('redditAntiDupSettings', JSON.stringify(settings));
}
// --- PAGE EXCLUSION LOGIC ---
function isPageAllowed() {
const path = window.location.pathname.toLowerCase();
// 1. Hardcoded Exclusions
if (path === '/' || path === '/news/' || path === '/r/popular/' || path === '/news' || path === '/r/popular') {
return false;
}
const settings = getSettings();
// 2. Subreddit Settings
if (path.startsWith('/r/')) {
if (!settings.enableSubreddits) return false;
const subMatch = path.match(/^\/r\/([^\/]+)/);
if (subMatch) {
const currentSub = subMatch[1].toLowerCase();
const excludedSubs = settings.excludedSubreddits.split(',').map(s => s.trim().toLowerCase()).filter(s => s);
if (excludedSubs.includes(currentSub)) return false;
}
}
// 3. Profile Settings
if (path.startsWith('/user/')) {
if (!settings.enableProfiles) return false;
const userMatch = path.match(/^\/user\/([^\/]+)/);
if (userMatch) {
const currentUser = userMatch[1].toLowerCase();
const excludedUsers = settings.excludedProfiles.split(',').map(s => s.trim().toLowerCase()).filter(s => s);
if (excludedUsers.includes(currentUser)) return false;
}
}
return true;
}
// --- UI: CONTROL PANEL ---
function injectControlPanel() {
if (document.getElementById('antidup-control-wrapper')) return;
const wrapper = document.createElement('div');
wrapper.id = 'antidup-control-wrapper';
wrapper.style.cssText = `
position: fixed;
top: 50%;
right: 15px;
transform: translateY(-50%);
z-index: 999999;
display: flex;
align-items: center;
font-family: Arial, sans-serif;
`;
const panel = document.createElement('div');
panel.id = 'antidup-panel';
panel.style.cssText = `
display: none;
background-color: #1a1a1b;
border: 1px solid #343536;
border-radius: 8px;
padding: 15px;
margin-right: 15px;
width: 250px;
color: #d7dadc;
box-shadow: 0 4px 12px rgba(0,0,0,0.5);
`;
const iconUrl = 'https://raw.githubusercontent.com/BD9Max/userscripts/33ebed2e9a48b78f1324d8c1d4bf5d0d37b6489b/media/icons/Reddit%20AntiDup%20Icon%2064.png';
const toggleBtn = document.createElement('div');
toggleBtn.style.cssText = `
width: 45px;
height: 45px;
border-radius: 50%;
background-image: url('${iconUrl}');
background-size: cover;
background-position: center;
cursor: pointer;
box-shadow: 0 2px 8px rgba(0,0,0,0.4);
border: 2px solid #343536;
transition: transform 0.2s;
`;
toggleBtn.onmouseover = () => toggleBtn.style.transform = 'scale(1.1)';
toggleBtn.onmouseout = () => toggleBtn.style.transform = 'scale(1)';
toggleBtn.onclick = () => {
panel.style.display = panel.style.display === 'none' ? 'block' : 'none';
};
const settings = getSettings();
panel.innerHTML = `
<h3 style="margin: 0 0 12px 0; font-size: 16px; border-bottom: 1px solid #343536; padding-bottom: 8px; text-align: center;">Reddit AntiDup</h3>
<div style="margin-bottom: 10px;">
<label style="display: flex; align-items: center; font-size: 13px; cursor: pointer;">
<input type="checkbox" id="ad-enable-subs" ${settings.enableSubreddits ? 'checked' : ''} style="margin-right: 8px;">
Enable in all /r/ Subreddits
</label>
</div>
<div style="margin-bottom: 10px;">
<label style="font-size: 12px; color: #818384;">Exclude Subs (comma separated):</label>
<input type="text" id="ad-exclude-subs" value="${settings.excludedSubreddits}" placeholder="gaming, aww" style="width: 100%; box-sizing: border-box; background: #272729; border: 1px solid #343536; color: white; padding: 4px; border-radius: 4px; margin-top: 4px;">
</div>
<div style="margin-bottom: 10px; margin-top: 15px;">
<label style="display: flex; align-items: center; font-size: 13px; cursor: pointer;">
<input type="checkbox" id="ad-enable-users" ${settings.enableProfiles ? 'checked' : ''} style="margin-right: 8px;">
Enable in all /user/ Profiles
</label>
</div>
<div style="margin-bottom: 15px;">
<label style="font-size: 12px; color: #818384;">Exclude Profiles (comma separated):</label>
<input type="text" id="ad-exclude-users" value="${settings.excludedProfiles}" placeholder="gallowboob" style="width: 100%; box-sizing: border-box; background: #272729; border: 1px solid #343536; color: white; padding: 4px; border-radius: 4px; margin-top: 4px;">
</div>
<div style="margin-bottom: 15px;">
<label style="font-size: 12px; color: #818384;">Excluded overrides All</label>
</div>
<button id="ad-save-btn" style="width: 100%; padding: 6px; background-color: #d7dadc; color: #1a1a1b; border: none; border-radius: 4px; font-weight: bold; cursor: pointer;">Save Settings</button>
<div id="ad-save-msg" style="display:none; color: #46d160; font-size: 11px; text-align: center; margin-top: 5px;">Saved! Refreshing logic...</div>
`;
wrapper.appendChild(panel);
wrapper.appendChild(toggleBtn);
document.body.appendChild(wrapper);
document.getElementById('ad-save-btn').onclick = () => {
const newSettings = {
enableSubreddits: document.getElementById('ad-enable-subs').checked,
enableProfiles: document.getElementById('ad-enable-users').checked,
excludedSubreddits: document.getElementById('ad-exclude-subs').value,
excludedProfiles: document.getElementById('ad-exclude-users').value
};
saveSettings(newSettings);
const msg = document.getElementById('ad-save-msg');
msg.style.display = 'block';
setTimeout(() => { msg.style.display = 'none'; }, 2000);
// Re-run immediately on new settings
processPosts();
};
}
// --- BK-TREE DATA STRUCTURE FOR FAST HAMMING DISTANCE LOOKUPS ---
class BKNode {
constructor(hash, postId) {
this.hash = hash;
this.postIds = [postId];
this.children = {}; // distance -> BKNode
}
}
class BKTree {
constructor() {
this.root = null;
}
// Compute Hamming Distance between two 64-bit BigInt hashes
static hammingDistance(h1, h2) {
let xor = h1 ^ h2;
let count = 0;
while (xor > 0n) {
if (xor & 1n) count++;
xor >>= 1n;
}
return count;
}
add(hash, postId) {
if (!this.root) {
this.root = new BKNode(hash, postId);
return null;
}
let curr = this.root;
while (true) {
const dist = BKTree.hammingDistance(curr.hash, hash);
if (dist === 0) {
curr.postIds.push(postId);
return curr.postIds[0];
}
if (curr.children[dist]) {
curr = curr.children[dist];
} else {
curr.children[dist] = new BKNode(hash, postId);
return null;
}
}
}
search(hash, maxDist, node = this.root, results = []) {
if (!node) return results;
const dist = BKTree.hammingDistance(node.hash, hash);
if (dist <= maxDist) {
results.push({ node, dist });
}
const minDist = dist - maxDist;
const highDist = dist + maxDist;
for (let d in node.children) {
const numericD = parseInt(d, 10);
if (numericD >= minDist && numericD <= highDist) {
this.search(hash, maxDist, node.children[numericD], results);
}
}
return results;
}
}
// --- DHASH IMAGE HASHING ALGORITHM ---
function computeDHash(imgSrc) {
return new Promise((resolve) => {
const img = new Image();
img.crossOrigin = 'anonymous';
img.src = imgSrc;
img.onload = () => {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
// Resize to 9x8 for dHash
canvas.width = 9;
canvas.height = 8;
ctx.drawImage(img, 0, 0, 9, 8);
let imgData;
try {
imgData = ctx.getImageData(0, 0, 9, 8).data;
} catch (e) {
resolve(null); // Bypass structural taint exceptions safely
return;
}
// Convert to Grayscale
const grayData = new Float32Array(9 * 8);
for (let i = 0; i < 72; i++) {
const r = imgData[i * 4];
const g = imgData[i * 4 + 1];
const b = imgData[i * 4 + 2];
grayData[i] = 0.299 * r + 0.587 * g + 0.114 * b; // Luminance
}
let hash = 0n;
let bitIndex = 0n;
// Compare adjacent pixels
for (let y = 0; y < 8; y++) {
for (let x = 0; x < 8; x++) {
const leftPixel = grayData[y * 9 + x];
const rightPixel = grayData[y * 9 + x + 1];
if (leftPixel > rightPixel) {
hash |= (1n << bitIndex);
}
bitIndex++;
}
}
resolve(hash);
};
img.onerror = () => resolve(null);
});
}
// --- TRACKING & ANTIDUPLICATION DATA LABELS ---
const seenUrls = new Map();
const imageTree = new BKTree();
const processedPostIds = new Set();
const antidupCounts = new Map(); // tracks running totals for targeted elements
function cleanUrl(urlStr) {
try {
const url = new URL(urlStr, window.location.origin);
return url.origin + url.pathname.replace(/\/$/, "");
} catch (e) {
return urlStr;
}
}
function getPostDetails(post) {
const id = post.getAttribute('id') || post.getAttribute('post-id') || post.getAttribute('permalink') || post.id;
let titleText = post.getAttribute('post-title');
let titleEl = post.querySelector('a[id^="post-title-"]') || post.querySelector('[data-adclicklocation="title"]') || post.querySelector('h3');
if (!titleText && titleEl) titleText = titleEl.innerText;
let url = post.getAttribute('content-href') || post.getAttribute('permalink');
if (!url && titleEl) url = titleEl.href;
let imgEl = null;
const imgs = post.querySelectorAll('img');
for (let img of imgs) {
const src = img.src || '';
if (src.includes('/avatar/') || src.includes('user_icon') || img.closest('faceplate-tracker[source="post_credit_bar"]')) {
continue;
}
if (src.includes('preview.redd.it') || src.includes('i.redd.it') || src.includes('external-preview') || img.getAttribute('slot') === 'thumbnail' || img.width > 100) {
imgEl = img;
break;
}
}
return { id, titleText, titleEl, url, imgEl };
}
function addNotice(titleEl, count) {
let notice = titleEl.parentElement.querySelector('.antidup-notice');
if (!notice) {
notice = document.createElement('span');
notice.className = 'antidup-notice';
notice.style.cssText = `
margin-left: 8px;
padding: 2px 5px;
background-color: #305050;
color: #ffffff;
border-radius: 3px;
font-size: 10px;
font-weight: bold;
display: inline-block;
vertical-align: middle;
`;
titleEl.after(notice);
}
notice.innerText = `[AntiDuplicated: ${count} item(s)]`;
}
function incrementDuplicateCount(originalPostId) {
const total = (antidupCounts.get(originalPostId) || 0) + 1;
antidupCounts.set(originalPostId, total);
const targets = document.querySelectorAll('shreddit-post, article, [data-testid="post-container"], .Post');
const originalPost = Array.from(targets).find(p => getPostDetails(p).id === originalPostId);
if (originalPost) {
const details = getPostDetails(originalPost);
if (details.titleEl) {
addNotice(details.titleEl, total);
}
}
}
function processPosts() {
// Stop execution if current page is excluded by rules or user settings
if (!isPageAllowed()) return;
const posts = document.querySelectorAll('shreddit-post, article, [data-testid="post-container"], .Post');
posts.forEach(post => {
const details = getPostDetails(post);
if (!details.id || processedPostIds.has(details.id)) return;
let isDuplicate = false;
// 1. Exact Outbound or Thread URL Antiduplication
if (details.url) {
const cleanedUrl = cleanUrl(details.url);
if (seenUrls.has(cleanedUrl)) {
const originalPostId = seenUrls.get(cleanedUrl);
incrementDuplicateCount(originalPostId);
post.style.setProperty('display', 'none', 'important');
isDuplicate = true;
} else {
seenUrls.set(cleanedUrl, details.id);
}
}
// 2. dHash Image Hashing
if (!isDuplicate && details.imgEl && details.imgEl.src) {
processedPostIds.add(details.id);
computeDHash(details.imgEl.src).then(hash => {
if (hash === null) return;
const matches = imageTree.search(hash, HAMMING_THRESHOLD);
if (matches.length > 0) {
const originalPostId = matches[0].node.postIds[0];
incrementDuplicateCount(originalPostId);
post.style.setProperty('display', 'none', 'important');
} else {
imageTree.add(hash, details.id);
}
});
return;
}
processedPostIds.add(details.id);
});
}
// Initialize UI and Observation
if (document.readyState === "complete" || document.readyState === "interactive") {
injectControlPanel();
} else {
window.addEventListener("DOMContentLoaded", injectControlPanel);
}
const observer = new MutationObserver(() => {
// The observer runs constantly as you scroll, so it checks isPageAllowed() inside processPosts()
// allowing it to gracefully handle SPA routing without page reloads.
processPosts();
});
observer.observe(document.body, { childList: true, subtree: true });
processPosts();
})();