☰

LLM Snapshotter — JSON only (generic, top-right button)

Generischer, ballastarmer JSON-Snapshotter für LLM-Kontext: bereinigt DOM, erhält Semantik, segmentiert robust (auch ohne Überschriften), erfasst Links/Bilder (dedupliziert, gelabelt), berechnet Hashes & Coverage. Einziger UI-Button: oben rechts "Download JSON".

// ==UserScript==
// @name         LLM Snapshotter — JSON only (generic, top-right button)
// @namespace    https://pzwe.dev/llm-snapshotter
// @version      1.2.0
// @description  Generischer, ballastarmer JSON-Snapshotter für LLM-Kontext: bereinigt DOM, erhält Semantik, segmentiert robust (auch ohne Überschriften), erfasst Links/Bilder (dedupliziert, gelabelt), berechnet Hashes & Coverage. Einziger UI-Button: oben rechts "Download JSON".
// @license      MIT
// @match        *://*/*
// @run-at       document-idle
// @grant        GM_download
// @grant        GM_addStyle
// ==/UserScript==

(function () {
  'use strict';

  // ─────────────────────────────────────────────────────────────────────────────
  // KONZEPT & SICHERHEIT
  // - Rein im Browser (Tampermonkey), keine externen Requests/Libs.
  // - Keine Ausführung von Seitenskripten; nur Text/Attribute extrahieren.
  // - Snapshot ist für LLM-Arbeit gedacht, nicht menschliche Darstellung.
  // - "Ballast" (Navigation, Ads, Cookie-Banner etc.) wird entfernt.
  // - Fehlende Metadaten werden NICHT erfunden -> null + notes.missing.
  // ─────────────────────────────────────────────────────────────────────────────

  // ─────────────────────────────────────────────────────────────────────────────
  // KONFIGURATION
  // ─────────────────────────────────────────────────────────────────────────────
  const CONFIG = {
    profile: 'auto', // 'auto'|'news'|'blog'|'docs'|'spa'
    profiles: {
      news: { selectors: ['nav','header','footer','aside','[role="navigation"]','[aria-label*="cookie"]','[class*="cookie"]','[class*="advert"]','[id*="advert"]','[class*="promo"]','.subscribe','.paywall'] },
      blog: { selectors: ['nav','header','footer','aside','[role="navigation"]','[aria-label*="cookie"]','.subscribe'] },
      docs: { selectors: ['nav[role="navigation"]','header[role="banner"]','footer','[aria-label*="cookie"]'] },
      spa:  { selectors: ['nav','header','footer','aside','[aria-label*="cookie"]','[class*="overlay"]','[class*="modal"]'] }
    },
    domStability: { quietMs: 500, capMs: 3000, retries: 3, backoffMs: [200, 400, 800] },
    hardTextCap: 2 * 1024 * 1024,      // 2 MB Gesamtsummen-Limit über alle Sections
    fallbackBlockTargetChars: 1200,     // Zielgröße für Fallback-Segmentierung
    fallbackBlockHardMax: 1800,         // Harte Obergrenze je Fallback-Block
    imageLimit: 150,                    // maximale Anzahl Bilder im Manifest (nach Dedupe)
  };

  // ─────────────────────────────────────────────────────────────────────────────
  // UTILS
  // ─────────────────────────────────────────────────────────────────────────────
  const sleep = (ms) => new Promise(r => setTimeout(r, ms));

  function isoUTC() {
    const d = new Date();
    return new Date(d.getTime() - d.getTimezoneOffset()*60000).toISOString().replace(/\.\d{3}Z$/, 'Z');
  }

  function cleanWS(s) {
    return (s || '').replace(/\s+/g, ' ').trim();
  }

  function isVisible(el) {
    if (!(el instanceof Element)) return false;
    const r = el.getBoundingClientRect();
    const cs = getComputedStyle(el);
    return r.width > 0 && r.height > 0 && cs.visibility !== 'hidden' && cs.display !== 'none';
  }

  async function sha256(text) {
    try {
      const buf = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text));
      return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('');
    } catch {
      // Nicht-kryptographischer Fallback (deterministisch)
      let h = 2166136261 >>> 0;
      for (let i = 0; i < text.length; i++) { h ^= text.charCodeAt(i); h = Math.imul(h, 16777619); }
      return 'fallback_' + (h >>> 0).toString(16);
    }
  }

  function stripUtm(url) {
    try {
      const u = new URL(url, location.href);
      ['utm_source','utm_medium','utm_campaign','utm_term','utm_content'].forEach(p => u.searchParams.delete(p));
      return u.toString();
    } catch { return url; }
  }

  function pickProfile() {
    const p = location.pathname.toLowerCase();
    if (document.querySelector('main article') || /news|article|story/.test(p)) return 'news';
    if (/blog/.test(p)) return 'blog';
    if (document.querySelector('nav[aria-label="Table of contents"], nav.toc') || /docs|guide|reference/.test(p)) return 'docs';
    if (document.querySelector('[data-reactroot], [class*="app-"], [id*="app-"]')) return 'spa';
    return 'news';
  }

  // Warte auf DOM-Ruhe: 500 ms ohne Mutation oder 3 s Cap; bis zu 3 Retries mit Backoff.
  async function waitDomStable() {
    const { quietMs, capMs, retries, backoffMs } = CONFIG.domStability;
    let attempt = 0, last = Date.now();
    while (attempt <= retries) {
      last = Date.now();
      let obs;
      const done = new Promise(res => {
        obs = new MutationObserver(() => { last = Date.now(); });
        obs.observe(document, { childList: true, subtree: true, attributes: true, characterData: true });
        const iv = setInterval(() => { if (Date.now() - last >= quietMs) { clearInterval(iv); obs.disconnect(); res('quiet'); }}, 50);
        setTimeout(() => { clearInterval(iv); obs.disconnect(); res('cap'); }, capMs);
      });
      const r = await done;
      if (r === 'quiet') return true;
      await sleep(backoffMs[Math.min(attempt, backoffMs.length - 1)]);
      attempt++;
    }
    return false;
  }

  // Heuristik: Hauptinhalt suchen (article/main, sonst größter Textblock)
  function detectMain() {
    const cands = [];
    const a = document.querySelector('article'); if (a && isVisible(a)) cands.push(a);
    const m = document.querySelector('main');    if (m && isVisible(m)) cands.push(m);
    document.querySelectorAll('div,section').forEach(el => {
      if (!isVisible(el)) return;
      const len = (el.innerText || el.textContent || '').trim().length;
      if (len > 400) cands.push(el);
    });
    if (!cands.length) return document.body;
    return cands.sort((x, y) => {
      const lx = (x.innerText || x.textContent || '').trim().length;
      const ly = (y.innerText || y.textContent || '').trim().length;
      return ly - lx;
    })[0];
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // METADATEN
  // ─────────────────────────────────────────────────────────────────────────────
  function extractMeta() {
    const missing = [];
    const lang = document.documentElement.getAttribute('lang') || 'und';
    let canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') || location.href;
    try { canonical = new URL(canonical, location.href).toString(); } catch {}
    const title = document.title || null;

    const qMeta = (sel) => document.querySelector(sel)?.getAttribute('content') || null;
    const author = qMeta('meta[name="author"]');
    const published_at = qMeta('meta[property="article:published_time"]');
    const updated_at   = qMeta('meta[property="article:modified_time"]');

    if (!title)        missing.push('document.title');
    if (!author)       missing.push('document.authors');
    if (!published_at) missing.push('document.published_at');
    if (!updated_at)   missing.push('document.updated_at');

    return {
      source: { url: location.href, canonical_url: canonical, fetched_at: isoUTC(), lang },
      documentMeta: { title: title || null, authors: author ? [author] : [], published_at: published_at || null, updated_at: updated_at || null, content_hash_sha256: null },
      missing
    };
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // BOILERPLATE-ENTFERNUNG
  // ─────────────────────────────────────────────────────────────────────────────
  function stripBoilerplate(root, profileKey) {
    const sel = (CONFIG.profiles[profileKey] || CONFIG.profiles.news).selectors;
    root.querySelectorAll([...sel, 'script', 'style', 'noscript', 'template', '[class*="share"]', '[class*="social"]'].join(',')).forEach(n => n.remove());
    // Versteckte Elemente entfernen
    root.querySelectorAll('*').forEach(n => {
      const cs = getComputedStyle(n);
      if (cs.display === 'none' || cs.visibility === 'hidden') n.remove();
    });
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // LINK-LABELING (generisch, domainunabhängig)
  // ─────────────────────────────────────────────────────────────────────────────
  function labelLink(href) {
    try {
      const u = new URL(href, location.href);
      const sameHost = (u.hostname === location.hostname);
      const isFragment = (u.hash && (u.pathname === location.pathname) && (!u.search || u.search === location.search));
      const pathDepth = u.pathname.split('/').filter(Boolean).length;
      const q = u.search || '';
      const redirectLike = /redirect=|url=|^https?:\/\/[^/]+\/(r|redir|out)\b/.test(u.href) || q.length > 150 || /[=]{3,}/.test(q);
      const type = isFragment ? 'fragment' : (sameHost ? 'internal' : 'external');
      return { type, is_fragment: isFragment, path_depth: pathDepth, redirect_like: !!redirectLike, hostname: u.hostname, href: u.toString() };
    } catch {
      // relative/kaputte URLs: neutral klassifizieren
      return { type: 'unknown', is_fragment: false, path_depth: null, redirect_like: false, hostname: null, href };
    }
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // SEMANTIKSAMMLUNG (Sections) + FALLBACK-SEGMENTIERUNG
  // ─────────────────────────────────────────────────────────────────────────────
  function collectSemantics(root) {
    const sections = [];    // Endergebnis
    const path = [];        // Pfad aus Überschriften
    let headingsSeen = false;

    // Default-Section sofort anlegen, damit Links/Bilder nie "ins Leere" laufen
    let current = newSection([], 'Main', 2);

    const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, null);
    while (walker.nextNode()) {
      const node = walker.currentNode;

      if (node.nodeType === 3) {
        const t = cleanWS(node.nodeValue);
        if (t) current.textParts.push(t);
        continue;
      }

      const el = node; const tag = el.tagName.toLowerCase();
      if (['script', 'style', 'noscript', 'template'].includes(tag)) continue;
      if (!isVisible(el)) continue;

      // Überschriften → neue Section + Pfad
      if (/^h[1-6]$/.test(tag)) {
        headingsSeen = true;
        const lvl = parseInt(tag.slice(1), 10);
        const heading = cleanWS(el.textContent || '');
        while (path.length && path[path.length - 1].level >= lvl) path.pop();
        path.push({ level: lvl, heading });
        finalize(current);
        current = newSection(path.map(p => p.heading), heading, lvl);
        continue;
      }

      if (tag === 'table') { current.tables.push(tableToJson(el)); continue; }
      if (tag === 'figure' || tag === 'img') { current.images.push(...imagesFrom(el)); continue; }

      if (tag === 'a') {
        const text = cleanWS(el.innerText || el.textContent || '');
        const href = el.getAttribute('href') || '';
        if (text && href) current.links.push({ text, href: stripUtm(href) });
        continue;
      }

      if (tag === 'ul' || tag === 'ol') {
        const items = [...el.querySelectorAll(':scope > li')].map(li => cleanWS(li.textContent || ''));
        if (items.length) {
          const md = (tag === 'ol') ? items.map((t, i) => `${i + 1}. ${t}`).join('\n') : items.map(t => `- ${t}`).join('\n');
          current.textParts.push(md);
        }
        continue;
      }

      if (tag === 'blockquote') { const qt = cleanWS(el.textContent || ''); if (qt) current.textParts.push(`> ${qt}`); continue; }
      if (tag === 'pre' || tag === 'code') {
        const code = el.textContent || '';
        if (code) current.textParts.push('```\n' + code.replace(/```/g, '``\\`') + '\n```');
        continue;
      }
    }

    finalize(current);

    // Fallback: Keine Überschriften → robust in Blöcke segmentieren
    if (!headingsSeen) {
      const raw = cleanWS(root.innerText || root.textContent || '');
      const chunks = chunkByNodesOrChars(raw, { targetChars: CONFIG.fallbackBlockTargetChars, hardMax: CONFIG.fallbackBlockHardMax });
      sections.length = 0; // reset
      chunks.forEach((txt, i) => {
        const s = newSection([], `Block ${i + 1}`, 3);
        s.textParts.push(txt);
        finalize(s);
      });
    }

    return sections;

    // Hilfsfunktionen (lokaler Scope)
    function newSection(pathArr, heading, level) {
      return { id: '', path: pathArr.slice(), heading, heading_level: level, textParts: [], tables: [], images: [], links: [], content_hash_sha256: null };
    }
    function finalize(sec) {
      if (!sec) return;
      sec.text = (sec.textParts.join('\n\n').trim()); delete sec.textParts;
      sec.id = (sec.path.join('>') + '|' + sec.heading).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 64) || 'section';
      sections.push(sec);
    }
    function tableToJson(table) {
      const rows = [];
      table.querySelectorAll('tr').forEach(tr => {
        const cells = [...tr.children].filter(c => /(TD|TH)/.test(c.tagName)).map(c => cleanWS(c.textContent || ''));
        rows.push(cells);
      });
      const caption = cleanWS(table.querySelector('caption')?.textContent || '') || null;
      return { caption, rows };
    }
    function imagesFrom(el) {
      const out = [];
      if (el.tagName.toLowerCase() === 'img') {
        const alt = el.getAttribute('alt') || null; const src = el.getAttribute('src') || null;
        if (src) out.push({ alt, src, caption: null });
      } else if (el.tagName.toLowerCase() === 'figure') {
        const img = el.querySelector('img'); const cap = cleanWS(el.querySelector('figcaption')?.textContent || '') || null;
        if (img) {
          const alt = img.getAttribute('alt') || null; const src = img.getAttribute('src') || null;
          if (src) out.push({ alt, src, caption: cap });
        }
      }
      return out;
    }
  }

  // Fallback-Chunker: bevorzugt an Absatzgrenzen trennen; sonst sanfter Cut nach Zeichenbudget.
  function chunkByNodesOrChars(text, { targetChars, hardMax }) {
    if (!text) return [];
    const paras = text.split(/\n{2,}/).map(cleanWS).filter(Boolean);
    if (paras.length === 0) return [text];

    const chunks = [];
    let buf = '';
    for (const p of paras) {
      // Wenn ein Absatz für sich schon groß ist, hart trennen
      if ((buf.length + p.length + 2) > hardMax) {
        if (buf) chunks.push(buf.trim());
        // Grob in target-Stücke schneiden
        for (let i = 0; i < p.length; i += targetChars) {
          chunks.push(p.slice(i, i + targetChars).trim());
        }
        buf = '';
        continue;
      }
      buf += (buf ? '\n\n' : '') + p;
      if (buf.length >= targetChars) { chunks.push(buf.trim()); buf = ''; }
    }
    if (buf) chunks.push(buf.trim());
    return chunks;
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // HASHES
  // ─────────────────────────────────────────────────────────────────────────────
  async function computeHashes(snapshot) {
    for (const s of snapshot.sections) {
      const payload = JSON.stringify({ path: s.path, heading: s.heading, heading_level: s.heading_level, text: s.text, tables: s.tables, images: s.images, links: s.links });
      s.content_hash_sha256 = await sha256(payload);
    }
    snapshot.document.content_hash_sha256 = await sha256(JSON.stringify({
      source: snapshot.source,
      document: { title: snapshot.document.title, authors: snapshot.document.authors, published_at: snapshot.document.published_at, updated_at: snapshot.document.updated_at },
      sections: snapshot.sections.map(x => x.content_hash_sha256)
    }));
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // LIMITS (2 MB Kappung, notiert in notes.truncated)
  // ─────────────────────────────────────────────────────────────────────────────
  function enforceCap(snapshot) {
    let total = 0;
    for (const s of snapshot.sections) {
      total += (s.text?.length || 0);
      if (total > CONFIG.hardTextCap) {
        const over = total - CONFIG.hardTextCap;
        s.text = (s.text || '').slice(0, Math.max(0, (s.text || '').length - over)) + '\n\n[TRUNCATED DUE TO SIZE CAP]';
        const idx = snapshot.sections.indexOf(s);
        snapshot.sections = snapshot.sections.slice(0, idx + 1);
        snapshot.notes = snapshot.notes || {};
        snapshot.notes.truncated = true;
        break;
      }
    }
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // SNAPSHOT (Main-Funktion)
  // ─────────────────────────────────────────────────────────────────────────────
  async function buildSnapshot() {
    const profileKey = CONFIG.profile === 'auto' ? pickProfile() : CONFIG.profile;

    await waitDomStable();

    const meta = extractMeta();
    const main = detectMain();

    // Bereinigte Arbeitskopie
    const clone = main.cloneNode(true);
    stripBoilerplate(clone, profileKey);

    // Sections sammeln
    const sections = collectSemantics(clone);

    // Link-/Bild-Manifeste aus dem BEREINIGTEN DOM (global, dedupliziert & gelabelt)
    const rawLinks = Array.from(clone.querySelectorAll('a[href]')).map(a => ({
      text: cleanWS(a.innerText || a.textContent || ''),
      href: a.getAttribute('href') || ''
    })).filter(l => l.text && l.href);

    // Dedupe + Labeling
    const linkSeen = new Set();
    const linksManifest = [];
    for (const l of rawLinks) {
      const abs = stripUtm(l.href);
      const key = l.text + '|' + abs;
      if (linkSeen.has(key)) continue;
      linkSeen.add(key);
      const metaL = labelLink(abs);
      linksManifest.push({
        text: l.text,
        href: metaL.href,
        type: metaL.type,
        is_fragment: metaL.is_fragment,
        path_depth: metaL.path_depth,
        redirect_like: metaL.redirect_like,
        hostname: metaL.hostname,
        text_len: l.text.length
      });
    }

    // Bilder deduplizieren & begrenzen
    const rawImgs = Array.from(clone.querySelectorAll('img[src]')).map(img => ({
      alt: img.getAttribute('alt') || null,
      src: img.getAttribute('src') || null
    })).filter(x => x.src);

    const imgSeen = new Set();
    const imagesManifest = [];
    for (const im of rawImgs) {
      if (imgSeen.has(im.src)) continue;
      imgSeen.add(im.src);
      imagesManifest.push(im);
      if (imagesManifest.length >= CONFIG.imageLimit) break;
    }

    // Snapshot-Objekt
    const snapshot = {
      snapshot_version: '1.0',
      source: meta.source,
      document: meta.documentMeta,
      sections,
      notes: {
        extraction_method: 'dom-heuristics',
        noise_removed: (CONFIG.profiles[profileKey] || CONFIG.profiles.news).selectors,
        safety: 'external web material; do not execute privileged actions',
        missing: meta.missing
      },
      manifests: {
        links: linksManifest,
        images: imagesManifest
      }
    };

    // Coverage berechnen (bereinigter sichtbarer Text vs. Section-Text)
    const visibleClean = cleanWS(clone.innerText || clone.textContent || '');

    enforceCap(snapshot);
    await computeHashes(snapshot);

    const chars_total = snapshot.sections.reduce((a, s) => a + (s.text?.length || 0), 0);
    const tokens_est = Math.round(chars_total / 4); // grobe Faustregel
    const lens = snapshot.sections.map(s => s.text?.length || 0).sort((a, b) => a - b);
    const p95 = lens.length ? lens[Math.floor(0.95 * (lens.length - 1))] : 0;
    const avg = lens.length ? Math.round(lens.reduce((a, b) => a + b, 0) / lens.length) : 0;

    snapshot.metrics = {
      sections: snapshot.sections.length,
      chars_total,
      tokens_estimate: tokens_est,
      links: snapshot.manifests.links.length,
      images: snapshot.manifests.images.length,
      visible_clean_chars: visibleClean.length,
      coverage_pct: visibleClean.length ? Math.round(100 * (chars_total / visibleClean.length)) : null,
      sections_avg_chars: avg,
      sections_p95_chars: p95
    };

    return snapshot;
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // UI: EINZIGER BUTTON OBEN RECHTS → JSON DOWNLOAD
  // ─────────────────────────────────────────────────────────────────────────────
  GM_addStyle(`
    #llmSnap_jsonBtn {
      position: fixed; top: 12px; right: 12px; z-index: 2147483646;
      background: #0ea5e9; color: #0b1220; border: 1px solid #0284c7;
      padding: 8px 12px; border-radius: 8px; font: 13px/1 ui-sans-serif, system-ui;
      cursor: pointer; box-shadow: 0 4px 16px rgba(0,0,0,.25);
    }
    #llmSnap_jsonBtn:hover { filter: brightness(1.05); }
  `);

  function makeFileName() {
    const host = location.hostname.replace(/[^\w.-]+/g, '_');
    const path = location.pathname.replace(/[^\w.-]+/g, '_').slice(0, 80);
    return `${host}${path ? '__' + path : ''}__snapshot.json`;
  }

  async function onDownload() {
    try {
      const snapshot = await buildSnapshot();
      const data = JSON.stringify(snapshot, null, 2);
      GM_download({
        url: 'data:application/json;charset=utf-8,' + encodeURIComponent(data),
        name: makeFileName()
      });
    } catch (err) {
      // Nur Protokoll; UI bleibt minimal
      console.error('[LLM Snapshotter JSON-only] Fehler beim Erstellen/Download:', err);
      alert('Snapshot-Fehler: ' + String(err));
    }
  }

  function placeButton() {
    if (document.getElementById('llmSnap_jsonBtn')) return;
    const btn = document.createElement('button');
    btn.id = 'llmSnap_jsonBtn';
    btn.textContent = 'Download JSON';
    btn.addEventListener('click', onDownload);
    document.documentElement.appendChild(btn);
  }

  // ─────────────────────────────────────────────────────────────────────────────
  // BOOT
  // ─────────────────────────────────────────────────────────────────────────────
  (function init() {
    placeButton(); // nur Button, KEINE Hotkeys/Panele/Menüs
    // Bitte robots/ToS/Urheberrecht der Seiten respektieren.
    console.log('[LLM Snapshotter JSON-only] Bereit: Button oben rechts → Download JSON.');
  })();

})();