Add self-hosted RCS backend, extension, and ops tooling

Ship the Go/SQLite API and Web UI, Chrome/Brave capture addon, Docker Compose, Pangolin reverse-proxy support, and a user-crontab watchdog so the binary stays running without systemd.
2026-08-06 22:02:45 +02:00
parent 3f05c97e1f
commit 5a0e2e630b
30 changed files with 4334 additions and 2 deletions
@@ -0,0 +1,511 @@
+// DOM scrapers for old.reddit.com and www.reddit.com (new / shreddit design).
+
+(function (global) {
+  function isOldReddit() {
+    if (location.hostname === "old.reddit.com") return true;
+    // www.reddit.com can still serve classic markup (user preference / redirects / extensions).
+    // Prefer classic scrapers whenever those nodes exist — don't rely on hostname alone.
+    return Boolean(
+      document.querySelector(
+        ".commentarea .thing.comment, #siteTable .thing.comment, .thing.comment[data-fullname^='t1_'], body.listing-page .thing.link"
+      )
+    );
+  }
+
+  function isShreddit() {
+    return Boolean(document.querySelector("shreddit-post, shreddit-comment, shreddit-comment-tree, shreddit-app"));
+  }
+
+  function cleanText(value) {
+    return (value || "").replace(/\s+/g, " ").trim();
+  }
+
+  function absoluteUrl(href) {
+    if (!href) return "";
+    try {
+      return new URL(href, location.origin).href;
+    } catch {
+      return href;
+    }
+  }
+
+  function normalizeThingId(raw, kind) {
+    if (!raw) return "";
+    raw = String(raw);
+    if (raw.startsWith("t1_") || raw.startsWith("t3_")) return raw;
+    if (kind === "comment") return `t1_${raw.replace(/^t1_/, "")}`;
+    if (kind === "post") return `t3_${raw.replace(/^t3_/, "")}`;
+    return raw;
+  }
+
+  // querySelectorAll does not pierce shadow roots; Reddit nests comments there.
+  function queryAllDeep(root, selector) {
+    const results = [];
+    const seen = new Set();
+
+    function visit(node) {
+      if (!node || !node.querySelectorAll) return;
+      let matches;
+      try {
+        matches = node.querySelectorAll(selector);
+      } catch {
+        return;
+      }
+      matches.forEach((el) => {
+        if (seen.has(el)) return;
+        seen.add(el);
+        results.push(el);
+      });
+      node.querySelectorAll("*").forEach((el) => {
+        if (el.shadowRoot) visit(el.shadowRoot);
+      });
+    }
+
+    visit(root);
+    return results;
+  }
+
+  function debugDomHints() {
+    const tags = new Set();
+    let openShadows = 0;
+    walkDom(document.documentElement, (el) => {
+      const name = (el.tagName || "").toLowerCase();
+      if (
+        name.includes("comment") ||
+        name.includes("shreddit") ||
+        name.includes("faceplate") ||
+        (el.id && String(el.id).includes("t1_")) ||
+        (el.getAttribute && (el.getAttribute("thingid") || "").includes("t1_"))
+      ) {
+        tags.add(name || "(anon)");
+      }
+      if (el.shadowRoot) openShadows += 1;
+    });
+    return { tags: [...tags].sort().slice(0, 40), openShadows };
+  }
+
+  function scrapeOldPost() {
+    const thing = document.querySelector("#siteTable .thing.link");
+    if (!thing) return null;
+    const postId = normalizeThingId(thing.getAttribute("data-fullname") || thing.id.replace(/^thing_/, ""), "post");
+    const titleEl = thing.querySelector("a.title");
+    const authorEl = thing.querySelector("a.author");
+    const selftext = thing.querySelector(".usertext-body .md");
+    const subreddit = thing.getAttribute("data-subreddit") || "";
+    return {
+      reddit_post_id: postId,
+      title: cleanText(titleEl && titleEl.textContent),
+      body: selftext ? selftext.innerText.trim() : "",
+      author_id: thing.getAttribute("data-author-fullname") || "",
+      author_name: cleanText((authorEl && authorEl.textContent) || thing.getAttribute("data-author") || "[deleted]"),
+      permalink: absoluteUrl(thing.getAttribute("data-permalink") || (titleEl && titleEl.getAttribute("href"))),
+      subreddit,
+      element: thing,
+    };
+  }
+
+  function scrapeOldComments(post) {
+    const nodes = document.querySelectorAll(".commentarea .thing.comment");
+    const comments = [];
+    nodes.forEach((node) => {
+      if (!node.getBoundingClientRect) return;
+      const id = normalizeThingId(node.getAttribute("data-fullname") || node.id.replace(/^thing_/, ""), "comment");
+      if (!id) return;
+      const authorEl = node.querySelector(":scope > .entry a.author, :scope > .entry .tagline a.author");
+      const bodyEl = node.querySelector(":scope > .entry .usertext-body .md, :scope > .entry .usertext-body");
+      const permalinkEl = node.querySelector(':scope > .entry a[data-event-action="permalink"], :scope > .entry a.bylink');
+      const shotTarget = node.querySelector(":scope > .entry") || node;
+      comments.push({
+        reddit_comment_id: id,
+        body: bodyEl ? bodyEl.innerText.trim() : "",
+        author_id: node.getAttribute("data-author-fullname") || "",
+        author_name: cleanText((authorEl && authorEl.textContent) || node.getAttribute("data-author") || "[deleted]"),
+        reddit_post_id: post ? post.reddit_post_id : "",
+        post_title: post ? post.title : "",
+        post_body: post ? post.body : "",
+        post_author_id: post ? post.author_id : "",
+        post_author_name: post ? post.author_name : "",
+        post_permalink: post ? post.permalink : "",
+        subreddit: post ? post.subreddit : (node.getAttribute("data-subreddit") || ""),
+        permalink: absoluteUrl(permalinkEl && permalinkEl.getAttribute("href")),
+        element: shotTarget,
+      });
+    });
+    return comments;
+  }
+
+  function scrapeNewPost() {
+    const post =
+      queryAllDeep(document, "shreddit-post")[0] ||
+      document.querySelector('[id^="t3_"]') ||
+      document.querySelector('div[data-testid="post-container"]');
+    if (!post) {
+      const match = location.pathname.match(/\/comments\/([a-z0-9]+)\//i);
+      if (!match) return null;
+      return {
+        reddit_post_id: normalizeThingId(match[1], "post"),
+        title: cleanText(document.title.replace(/\s*:\s*r\/.*$/, "")),
+        body: "",
+        author_id: "",
+        author_name: "",
+        permalink: location.href.split("?")[0],
+        subreddit: (location.pathname.match(/\/r\/([^/]+)/) || [])[1] || "",
+        element: document.querySelector("main") || document.body,
+      };
+    }
+
+    const postId =
+      normalizeThingId(post.getAttribute("id") || post.getAttribute("thingid") || post.getAttribute("post-id"), "post") ||
+      (() => {
+        const m = location.pathname.match(/\/comments\/([a-z0-9]+)\//i);
+        return m ? normalizeThingId(m[1], "post") : "";
+      })();
+
+    const title =
+      post.getAttribute("post-title") ||
+      cleanText((post.querySelector('h1, [slot="title"], a[slot="title"]') || {}).textContent) ||
+      cleanText(document.title);
+
+    const authorName =
+      post.getAttribute("author") ||
+      cleanText((post.querySelector('a[href*="/user/"], [slot="authorName"]') || {}).textContent) ||
+      "";
+
+    const bodyEl =
+      post.querySelector('[slot="text-body"], .text-container, [data-testid="post-content"]') ||
+      post.querySelector("p");
+
+    const subreddit =
+      post.getAttribute("subreddit-name") ||
+      (location.pathname.match(/\/r\/([^/]+)/) || [])[1] ||
+      "";
+
+    const permalink =
+      absoluteUrl(post.getAttribute("permalink") || post.getAttribute("content-href")) ||
+      location.href.split("?")[0];
+
+    return {
+      reddit_post_id: postId,
+      title,
+      body: bodyEl ? bodyEl.innerText.trim() : "",
+      author_id: post.getAttribute("author-id") || "",
+      author_name: authorName || "[deleted]",
+      permalink,
+      subreddit,
+      element: post,
+    };
+  }
+
+  function findCommentElement(commentId) {
+    const bare = String(commentId || "").replace(/^t1_/, "");
+    const full = String(commentId || "").startsWith("t1_") ? String(commentId) : `t1_${bare}`;
+    const want = new Set([full, `t1_${bare}`, bare, `thing_${full}`, `thing_${bare}`].filter(Boolean));
+
+    // Classic old Reddit markup (www or old.reddit.com).
+    const classic =
+      document.querySelector(`.thing.comment[data-fullname="${full}"]`) ||
+      document.querySelector(`.thing.comment[data-fullname="${bare}"]`) ||
+      document.querySelector(`#thing_${full}`) ||
+      document.querySelector(`#thing_${bare}`);
+    if (classic) {
+      return classic.querySelector(":scope > .entry") || classic;
+    }
+
+    const selectors = [
+      `shreddit-comment[thingid="${full}"]`,
+      `shreddit-comment[thingid="t1_${bare}"]`,
+      `shreddit-comment[id="${full}"]`,
+      `[thingid="${full}"]`,
+      `[thingid="t1_${bare}"]`,
+      `[comment-id="${full}"]`,
+      `[data-fullname="${full}"]`,
+      `[data-fullname="${bare}"]`,
+      `[id="t1_${bare}"]`,
+      `[id="${bare}"]`,
+      `[id="thing_${full}"]`,
+      `[id="thing_${bare}"]`,
+    ];
+    for (const selector of selectors) {
+      try {
+        const hit = queryAllDeep(document, selector)[0];
+        if (hit) {
+          if (hit.classList && hit.classList.contains("comment")) {
+            return hit.querySelector(":scope > .entry") || hit;
+          }
+          return preferShotTarget(hit);
+        }
+      } catch {
+        // ignore
+      }
+    }
+
+    let found = null;
+    walkDom(document.documentElement, (el) => {
+      if (found || !el.getAttribute) return;
+      const candidates = [
+        el.getAttribute("thingid"),
+        el.getAttribute("comment-id"),
+        el.getAttribute("data-fullname"),
+        el.getAttribute("id"),
+        el.getAttribute("data-comment-id"),
+        el.getAttribute("name"),
+      ];
+      for (const raw of candidates) {
+        if (!raw) continue;
+        const cleaned = String(raw).replace(/^thing_/, "");
+        const normalized = normalizeThingId(cleaned, "comment");
+        const stripped = cleaned.replace(/^t1_/, "");
+        if (want.has(raw) || want.has(cleaned) || want.has(normalized) || want.has(stripped)) {
+          if (el.classList && el.classList.contains("comment")) {
+            found = el.querySelector(":scope > .entry") || el;
+          } else {
+            found = preferShotTarget(el);
+          }
+          return;
+        }
+      }
+      if (el.tagName === "A") {
+        const href = el.getAttribute("href") || "";
+        if (href.includes(`/${bare}`) || href.includes(full) || href.includes(`t1_${bare}`)) {
+          found = preferShotTarget(el);
+        }
+      }
+    });
+    return found;
+  }
+
+  function preferShotTarget(el) {
+    if (!el) return null;
+    // Prefer a comment-sized container over a tiny permalink icon/link.
+    let best = el;
+    let node = el;
+    for (let depth = 0; depth < 10 && node; depth += 1) {
+      const rect = node.getBoundingClientRect ? node.getBoundingClientRect() : null;
+      if (rect && rect.height >= 48 && rect.width >= 120) {
+        best = node;
+        // Prefer shreddit-comment host when available.
+        if ((node.tagName || "").toLowerCase() === "shreddit-comment") break;
+      }
+      node = node.parentElement;
+    }
+    return best;
+  }
+
+  function walkDom(root, visit) {
+    const stack = [root];
+    while (stack.length) {
+      const node = stack.pop();
+      if (!node) continue;
+      visit(node);
+      if (node.shadowRoot) stack.push(node.shadowRoot);
+      const children = node.children;
+      if (children) {
+        for (let i = 0; i < children.length; i += 1) {
+          stack.push(children[i]);
+        }
+      }
+    }
+  }
+
+  function scrapeNewComments(post) {
+    const selector = [
+      "shreddit-comment",
+      "[thingid^='t1_']",
+      "[id^='t1_']",
+      'div[data-testid="comment"]',
+      "faceplate-tracker[noun='comment']",
+    ].join(", ");
+    const nodes = queryAllDeep(document, selector);
+    const comments = [];
+    const seen = new Set();
+
+    nodes.forEach((node) => {
+      let id =
+        node.getAttribute("thingid") ||
+        node.getAttribute("comment-id") ||
+        node.getAttribute("id") ||
+        "";
+      // Some wrappers put the id on a child.
+      if (!id || (!id.startsWith("t1_") && !/^t1_/.test(id))) {
+        const nested = node.querySelector("[thingid^='t1_'], [id^='t1_']");
+        if (nested) {
+          id = nested.getAttribute("thingid") || nested.id || id;
+        }
+      }
+      id = normalizeThingId(id, "comment");
+      if (!id || !id.startsWith("t1_") || seen.has(id)) return;
+      seen.add(id);
+
+      const authorName =
+        node.getAttribute("author") ||
+        cleanText((node.querySelector('a[href*="/user/"]') || {}).textContent) ||
+        "[deleted]";
+
+      const bodyEl =
+        node.querySelector('[slot="comment"], .md, [id$="-post-rtjson-content"], [data-testid="comment"]') ||
+        node.querySelector("p");
+
+      const permalinkAttr = node.getAttribute("permalink");
+      const permalinkLink = node.querySelector('a[href*="/comments/"]');
+      const permalink =
+        absoluteUrl(permalinkAttr) ||
+        absoluteUrl(permalinkLink && permalinkLink.getAttribute("href")) ||
+        "";
+
+      comments.push({
+        reddit_comment_id: id,
+        body: bodyEl ? bodyEl.innerText.trim() : cleanText(node.innerText).slice(0, 5000),
+        author_id: node.getAttribute("author-id") || "",
+        author_name: authorName,
+        reddit_post_id: post ? post.reddit_post_id : "",
+        post_title: post ? post.title : "",
+        post_body: post ? post.body : "",
+        post_author_id: post ? post.author_id : "",
+        post_author_name: post ? post.author_name : "",
+        post_permalink: post ? post.permalink : "",
+        subreddit: post ? post.subreddit : "",
+        permalink,
+        element: node,
+      });
+    });
+    return comments;
+  }
+
+  function walkListingComments(children, out) {
+    if (!Array.isArray(children)) return;
+    children.forEach((child) => {
+      if (!child || child.kind !== "t1" || !child.data) return;
+      const d = child.data;
+      const id = normalizeThingId(d.name || d.id, "comment");
+      if (!id) return;
+      out.push({
+        reddit_comment_id: id,
+        body: d.body || "",
+        author_id: d.author_fullname || "",
+        author_name: d.author || "[deleted]",
+        permalink: absoluteUrl(d.permalink),
+        subreddit: d.subreddit || "",
+      });
+      if (d.replies && d.replies.data && d.replies.data.children) {
+        walkListingComments(d.replies.data.children, out);
+      }
+    });
+  }
+
+  let jsonCache = { postId: "", at: 0, comments: [], error: "" };
+
+  async function fetchThreadCommentsJson(post) {
+    if (!post || !post.reddit_post_id) return [];
+    const now = Date.now();
+    if (jsonCache.postId === post.reddit_post_id && now - jsonCache.at < 15000) {
+      if (jsonCache.error) throw new Error(jsonCache.error);
+      return jsonCache.comments.map((c) => ({
+        ...c,
+        element: findCommentElement(c.reddit_comment_id),
+      }));
+    }
+    const bare = post.reddit_post_id.replace(/^t3_/, "");
+    try {
+      const response = await chrome.runtime.sendMessage({
+        type: "rcs.redditJson",
+        path: `/comments/${bare}.json?limit=500&raw_json=1`,
+      });
+      if (!response || !response.ok) {
+        throw new Error((response && response.error) || "reddit json failed");
+      }
+      const listing = response.data;
+      const comments = [];
+      if (Array.isArray(listing) && listing[1] && listing[1].data) {
+        walkListingComments(listing[1].data.children, comments);
+      }
+      const mapped = comments.map((c) => ({
+        ...c,
+        reddit_post_id: post.reddit_post_id,
+        post_title: post.title,
+        post_body: post.body,
+        post_author_id: post.author_id,
+        post_author_name: post.author_name,
+        post_permalink: post.permalink,
+        subreddit: c.subreddit || post.subreddit,
+      }));
+      jsonCache = { postId: post.reddit_post_id, at: now, comments: mapped, error: "" };
+      return mapped.map((c) => ({
+        ...c,
+        element: findCommentElement(c.reddit_comment_id),
+      }));
+    } catch (err) {
+      jsonCache = {
+        postId: post.reddit_post_id,
+        at: now,
+        comments: [],
+        error: String(err && err.message ? err.message : err),
+      };
+      throw err;
+    }
+  }
+
+  function mergeComments(domComments, jsonComments) {
+    const byId = new Map();
+    domComments.forEach((c) => byId.set(c.reddit_comment_id, c));
+    jsonComments.forEach((c) => {
+      const existing = byId.get(c.reddit_comment_id);
+      if (!existing) {
+        byId.set(c.reddit_comment_id, c);
+        return;
+      }
+      // Prefer non-empty body / author from JSON when DOM is empty.
+      if (!existing.body && c.body) existing.body = c.body;
+      if ((!existing.author_name || existing.author_name === "[deleted]") && c.author_name) {
+        existing.author_name = c.author_name;
+      }
+      if (!existing.permalink && c.permalink) existing.permalink = c.permalink;
+      if (!existing.element && c.element) existing.element = c.element;
+    });
+    return [...byId.values()];
+  }
+
+  function scrapePage() {
+    const classic = isOldReddit();
+    const post = classic ? scrapeOldPost() : scrapeNewPost();
+    const comments = classic ? scrapeOldComments(post) : scrapeNewComments(post);
+    return { post, comments, isOld: classic, isShreddit: !classic && isShreddit() };
+  }
+
+  async function scrapePageAsync() {
+    const base = scrapePage();
+    if (base.isOld) return base;
+    try {
+      const jsonComments = await fetchThreadCommentsJson(base.post);
+      return {
+        ...base,
+        comments: mergeComments(base.comments, jsonComments),
+        jsonCount: jsonComments.length,
+        domCount: base.comments.length,
+      };
+    } catch (err) {
+      return {
+        ...base,
+        jsonError: String(err && err.message ? err.message : err),
+        jsonCount: 0,
+        domCount: base.comments.length,
+      };
+    }
+  }
+
+  function isElementMostlyVisible(el) {
+    if (!el) return false;
+    const rect = el.getBoundingClientRect();
+    if (rect.width < 2 || rect.height < 2) return false;
+    const vh = window.innerHeight || document.documentElement.clientHeight;
+    return rect.bottom > 0 && rect.top < vh;
+  }
+
+  global.RCSScrape = {
+    scrapePage,
+    scrapePageAsync,
+    isElementMostlyVisible,
+    isOldReddit,
+    debugDomHints,
+    findCommentElement,
+  };
+})(typeof window !== "undefined" ? window : self);