Add self-hosted RCS backend, extension, and ops tooling
Ship the Go/SQLite API and Web UI, Chrome/Brave capture addon, Docker Compose, Pangolin reverse-proxy support, and a user-crontab watchdog so the binary stays running without systemd.
This commit is contained in:
@@ -0,0 +1,479 @@
|
||||
// Auto-capture visible comments/posts with backend dedupe and queued screenshots.
|
||||
// Backend I/O goes through lib/api.js → background service worker (never localhost fetch here).
|
||||
|
||||
(function () {
|
||||
const processed = new Set(); // fully done (saved + screenshot, or blocked)
|
||||
const needsScreenshot = new Set(); // saved text but still missing screenshot
|
||||
const queue = [];
|
||||
let busy = false;
|
||||
let configured = false;
|
||||
let observer = null;
|
||||
let scanCount = 0;
|
||||
let blockedUsers = new Set();
|
||||
|
||||
async function refreshBlockedUsers() {
|
||||
const list = await rcsGetCachedBlocklist();
|
||||
blockedUsers = new Set((list || []).map(rcsNormalizeUsername).filter(Boolean));
|
||||
}
|
||||
|
||||
function isAuthorBlocked(authorName) {
|
||||
const normalized = rcsNormalizeUsername(authorName);
|
||||
return Boolean(normalized) && blockedUsers.has(normalized);
|
||||
}
|
||||
|
||||
async function refreshConfig() {
|
||||
const settings = await rcsGetSettings();
|
||||
configured = Boolean(settings.backendUrl);
|
||||
return configured;
|
||||
}
|
||||
|
||||
function enqueue(job) {
|
||||
queue.push(job);
|
||||
pump();
|
||||
}
|
||||
|
||||
async function pump() {
|
||||
if (busy) return;
|
||||
busy = true;
|
||||
while (queue.length) {
|
||||
const job = queue.shift();
|
||||
try {
|
||||
await job();
|
||||
} catch (err) {
|
||||
console.warn("[RCS] queue job failed:", err && err.message ? err.message : err);
|
||||
}
|
||||
await sleep(200);
|
||||
}
|
||||
busy = false;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function resolveCommentElement(comment) {
|
||||
if (comment && comment.element && comment.element.isConnected) {
|
||||
return comment.element;
|
||||
}
|
||||
if (!window.RCSScrape || !comment || !comment.reddit_comment_id) return null;
|
||||
return window.RCSScrape.findCommentElement(comment.reddit_comment_id);
|
||||
}
|
||||
|
||||
function clampRectToViewport(rect) {
|
||||
const vw = window.innerWidth || document.documentElement.clientWidth;
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight;
|
||||
const left = Math.max(0, rect.left);
|
||||
const top = Math.max(0, rect.top);
|
||||
const right = Math.min(vw, rect.right);
|
||||
const bottom = Math.min(vh, rect.bottom);
|
||||
return {
|
||||
x: left,
|
||||
y: top,
|
||||
width: Math.max(0, right - left),
|
||||
height: Math.max(0, bottom - top),
|
||||
};
|
||||
}
|
||||
|
||||
function isElementInViewportEnough(element) {
|
||||
if (!element || !element.getBoundingClientRect) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight;
|
||||
const vw = window.innerWidth || document.documentElement.clientWidth;
|
||||
if (rect.width < 40 || rect.height < 20) return false;
|
||||
const visibleH = Math.min(rect.bottom, vh) - Math.max(rect.top, 0);
|
||||
const visibleW = Math.min(rect.right, vw) - Math.max(rect.left, 0);
|
||||
return visibleH >= Math.min(rect.height, 40) && visibleW >= Math.min(rect.width, 40);
|
||||
}
|
||||
|
||||
// Never scrolls the page — only crops if the element is already on-screen.
|
||||
async function captureElementScreenshot(element) {
|
||||
if (!element) return "";
|
||||
if (!isElementInViewportEnough(element)) return "";
|
||||
|
||||
const fresh = element.getBoundingClientRect();
|
||||
const clipped = clampRectToViewport(fresh);
|
||||
if (clipped.width < 10 || clipped.height < 10) return "";
|
||||
|
||||
const response = await chrome.runtime.sendMessage({
|
||||
type: "rcs.captureCrop",
|
||||
rect: clipped,
|
||||
dpr: window.devicePixelRatio || 1,
|
||||
});
|
||||
|
||||
if (!response || !response.ok) {
|
||||
throw new Error((response && response.error) || "screenshot failed");
|
||||
}
|
||||
return response.dataUrl || "";
|
||||
}
|
||||
|
||||
// When Reddit hides comments in closed shadow DOM, render a card ourselves.
|
||||
async function renderSyntheticScreenshot(comment) {
|
||||
const width = 720;
|
||||
const pad = 20;
|
||||
const author = `u/${comment.author_name || "[deleted]"}`;
|
||||
const subreddit = String(comment.subreddit || "").replace(/^r\//i, "").trim();
|
||||
const byline = subreddit ? `${author} · r/${subreddit}` : author;
|
||||
const body = String(comment.body || "").trim() || "(empty comment)";
|
||||
const meta = comment.permalink || comment.reddit_comment_id || "";
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
const ctx = canvas.getContext("2d");
|
||||
const dpr = Math.min(window.devicePixelRatio || 1, 2);
|
||||
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
||||
|
||||
function wrap(text, maxWidth) {
|
||||
const words = text.split(/\s+/);
|
||||
const lines = [];
|
||||
let line = "";
|
||||
words.forEach((word) => {
|
||||
const test = line ? `${line} ${word}` : word;
|
||||
if (ctx.measureText(test).width > maxWidth && line) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = test;
|
||||
}
|
||||
});
|
||||
if (line) lines.push(line);
|
||||
// Hard-wrap very long tokens
|
||||
return lines.flatMap((l) => {
|
||||
if (ctx.measureText(l).width <= maxWidth) return [l];
|
||||
const out = [];
|
||||
let cur = "";
|
||||
for (const ch of l) {
|
||||
const t = cur + ch;
|
||||
if (ctx.measureText(t).width > maxWidth && cur) {
|
||||
out.push(cur);
|
||||
cur = ch;
|
||||
} else {
|
||||
cur = t;
|
||||
}
|
||||
}
|
||||
if (cur) out.push(cur);
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
const maxText = width - pad * 2;
|
||||
ctx.font = "bold 15px system-ui, Segoe UI, sans-serif";
|
||||
const bylineLines = wrap(byline, maxText);
|
||||
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
||||
const bodyLines = wrap(body.slice(0, 4000), maxText);
|
||||
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
||||
const metaLines = wrap(meta, maxText);
|
||||
|
||||
const lineH = 22;
|
||||
const height = pad * 2 + 28 + bylineLines.length * lineH + 8 + bodyLines.length * lineH + 12 + metaLines.length * 16 + 24;
|
||||
canvas.width = Math.floor(width * dpr);
|
||||
canvas.height = Math.floor(height * dpr);
|
||||
ctx.scale(dpr, dpr);
|
||||
|
||||
// Card background
|
||||
ctx.fillStyle = "#1a1a1b";
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
ctx.strokeStyle = "#343536";
|
||||
ctx.lineWidth = 2;
|
||||
ctx.strokeRect(1, 1, width - 2, height - 2);
|
||||
|
||||
let y = pad + 8;
|
||||
ctx.fillStyle = "#818384";
|
||||
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
||||
ctx.fillText("RCS comment preview", pad, y);
|
||||
y += 24;
|
||||
|
||||
ctx.fillStyle = "#4fbcff";
|
||||
ctx.font = "bold 15px system-ui, Segoe UI, sans-serif";
|
||||
bylineLines.forEach((line) => {
|
||||
// Draw author in blue, subreddit segment in muted if present on same line.
|
||||
if (subreddit && line.includes(" · r/")) {
|
||||
const parts = line.split(" · ");
|
||||
let x = pad;
|
||||
ctx.fillStyle = "#4fbcff";
|
||||
ctx.fillText(parts[0] || "", x, y);
|
||||
x += ctx.measureText(parts[0] || "").width;
|
||||
if (parts[1]) {
|
||||
ctx.fillStyle = "#818384";
|
||||
const sep = " · ";
|
||||
ctx.fillText(sep, x, y);
|
||||
x += ctx.measureText(sep).width;
|
||||
ctx.fillStyle = "#ff7a45";
|
||||
ctx.fillText(parts[1], x, y);
|
||||
}
|
||||
} else {
|
||||
ctx.fillStyle = "#4fbcff";
|
||||
ctx.fillText(line, pad, y);
|
||||
}
|
||||
y += lineH;
|
||||
});
|
||||
y += 6;
|
||||
|
||||
ctx.fillStyle = "#d7dadc";
|
||||
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
||||
bodyLines.forEach((line) => {
|
||||
ctx.fillText(line, pad, y);
|
||||
y += lineH;
|
||||
});
|
||||
y += 10;
|
||||
|
||||
ctx.fillStyle = "#818384";
|
||||
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
||||
metaLines.forEach((line) => {
|
||||
ctx.fillText(line, pad, y);
|
||||
y += 16;
|
||||
});
|
||||
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
const pendingLogAt = new Map();
|
||||
|
||||
async function savePostIfNeeded(post) {
|
||||
if (!post || !post.reddit_post_id || processed.has("post:" + post.reddit_post_id)) return;
|
||||
processed.add("post:" + post.reddit_post_id);
|
||||
const exists = await rcsPostExists(post.reddit_post_id);
|
||||
if (exists) {
|
||||
console.debug("[RCS] post already saved:", post.reddit_post_id);
|
||||
return;
|
||||
}
|
||||
await rcsSavePost({
|
||||
reddit_post_id: post.reddit_post_id,
|
||||
title: post.title,
|
||||
body: post.body,
|
||||
author_id: post.author_id,
|
||||
author_name: post.author_name,
|
||||
permalink: post.permalink,
|
||||
subreddit: post.subreddit,
|
||||
});
|
||||
console.info("[RCS] saved post:", post.reddit_post_id, post.title);
|
||||
}
|
||||
|
||||
async function captureAndMaybeBackfill(comment, status) {
|
||||
const element = resolveCommentElement(comment);
|
||||
comment.element = element;
|
||||
|
||||
let screenshot = "";
|
||||
if (element && isElementInViewportEnough(element)) {
|
||||
try {
|
||||
screenshot = await captureElementScreenshot(element);
|
||||
if (screenshot) {
|
||||
console.info("[RCS] DOM screenshot captured (no scroll):", comment.reddit_comment_id);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[RCS] DOM screenshot failed:", comment.reddit_comment_id, err && err.message ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
if (!screenshot) {
|
||||
// Off-screen / closed-shadow: render without moving the page.
|
||||
try {
|
||||
screenshot = await renderSyntheticScreenshot(comment);
|
||||
console.info("[RCS] synthetic screenshot rendered:", comment.reddit_comment_id);
|
||||
} catch (err) {
|
||||
needsScreenshot.add(comment.reddit_comment_id);
|
||||
const now = Date.now();
|
||||
const last = pendingLogAt.get(comment.reddit_comment_id) || 0;
|
||||
if (now - last > 10000) {
|
||||
pendingLogAt.set(comment.reddit_comment_id, now);
|
||||
console.info("[RCS] screenshot pending (no DOM yet):", comment.reddit_comment_id);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
if (!screenshot) {
|
||||
needsScreenshot.add(comment.reddit_comment_id);
|
||||
return "";
|
||||
}
|
||||
|
||||
if (status && status.exists && !status.has_screenshot) {
|
||||
await rcsAttachScreenshot(comment.reddit_comment_id, screenshot);
|
||||
console.info("[RCS] backfilled screenshot:", comment.reddit_comment_id);
|
||||
}
|
||||
|
||||
needsScreenshot.delete(comment.reddit_comment_id);
|
||||
pendingLogAt.delete(comment.reddit_comment_id);
|
||||
return screenshot;
|
||||
}
|
||||
|
||||
async function saveCommentIfNeeded(comment) {
|
||||
if (!comment || !comment.reddit_comment_id) return;
|
||||
if (isAuthorBlocked(comment.author_name)) {
|
||||
console.info("[RCS] skipped blocked user:", comment.author_name, comment.reddit_comment_id);
|
||||
processed.add("comment:" + comment.reddit_comment_id);
|
||||
return;
|
||||
}
|
||||
|
||||
const id = comment.reddit_comment_id;
|
||||
const doneKey = "comment:" + id;
|
||||
if (processed.has(doneKey) && !needsScreenshot.has(id)) return;
|
||||
|
||||
const status = await rcsCommentExists(id);
|
||||
|
||||
if (status.exists && status.has_screenshot) {
|
||||
processed.add(doneKey);
|
||||
needsScreenshot.delete(id);
|
||||
console.debug("[RCS] comment complete:", id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.exists && !status.has_screenshot) {
|
||||
const shot = await captureAndMaybeBackfill(comment, status);
|
||||
if (shot) {
|
||||
processed.add(doneKey);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// New comment
|
||||
const screenshot = await captureAndMaybeBackfill(comment, status);
|
||||
const result = await rcsSaveComment({
|
||||
reddit_comment_id: id,
|
||||
body: comment.body,
|
||||
author_id: comment.author_id,
|
||||
author_name: comment.author_name,
|
||||
reddit_post_id: comment.reddit_post_id,
|
||||
post_title: comment.post_title,
|
||||
post_body: comment.post_body,
|
||||
post_author_id: comment.post_author_id,
|
||||
post_author_name: comment.post_author_name,
|
||||
post_permalink: comment.post_permalink,
|
||||
subreddit: comment.subreddit,
|
||||
permalink: comment.permalink,
|
||||
screenshot_base64: screenshot,
|
||||
});
|
||||
|
||||
if (screenshot) {
|
||||
processed.add(doneKey);
|
||||
needsScreenshot.delete(id);
|
||||
} else {
|
||||
needsScreenshot.add(id);
|
||||
}
|
||||
|
||||
console.info(
|
||||
"[RCS] saved comment:",
|
||||
id,
|
||||
"by",
|
||||
comment.author_name,
|
||||
result && result.saved ? "(new)" : "(exists)",
|
||||
screenshot ? "+screenshot" : "no-screenshot-yet"
|
||||
);
|
||||
}
|
||||
|
||||
function shouldProcessComment(comment) {
|
||||
if (!comment || !comment.reddit_comment_id) return false;
|
||||
const id = comment.reddit_comment_id;
|
||||
if (processed.has("comment:" + id) && !needsScreenshot.has(id)) return false;
|
||||
|
||||
const element = resolveCommentElement(comment);
|
||||
comment.element = element;
|
||||
|
||||
// Prefer comments that are on-screen (or have a node we can scroll to for backfill).
|
||||
if (needsScreenshot.has(id) && element) return true;
|
||||
if (window.RCSScrape.isElementMostlyVisible(element)) return true;
|
||||
// JSON-only with no DOM: still allow text save once.
|
||||
if (!element && !processed.has("comment:" + id)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
async function scan() {
|
||||
if (!configured || !window.RCSScrape) return;
|
||||
scanCount += 1;
|
||||
const page = await window.RCSScrape.scrapePageAsync();
|
||||
const { post, comments, isOld } = page;
|
||||
|
||||
// Re-attach DOM nodes for pending screenshot ids.
|
||||
comments.forEach((c) => {
|
||||
if (!c.element) c.element = resolveCommentElement(c);
|
||||
});
|
||||
|
||||
const candidates = comments.filter(shouldProcessComment);
|
||||
if (scanCount <= 5 || scanCount % 20 === 0 || comments.length === 0) {
|
||||
console.info(
|
||||
"[RCS] scan#" + scanCount,
|
||||
isOld ? "classic-ui" : page.isShreddit ? "shreddit" : "www.reddit",
|
||||
"post=",
|
||||
post && post.reddit_post_id,
|
||||
"dom=",
|
||||
page.domCount != null ? page.domCount : "?",
|
||||
"json=",
|
||||
page.jsonCount != null ? page.jsonCount : "n/a",
|
||||
"merged=",
|
||||
comments.length,
|
||||
"queue=",
|
||||
candidates.length,
|
||||
"needShot=",
|
||||
needsScreenshot.size,
|
||||
page.jsonError ? "jsonError=" + page.jsonError : ""
|
||||
);
|
||||
if (comments.length === 0 && !isOld) {
|
||||
console.info("[RCS] DOM hints:", window.RCSScrape.debugDomHints());
|
||||
}
|
||||
}
|
||||
if (post) {
|
||||
enqueue(() => savePostIfNeeded(post));
|
||||
}
|
||||
candidates.forEach((comment) => {
|
||||
enqueue(() => saveCommentIfNeeded(comment));
|
||||
});
|
||||
}
|
||||
|
||||
function startObserver() {
|
||||
if (observer) return;
|
||||
let timer = null;
|
||||
observer = new MutationObserver(() => {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(scan, 400);
|
||||
});
|
||||
observer.observe(document.documentElement, { childList: true, subtree: true });
|
||||
window.addEventListener("scroll", () => {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(scan, 400);
|
||||
}, { passive: true });
|
||||
}
|
||||
|
||||
async function boot() {
|
||||
await refreshConfig();
|
||||
if (!configured) {
|
||||
console.info("[RCS] backend URL not set — open the extension popup to configure.");
|
||||
return;
|
||||
}
|
||||
console.info("[RCS] configured; capturing via background API proxy (no page→localhost fetch).");
|
||||
try {
|
||||
await rcsHealth();
|
||||
console.info("[RCS] backend health OK");
|
||||
} catch (err) {
|
||||
console.warn("[RCS] backend health failed:", err && err.message ? err.message : err);
|
||||
}
|
||||
try {
|
||||
const list = await rcsPullBlocklist();
|
||||
blockedUsers = new Set(list.map(rcsNormalizeUsername));
|
||||
console.info("[RCS] blocklist synced:", list.length, "users");
|
||||
} catch (err) {
|
||||
await refreshBlockedUsers();
|
||||
console.warn("[RCS] blocklist pull failed, using cache:", err && err.message ? err.message : err);
|
||||
}
|
||||
startObserver();
|
||||
scan();
|
||||
}
|
||||
|
||||
chrome.storage.onChanged.addListener((changes, area) => {
|
||||
if (area !== "sync") return;
|
||||
if (changes.blockedUsernames) {
|
||||
refreshBlockedUsers();
|
||||
}
|
||||
if (changes.backendUrl || changes.apiKey) {
|
||||
refreshConfig().then(() => {
|
||||
if (configured) {
|
||||
startObserver();
|
||||
scan();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (document.readyState === "loading") {
|
||||
document.addEventListener("DOMContentLoaded", boot);
|
||||
} else {
|
||||
boot();
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,511 @@
|
||||
// DOM scrapers for old.reddit.com and www.reddit.com (new / shreddit design).
|
||||
|
||||
(function (global) {
|
||||
function isOldReddit() {
|
||||
if (location.hostname === "old.reddit.com") return true;
|
||||
// www.reddit.com can still serve classic markup (user preference / redirects / extensions).
|
||||
// Prefer classic scrapers whenever those nodes exist — don't rely on hostname alone.
|
||||
return Boolean(
|
||||
document.querySelector(
|
||||
".commentarea .thing.comment, #siteTable .thing.comment, .thing.comment[data-fullname^='t1_'], body.listing-page .thing.link"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function isShreddit() {
|
||||
return Boolean(document.querySelector("shreddit-post, shreddit-comment, shreddit-comment-tree, shreddit-app"));
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return (value || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function absoluteUrl(href) {
|
||||
if (!href) return "";
|
||||
try {
|
||||
return new URL(href, location.origin).href;
|
||||
} catch {
|
||||
return href;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeThingId(raw, kind) {
|
||||
if (!raw) return "";
|
||||
raw = String(raw);
|
||||
if (raw.startsWith("t1_") || raw.startsWith("t3_")) return raw;
|
||||
if (kind === "comment") return `t1_${raw.replace(/^t1_/, "")}`;
|
||||
if (kind === "post") return `t3_${raw.replace(/^t3_/, "")}`;
|
||||
return raw;
|
||||
}
|
||||
|
||||
// querySelectorAll does not pierce shadow roots; Reddit nests comments there.
|
||||
function queryAllDeep(root, selector) {
|
||||
const results = [];
|
||||
const seen = new Set();
|
||||
|
||||
function visit(node) {
|
||||
if (!node || !node.querySelectorAll) return;
|
||||
let matches;
|
||||
try {
|
||||
matches = node.querySelectorAll(selector);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
matches.forEach((el) => {
|
||||
if (seen.has(el)) return;
|
||||
seen.add(el);
|
||||
results.push(el);
|
||||
});
|
||||
node.querySelectorAll("*").forEach((el) => {
|
||||
if (el.shadowRoot) visit(el.shadowRoot);
|
||||
});
|
||||
}
|
||||
|
||||
visit(root);
|
||||
return results;
|
||||
}
|
||||
|
||||
function debugDomHints() {
|
||||
const tags = new Set();
|
||||
let openShadows = 0;
|
||||
walkDom(document.documentElement, (el) => {
|
||||
const name = (el.tagName || "").toLowerCase();
|
||||
if (
|
||||
name.includes("comment") ||
|
||||
name.includes("shreddit") ||
|
||||
name.includes("faceplate") ||
|
||||
(el.id && String(el.id).includes("t1_")) ||
|
||||
(el.getAttribute && (el.getAttribute("thingid") || "").includes("t1_"))
|
||||
) {
|
||||
tags.add(name || "(anon)");
|
||||
}
|
||||
if (el.shadowRoot) openShadows += 1;
|
||||
});
|
||||
return { tags: [...tags].sort().slice(0, 40), openShadows };
|
||||
}
|
||||
|
||||
function scrapeOldPost() {
|
||||
const thing = document.querySelector("#siteTable .thing.link");
|
||||
if (!thing) return null;
|
||||
const postId = normalizeThingId(thing.getAttribute("data-fullname") || thing.id.replace(/^thing_/, ""), "post");
|
||||
const titleEl = thing.querySelector("a.title");
|
||||
const authorEl = thing.querySelector("a.author");
|
||||
const selftext = thing.querySelector(".usertext-body .md");
|
||||
const subreddit = thing.getAttribute("data-subreddit") || "";
|
||||
return {
|
||||
reddit_post_id: postId,
|
||||
title: cleanText(titleEl && titleEl.textContent),
|
||||
body: selftext ? selftext.innerText.trim() : "",
|
||||
author_id: thing.getAttribute("data-author-fullname") || "",
|
||||
author_name: cleanText((authorEl && authorEl.textContent) || thing.getAttribute("data-author") || "[deleted]"),
|
||||
permalink: absoluteUrl(thing.getAttribute("data-permalink") || (titleEl && titleEl.getAttribute("href"))),
|
||||
subreddit,
|
||||
element: thing,
|
||||
};
|
||||
}
|
||||
|
||||
function scrapeOldComments(post) {
|
||||
const nodes = document.querySelectorAll(".commentarea .thing.comment");
|
||||
const comments = [];
|
||||
nodes.forEach((node) => {
|
||||
if (!node.getBoundingClientRect) return;
|
||||
const id = normalizeThingId(node.getAttribute("data-fullname") || node.id.replace(/^thing_/, ""), "comment");
|
||||
if (!id) return;
|
||||
const authorEl = node.querySelector(":scope > .entry a.author, :scope > .entry .tagline a.author");
|
||||
const bodyEl = node.querySelector(":scope > .entry .usertext-body .md, :scope > .entry .usertext-body");
|
||||
const permalinkEl = node.querySelector(':scope > .entry a[data-event-action="permalink"], :scope > .entry a.bylink');
|
||||
const shotTarget = node.querySelector(":scope > .entry") || node;
|
||||
comments.push({
|
||||
reddit_comment_id: id,
|
||||
body: bodyEl ? bodyEl.innerText.trim() : "",
|
||||
author_id: node.getAttribute("data-author-fullname") || "",
|
||||
author_name: cleanText((authorEl && authorEl.textContent) || node.getAttribute("data-author") || "[deleted]"),
|
||||
reddit_post_id: post ? post.reddit_post_id : "",
|
||||
post_title: post ? post.title : "",
|
||||
post_body: post ? post.body : "",
|
||||
post_author_id: post ? post.author_id : "",
|
||||
post_author_name: post ? post.author_name : "",
|
||||
post_permalink: post ? post.permalink : "",
|
||||
subreddit: post ? post.subreddit : (node.getAttribute("data-subreddit") || ""),
|
||||
permalink: absoluteUrl(permalinkEl && permalinkEl.getAttribute("href")),
|
||||
element: shotTarget,
|
||||
});
|
||||
});
|
||||
return comments;
|
||||
}
|
||||
|
||||
function scrapeNewPost() {
|
||||
const post =
|
||||
queryAllDeep(document, "shreddit-post")[0] ||
|
||||
document.querySelector('[id^="t3_"]') ||
|
||||
document.querySelector('div[data-testid="post-container"]');
|
||||
if (!post) {
|
||||
const match = location.pathname.match(/\/comments\/([a-z0-9]+)\//i);
|
||||
if (!match) return null;
|
||||
return {
|
||||
reddit_post_id: normalizeThingId(match[1], "post"),
|
||||
title: cleanText(document.title.replace(/\s*:\s*r\/.*$/, "")),
|
||||
body: "",
|
||||
author_id: "",
|
||||
author_name: "",
|
||||
permalink: location.href.split("?")[0],
|
||||
subreddit: (location.pathname.match(/\/r\/([^/]+)/) || [])[1] || "",
|
||||
element: document.querySelector("main") || document.body,
|
||||
};
|
||||
}
|
||||
|
||||
const postId =
|
||||
normalizeThingId(post.getAttribute("id") || post.getAttribute("thingid") || post.getAttribute("post-id"), "post") ||
|
||||
(() => {
|
||||
const m = location.pathname.match(/\/comments\/([a-z0-9]+)\//i);
|
||||
return m ? normalizeThingId(m[1], "post") : "";
|
||||
})();
|
||||
|
||||
const title =
|
||||
post.getAttribute("post-title") ||
|
||||
cleanText((post.querySelector('h1, [slot="title"], a[slot="title"]') || {}).textContent) ||
|
||||
cleanText(document.title);
|
||||
|
||||
const authorName =
|
||||
post.getAttribute("author") ||
|
||||
cleanText((post.querySelector('a[href*="/user/"], [slot="authorName"]') || {}).textContent) ||
|
||||
"";
|
||||
|
||||
const bodyEl =
|
||||
post.querySelector('[slot="text-body"], .text-container, [data-testid="post-content"]') ||
|
||||
post.querySelector("p");
|
||||
|
||||
const subreddit =
|
||||
post.getAttribute("subreddit-name") ||
|
||||
(location.pathname.match(/\/r\/([^/]+)/) || [])[1] ||
|
||||
"";
|
||||
|
||||
const permalink =
|
||||
absoluteUrl(post.getAttribute("permalink") || post.getAttribute("content-href")) ||
|
||||
location.href.split("?")[0];
|
||||
|
||||
return {
|
||||
reddit_post_id: postId,
|
||||
title,
|
||||
body: bodyEl ? bodyEl.innerText.trim() : "",
|
||||
author_id: post.getAttribute("author-id") || "",
|
||||
author_name: authorName || "[deleted]",
|
||||
permalink,
|
||||
subreddit,
|
||||
element: post,
|
||||
};
|
||||
}
|
||||
|
||||
function findCommentElement(commentId) {
|
||||
const bare = String(commentId || "").replace(/^t1_/, "");
|
||||
const full = String(commentId || "").startsWith("t1_") ? String(commentId) : `t1_${bare}`;
|
||||
const want = new Set([full, `t1_${bare}`, bare, `thing_${full}`, `thing_${bare}`].filter(Boolean));
|
||||
|
||||
// Classic old Reddit markup (www or old.reddit.com).
|
||||
const classic =
|
||||
document.querySelector(`.thing.comment[data-fullname="${full}"]`) ||
|
||||
document.querySelector(`.thing.comment[data-fullname="${bare}"]`) ||
|
||||
document.querySelector(`#thing_${full}`) ||
|
||||
document.querySelector(`#thing_${bare}`);
|
||||
if (classic) {
|
||||
return classic.querySelector(":scope > .entry") || classic;
|
||||
}
|
||||
|
||||
const selectors = [
|
||||
`shreddit-comment[thingid="${full}"]`,
|
||||
`shreddit-comment[thingid="t1_${bare}"]`,
|
||||
`shreddit-comment[id="${full}"]`,
|
||||
`[thingid="${full}"]`,
|
||||
`[thingid="t1_${bare}"]`,
|
||||
`[comment-id="${full}"]`,
|
||||
`[data-fullname="${full}"]`,
|
||||
`[data-fullname="${bare}"]`,
|
||||
`[id="t1_${bare}"]`,
|
||||
`[id="${bare}"]`,
|
||||
`[id="thing_${full}"]`,
|
||||
`[id="thing_${bare}"]`,
|
||||
];
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
const hit = queryAllDeep(document, selector)[0];
|
||||
if (hit) {
|
||||
if (hit.classList && hit.classList.contains("comment")) {
|
||||
return hit.querySelector(":scope > .entry") || hit;
|
||||
}
|
||||
return preferShotTarget(hit);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
let found = null;
|
||||
walkDom(document.documentElement, (el) => {
|
||||
if (found || !el.getAttribute) return;
|
||||
const candidates = [
|
||||
el.getAttribute("thingid"),
|
||||
el.getAttribute("comment-id"),
|
||||
el.getAttribute("data-fullname"),
|
||||
el.getAttribute("id"),
|
||||
el.getAttribute("data-comment-id"),
|
||||
el.getAttribute("name"),
|
||||
];
|
||||
for (const raw of candidates) {
|
||||
if (!raw) continue;
|
||||
const cleaned = String(raw).replace(/^thing_/, "");
|
||||
const normalized = normalizeThingId(cleaned, "comment");
|
||||
const stripped = cleaned.replace(/^t1_/, "");
|
||||
if (want.has(raw) || want.has(cleaned) || want.has(normalized) || want.has(stripped)) {
|
||||
if (el.classList && el.classList.contains("comment")) {
|
||||
found = el.querySelector(":scope > .entry") || el;
|
||||
} else {
|
||||
found = preferShotTarget(el);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (el.tagName === "A") {
|
||||
const href = el.getAttribute("href") || "";
|
||||
if (href.includes(`/${bare}`) || href.includes(full) || href.includes(`t1_${bare}`)) {
|
||||
found = preferShotTarget(el);
|
||||
}
|
||||
}
|
||||
});
|
||||
return found;
|
||||
}
|
||||
|
||||
function preferShotTarget(el) {
|
||||
if (!el) return null;
|
||||
// Prefer a comment-sized container over a tiny permalink icon/link.
|
||||
let best = el;
|
||||
let node = el;
|
||||
for (let depth = 0; depth < 10 && node; depth += 1) {
|
||||
const rect = node.getBoundingClientRect ? node.getBoundingClientRect() : null;
|
||||
if (rect && rect.height >= 48 && rect.width >= 120) {
|
||||
best = node;
|
||||
// Prefer shreddit-comment host when available.
|
||||
if ((node.tagName || "").toLowerCase() === "shreddit-comment") break;
|
||||
}
|
||||
node = node.parentElement;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function walkDom(root, visit) {
|
||||
const stack = [root];
|
||||
while (stack.length) {
|
||||
const node = stack.pop();
|
||||
if (!node) continue;
|
||||
visit(node);
|
||||
if (node.shadowRoot) stack.push(node.shadowRoot);
|
||||
const children = node.children;
|
||||
if (children) {
|
||||
for (let i = 0; i < children.length; i += 1) {
|
||||
stack.push(children[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function scrapeNewComments(post) {
|
||||
const selector = [
|
||||
"shreddit-comment",
|
||||
"[thingid^='t1_']",
|
||||
"[id^='t1_']",
|
||||
'div[data-testid="comment"]',
|
||||
"faceplate-tracker[noun='comment']",
|
||||
].join(", ");
|
||||
const nodes = queryAllDeep(document, selector);
|
||||
const comments = [];
|
||||
const seen = new Set();
|
||||
|
||||
nodes.forEach((node) => {
|
||||
let id =
|
||||
node.getAttribute("thingid") ||
|
||||
node.getAttribute("comment-id") ||
|
||||
node.getAttribute("id") ||
|
||||
"";
|
||||
// Some wrappers put the id on a child.
|
||||
if (!id || (!id.startsWith("t1_") && !/^t1_/.test(id))) {
|
||||
const nested = node.querySelector("[thingid^='t1_'], [id^='t1_']");
|
||||
if (nested) {
|
||||
id = nested.getAttribute("thingid") || nested.id || id;
|
||||
}
|
||||
}
|
||||
id = normalizeThingId(id, "comment");
|
||||
if (!id || !id.startsWith("t1_") || seen.has(id)) return;
|
||||
seen.add(id);
|
||||
|
||||
const authorName =
|
||||
node.getAttribute("author") ||
|
||||
cleanText((node.querySelector('a[href*="/user/"]') || {}).textContent) ||
|
||||
"[deleted]";
|
||||
|
||||
const bodyEl =
|
||||
node.querySelector('[slot="comment"], .md, [id$="-post-rtjson-content"], [data-testid="comment"]') ||
|
||||
node.querySelector("p");
|
||||
|
||||
const permalinkAttr = node.getAttribute("permalink");
|
||||
const permalinkLink = node.querySelector('a[href*="/comments/"]');
|
||||
const permalink =
|
||||
absoluteUrl(permalinkAttr) ||
|
||||
absoluteUrl(permalinkLink && permalinkLink.getAttribute("href")) ||
|
||||
"";
|
||||
|
||||
comments.push({
|
||||
reddit_comment_id: id,
|
||||
body: bodyEl ? bodyEl.innerText.trim() : cleanText(node.innerText).slice(0, 5000),
|
||||
author_id: node.getAttribute("author-id") || "",
|
||||
author_name: authorName,
|
||||
reddit_post_id: post ? post.reddit_post_id : "",
|
||||
post_title: post ? post.title : "",
|
||||
post_body: post ? post.body : "",
|
||||
post_author_id: post ? post.author_id : "",
|
||||
post_author_name: post ? post.author_name : "",
|
||||
post_permalink: post ? post.permalink : "",
|
||||
subreddit: post ? post.subreddit : "",
|
||||
permalink,
|
||||
element: node,
|
||||
});
|
||||
});
|
||||
return comments;
|
||||
}
|
||||
|
||||
function walkListingComments(children, out) {
|
||||
if (!Array.isArray(children)) return;
|
||||
children.forEach((child) => {
|
||||
if (!child || child.kind !== "t1" || !child.data) return;
|
||||
const d = child.data;
|
||||
const id = normalizeThingId(d.name || d.id, "comment");
|
||||
if (!id) return;
|
||||
out.push({
|
||||
reddit_comment_id: id,
|
||||
body: d.body || "",
|
||||
author_id: d.author_fullname || "",
|
||||
author_name: d.author || "[deleted]",
|
||||
permalink: absoluteUrl(d.permalink),
|
||||
subreddit: d.subreddit || "",
|
||||
});
|
||||
if (d.replies && d.replies.data && d.replies.data.children) {
|
||||
walkListingComments(d.replies.data.children, out);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let jsonCache = { postId: "", at: 0, comments: [], error: "" };
|
||||
|
||||
async function fetchThreadCommentsJson(post) {
|
||||
if (!post || !post.reddit_post_id) return [];
|
||||
const now = Date.now();
|
||||
if (jsonCache.postId === post.reddit_post_id && now - jsonCache.at < 15000) {
|
||||
if (jsonCache.error) throw new Error(jsonCache.error);
|
||||
return jsonCache.comments.map((c) => ({
|
||||
...c,
|
||||
element: findCommentElement(c.reddit_comment_id),
|
||||
}));
|
||||
}
|
||||
const bare = post.reddit_post_id.replace(/^t3_/, "");
|
||||
try {
|
||||
const response = await chrome.runtime.sendMessage({
|
||||
type: "rcs.redditJson",
|
||||
path: `/comments/${bare}.json?limit=500&raw_json=1`,
|
||||
});
|
||||
if (!response || !response.ok) {
|
||||
throw new Error((response && response.error) || "reddit json failed");
|
||||
}
|
||||
const listing = response.data;
|
||||
const comments = [];
|
||||
if (Array.isArray(listing) && listing[1] && listing[1].data) {
|
||||
walkListingComments(listing[1].data.children, comments);
|
||||
}
|
||||
const mapped = comments.map((c) => ({
|
||||
...c,
|
||||
reddit_post_id: post.reddit_post_id,
|
||||
post_title: post.title,
|
||||
post_body: post.body,
|
||||
post_author_id: post.author_id,
|
||||
post_author_name: post.author_name,
|
||||
post_permalink: post.permalink,
|
||||
subreddit: c.subreddit || post.subreddit,
|
||||
}));
|
||||
jsonCache = { postId: post.reddit_post_id, at: now, comments: mapped, error: "" };
|
||||
return mapped.map((c) => ({
|
||||
...c,
|
||||
element: findCommentElement(c.reddit_comment_id),
|
||||
}));
|
||||
} catch (err) {
|
||||
jsonCache = {
|
||||
postId: post.reddit_post_id,
|
||||
at: now,
|
||||
comments: [],
|
||||
error: String(err && err.message ? err.message : err),
|
||||
};
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function mergeComments(domComments, jsonComments) {
|
||||
const byId = new Map();
|
||||
domComments.forEach((c) => byId.set(c.reddit_comment_id, c));
|
||||
jsonComments.forEach((c) => {
|
||||
const existing = byId.get(c.reddit_comment_id);
|
||||
if (!existing) {
|
||||
byId.set(c.reddit_comment_id, c);
|
||||
return;
|
||||
}
|
||||
// Prefer non-empty body / author from JSON when DOM is empty.
|
||||
if (!existing.body && c.body) existing.body = c.body;
|
||||
if ((!existing.author_name || existing.author_name === "[deleted]") && c.author_name) {
|
||||
existing.author_name = c.author_name;
|
||||
}
|
||||
if (!existing.permalink && c.permalink) existing.permalink = c.permalink;
|
||||
if (!existing.element && c.element) existing.element = c.element;
|
||||
});
|
||||
return [...byId.values()];
|
||||
}
|
||||
|
||||
function scrapePage() {
|
||||
const classic = isOldReddit();
|
||||
const post = classic ? scrapeOldPost() : scrapeNewPost();
|
||||
const comments = classic ? scrapeOldComments(post) : scrapeNewComments(post);
|
||||
return { post, comments, isOld: classic, isShreddit: !classic && isShreddit() };
|
||||
}
|
||||
|
||||
async function scrapePageAsync() {
|
||||
const base = scrapePage();
|
||||
if (base.isOld) return base;
|
||||
try {
|
||||
const jsonComments = await fetchThreadCommentsJson(base.post);
|
||||
return {
|
||||
...base,
|
||||
comments: mergeComments(base.comments, jsonComments),
|
||||
jsonCount: jsonComments.length,
|
||||
domCount: base.comments.length,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
...base,
|
||||
jsonError: String(err && err.message ? err.message : err),
|
||||
jsonCount: 0,
|
||||
domCount: base.comments.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function isElementMostlyVisible(el) {
|
||||
if (!el) return false;
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width < 2 || rect.height < 2) return false;
|
||||
const vh = window.innerHeight || document.documentElement.clientHeight;
|
||||
return rect.bottom > 0 && rect.top < vh;
|
||||
}
|
||||
|
||||
global.RCSScrape = {
|
||||
scrapePage,
|
||||
scrapePageAsync,
|
||||
isElementMostlyVisible,
|
||||
isOldReddit,
|
||||
debugDomHints,
|
||||
findCommentElement,
|
||||
};
|
||||
})(typeof window !== "undefined" ? window : self);
|
||||
Reference in New Issue
Block a user