Squid
5a0e2e630b
Ship the Go/SQLite API and Web UI, Chrome/Brave capture addon, Docker Compose, Pangolin reverse-proxy support, and a user-crontab watchdog so the binary stays running without systemd.
480 lines
15 KiB
JavaScript
480 lines
15 KiB
JavaScript
// Auto-capture visible comments/posts with backend dedupe and queued screenshots.
|
|
// Backend I/O goes through lib/api.js → background service worker (never localhost fetch here).
|
|
|
|
(function () {
|
|
const processed = new Set(); // fully done (saved + screenshot, or blocked)
|
|
const needsScreenshot = new Set(); // saved text but still missing screenshot
|
|
const queue = [];
|
|
let busy = false;
|
|
let configured = false;
|
|
let observer = null;
|
|
let scanCount = 0;
|
|
let blockedUsers = new Set();
|
|
|
|
async function refreshBlockedUsers() {
|
|
const list = await rcsGetCachedBlocklist();
|
|
blockedUsers = new Set((list || []).map(rcsNormalizeUsername).filter(Boolean));
|
|
}
|
|
|
|
function isAuthorBlocked(authorName) {
|
|
const normalized = rcsNormalizeUsername(authorName);
|
|
return Boolean(normalized) && blockedUsers.has(normalized);
|
|
}
|
|
|
|
async function refreshConfig() {
|
|
const settings = await rcsGetSettings();
|
|
configured = Boolean(settings.backendUrl);
|
|
return configured;
|
|
}
|
|
|
|
function enqueue(job) {
|
|
queue.push(job);
|
|
pump();
|
|
}
|
|
|
|
async function pump() {
|
|
if (busy) return;
|
|
busy = true;
|
|
while (queue.length) {
|
|
const job = queue.shift();
|
|
try {
|
|
await job();
|
|
} catch (err) {
|
|
console.warn("[RCS] queue job failed:", err && err.message ? err.message : err);
|
|
}
|
|
await sleep(200);
|
|
}
|
|
busy = false;
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function resolveCommentElement(comment) {
|
|
if (comment && comment.element && comment.element.isConnected) {
|
|
return comment.element;
|
|
}
|
|
if (!window.RCSScrape || !comment || !comment.reddit_comment_id) return null;
|
|
return window.RCSScrape.findCommentElement(comment.reddit_comment_id);
|
|
}
|
|
|
|
function clampRectToViewport(rect) {
|
|
const vw = window.innerWidth || document.documentElement.clientWidth;
|
|
const vh = window.innerHeight || document.documentElement.clientHeight;
|
|
const left = Math.max(0, rect.left);
|
|
const top = Math.max(0, rect.top);
|
|
const right = Math.min(vw, rect.right);
|
|
const bottom = Math.min(vh, rect.bottom);
|
|
return {
|
|
x: left,
|
|
y: top,
|
|
width: Math.max(0, right - left),
|
|
height: Math.max(0, bottom - top),
|
|
};
|
|
}
|
|
|
|
function isElementInViewportEnough(element) {
|
|
if (!element || !element.getBoundingClientRect) return false;
|
|
const rect = element.getBoundingClientRect();
|
|
const vh = window.innerHeight || document.documentElement.clientHeight;
|
|
const vw = window.innerWidth || document.documentElement.clientWidth;
|
|
if (rect.width < 40 || rect.height < 20) return false;
|
|
const visibleH = Math.min(rect.bottom, vh) - Math.max(rect.top, 0);
|
|
const visibleW = Math.min(rect.right, vw) - Math.max(rect.left, 0);
|
|
return visibleH >= Math.min(rect.height, 40) && visibleW >= Math.min(rect.width, 40);
|
|
}
|
|
|
|
// Never scrolls the page — only crops if the element is already on-screen.
|
|
async function captureElementScreenshot(element) {
|
|
if (!element) return "";
|
|
if (!isElementInViewportEnough(element)) return "";
|
|
|
|
const fresh = element.getBoundingClientRect();
|
|
const clipped = clampRectToViewport(fresh);
|
|
if (clipped.width < 10 || clipped.height < 10) return "";
|
|
|
|
const response = await chrome.runtime.sendMessage({
|
|
type: "rcs.captureCrop",
|
|
rect: clipped,
|
|
dpr: window.devicePixelRatio || 1,
|
|
});
|
|
|
|
if (!response || !response.ok) {
|
|
throw new Error((response && response.error) || "screenshot failed");
|
|
}
|
|
return response.dataUrl || "";
|
|
}
|
|
|
|
// When Reddit hides comments in closed shadow DOM, render a card ourselves.
|
|
async function renderSyntheticScreenshot(comment) {
|
|
const width = 720;
|
|
const pad = 20;
|
|
const author = `u/${comment.author_name || "[deleted]"}`;
|
|
const subreddit = String(comment.subreddit || "").replace(/^r\//i, "").trim();
|
|
const byline = subreddit ? `${author} · r/${subreddit}` : author;
|
|
const body = String(comment.body || "").trim() || "(empty comment)";
|
|
const meta = comment.permalink || comment.reddit_comment_id || "";
|
|
|
|
const canvas = document.createElement("canvas");
|
|
const ctx = canvas.getContext("2d");
|
|
const dpr = Math.min(window.devicePixelRatio || 1, 2);
|
|
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
|
|
|
function wrap(text, maxWidth) {
|
|
const words = text.split(/\s+/);
|
|
const lines = [];
|
|
let line = "";
|
|
words.forEach((word) => {
|
|
const test = line ? `${line} ${word}` : word;
|
|
if (ctx.measureText(test).width > maxWidth && line) {
|
|
lines.push(line);
|
|
line = word;
|
|
} else {
|
|
line = test;
|
|
}
|
|
});
|
|
if (line) lines.push(line);
|
|
// Hard-wrap very long tokens
|
|
return lines.flatMap((l) => {
|
|
if (ctx.measureText(l).width <= maxWidth) return [l];
|
|
const out = [];
|
|
let cur = "";
|
|
for (const ch of l) {
|
|
const t = cur + ch;
|
|
if (ctx.measureText(t).width > maxWidth && cur) {
|
|
out.push(cur);
|
|
cur = ch;
|
|
} else {
|
|
cur = t;
|
|
}
|
|
}
|
|
if (cur) out.push(cur);
|
|
return out;
|
|
});
|
|
}
|
|
|
|
const maxText = width - pad * 2;
|
|
ctx.font = "bold 15px system-ui, Segoe UI, sans-serif";
|
|
const bylineLines = wrap(byline, maxText);
|
|
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
|
const bodyLines = wrap(body.slice(0, 4000), maxText);
|
|
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
|
const metaLines = wrap(meta, maxText);
|
|
|
|
const lineH = 22;
|
|
const height = pad * 2 + 28 + bylineLines.length * lineH + 8 + bodyLines.length * lineH + 12 + metaLines.length * 16 + 24;
|
|
canvas.width = Math.floor(width * dpr);
|
|
canvas.height = Math.floor(height * dpr);
|
|
ctx.scale(dpr, dpr);
|
|
|
|
// Card background
|
|
ctx.fillStyle = "#1a1a1b";
|
|
ctx.fillRect(0, 0, width, height);
|
|
ctx.strokeStyle = "#343536";
|
|
ctx.lineWidth = 2;
|
|
ctx.strokeRect(1, 1, width - 2, height - 2);
|
|
|
|
let y = pad + 8;
|
|
ctx.fillStyle = "#818384";
|
|
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
|
ctx.fillText("RCS comment preview", pad, y);
|
|
y += 24;
|
|
|
|
ctx.fillStyle = "#4fbcff";
|
|
ctx.font = "bold 15px system-ui, Segoe UI, sans-serif";
|
|
bylineLines.forEach((line) => {
|
|
// Draw author in blue, subreddit segment in muted if present on same line.
|
|
if (subreddit && line.includes(" · r/")) {
|
|
const parts = line.split(" · ");
|
|
let x = pad;
|
|
ctx.fillStyle = "#4fbcff";
|
|
ctx.fillText(parts[0] || "", x, y);
|
|
x += ctx.measureText(parts[0] || "").width;
|
|
if (parts[1]) {
|
|
ctx.fillStyle = "#818384";
|
|
const sep = " · ";
|
|
ctx.fillText(sep, x, y);
|
|
x += ctx.measureText(sep).width;
|
|
ctx.fillStyle = "#ff7a45";
|
|
ctx.fillText(parts[1], x, y);
|
|
}
|
|
} else {
|
|
ctx.fillStyle = "#4fbcff";
|
|
ctx.fillText(line, pad, y);
|
|
}
|
|
y += lineH;
|
|
});
|
|
y += 6;
|
|
|
|
ctx.fillStyle = "#d7dadc";
|
|
ctx.font = "16px system-ui, Segoe UI, sans-serif";
|
|
bodyLines.forEach((line) => {
|
|
ctx.fillText(line, pad, y);
|
|
y += lineH;
|
|
});
|
|
y += 10;
|
|
|
|
ctx.fillStyle = "#818384";
|
|
ctx.font = "12px system-ui, Segoe UI, sans-serif";
|
|
metaLines.forEach((line) => {
|
|
ctx.fillText(line, pad, y);
|
|
y += 16;
|
|
});
|
|
|
|
return canvas.toDataURL("image/png");
|
|
}
|
|
|
|
const pendingLogAt = new Map();
|
|
|
|
async function savePostIfNeeded(post) {
|
|
if (!post || !post.reddit_post_id || processed.has("post:" + post.reddit_post_id)) return;
|
|
processed.add("post:" + post.reddit_post_id);
|
|
const exists = await rcsPostExists(post.reddit_post_id);
|
|
if (exists) {
|
|
console.debug("[RCS] post already saved:", post.reddit_post_id);
|
|
return;
|
|
}
|
|
await rcsSavePost({
|
|
reddit_post_id: post.reddit_post_id,
|
|
title: post.title,
|
|
body: post.body,
|
|
author_id: post.author_id,
|
|
author_name: post.author_name,
|
|
permalink: post.permalink,
|
|
subreddit: post.subreddit,
|
|
});
|
|
console.info("[RCS] saved post:", post.reddit_post_id, post.title);
|
|
}
|
|
|
|
async function captureAndMaybeBackfill(comment, status) {
|
|
const element = resolveCommentElement(comment);
|
|
comment.element = element;
|
|
|
|
let screenshot = "";
|
|
if (element && isElementInViewportEnough(element)) {
|
|
try {
|
|
screenshot = await captureElementScreenshot(element);
|
|
if (screenshot) {
|
|
console.info("[RCS] DOM screenshot captured (no scroll):", comment.reddit_comment_id);
|
|
}
|
|
} catch (err) {
|
|
console.warn("[RCS] DOM screenshot failed:", comment.reddit_comment_id, err && err.message ? err.message : err);
|
|
}
|
|
}
|
|
|
|
if (!screenshot) {
|
|
// Off-screen / closed-shadow: render without moving the page.
|
|
try {
|
|
screenshot = await renderSyntheticScreenshot(comment);
|
|
console.info("[RCS] synthetic screenshot rendered:", comment.reddit_comment_id);
|
|
} catch (err) {
|
|
needsScreenshot.add(comment.reddit_comment_id);
|
|
const now = Date.now();
|
|
const last = pendingLogAt.get(comment.reddit_comment_id) || 0;
|
|
if (now - last > 10000) {
|
|
pendingLogAt.set(comment.reddit_comment_id, now);
|
|
console.info("[RCS] screenshot pending (no DOM yet):", comment.reddit_comment_id);
|
|
}
|
|
return "";
|
|
}
|
|
}
|
|
|
|
if (!screenshot) {
|
|
needsScreenshot.add(comment.reddit_comment_id);
|
|
return "";
|
|
}
|
|
|
|
if (status && status.exists && !status.has_screenshot) {
|
|
await rcsAttachScreenshot(comment.reddit_comment_id, screenshot);
|
|
console.info("[RCS] backfilled screenshot:", comment.reddit_comment_id);
|
|
}
|
|
|
|
needsScreenshot.delete(comment.reddit_comment_id);
|
|
pendingLogAt.delete(comment.reddit_comment_id);
|
|
return screenshot;
|
|
}
|
|
|
|
async function saveCommentIfNeeded(comment) {
|
|
if (!comment || !comment.reddit_comment_id) return;
|
|
if (isAuthorBlocked(comment.author_name)) {
|
|
console.info("[RCS] skipped blocked user:", comment.author_name, comment.reddit_comment_id);
|
|
processed.add("comment:" + comment.reddit_comment_id);
|
|
return;
|
|
}
|
|
|
|
const id = comment.reddit_comment_id;
|
|
const doneKey = "comment:" + id;
|
|
if (processed.has(doneKey) && !needsScreenshot.has(id)) return;
|
|
|
|
const status = await rcsCommentExists(id);
|
|
|
|
if (status.exists && status.has_screenshot) {
|
|
processed.add(doneKey);
|
|
needsScreenshot.delete(id);
|
|
console.debug("[RCS] comment complete:", id);
|
|
return;
|
|
}
|
|
|
|
if (status.exists && !status.has_screenshot) {
|
|
const shot = await captureAndMaybeBackfill(comment, status);
|
|
if (shot) {
|
|
processed.add(doneKey);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// New comment
|
|
const screenshot = await captureAndMaybeBackfill(comment, status);
|
|
const result = await rcsSaveComment({
|
|
reddit_comment_id: id,
|
|
body: comment.body,
|
|
author_id: comment.author_id,
|
|
author_name: comment.author_name,
|
|
reddit_post_id: comment.reddit_post_id,
|
|
post_title: comment.post_title,
|
|
post_body: comment.post_body,
|
|
post_author_id: comment.post_author_id,
|
|
post_author_name: comment.post_author_name,
|
|
post_permalink: comment.post_permalink,
|
|
subreddit: comment.subreddit,
|
|
permalink: comment.permalink,
|
|
screenshot_base64: screenshot,
|
|
});
|
|
|
|
if (screenshot) {
|
|
processed.add(doneKey);
|
|
needsScreenshot.delete(id);
|
|
} else {
|
|
needsScreenshot.add(id);
|
|
}
|
|
|
|
console.info(
|
|
"[RCS] saved comment:",
|
|
id,
|
|
"by",
|
|
comment.author_name,
|
|
result && result.saved ? "(new)" : "(exists)",
|
|
screenshot ? "+screenshot" : "no-screenshot-yet"
|
|
);
|
|
}
|
|
|
|
function shouldProcessComment(comment) {
|
|
if (!comment || !comment.reddit_comment_id) return false;
|
|
const id = comment.reddit_comment_id;
|
|
if (processed.has("comment:" + id) && !needsScreenshot.has(id)) return false;
|
|
|
|
const element = resolveCommentElement(comment);
|
|
comment.element = element;
|
|
|
|
// Prefer comments that are on-screen (or have a node we can scroll to for backfill).
|
|
if (needsScreenshot.has(id) && element) return true;
|
|
if (window.RCSScrape.isElementMostlyVisible(element)) return true;
|
|
// JSON-only with no DOM: still allow text save once.
|
|
if (!element && !processed.has("comment:" + id)) return true;
|
|
return false;
|
|
}
|
|
|
|
async function scan() {
|
|
if (!configured || !window.RCSScrape) return;
|
|
scanCount += 1;
|
|
const page = await window.RCSScrape.scrapePageAsync();
|
|
const { post, comments, isOld } = page;
|
|
|
|
// Re-attach DOM nodes for pending screenshot ids.
|
|
comments.forEach((c) => {
|
|
if (!c.element) c.element = resolveCommentElement(c);
|
|
});
|
|
|
|
const candidates = comments.filter(shouldProcessComment);
|
|
if (scanCount <= 5 || scanCount % 20 === 0 || comments.length === 0) {
|
|
console.info(
|
|
"[RCS] scan#" + scanCount,
|
|
isOld ? "classic-ui" : page.isShreddit ? "shreddit" : "www.reddit",
|
|
"post=",
|
|
post && post.reddit_post_id,
|
|
"dom=",
|
|
page.domCount != null ? page.domCount : "?",
|
|
"json=",
|
|
page.jsonCount != null ? page.jsonCount : "n/a",
|
|
"merged=",
|
|
comments.length,
|
|
"queue=",
|
|
candidates.length,
|
|
"needShot=",
|
|
needsScreenshot.size,
|
|
page.jsonError ? "jsonError=" + page.jsonError : ""
|
|
);
|
|
if (comments.length === 0 && !isOld) {
|
|
console.info("[RCS] DOM hints:", window.RCSScrape.debugDomHints());
|
|
}
|
|
}
|
|
if (post) {
|
|
enqueue(() => savePostIfNeeded(post));
|
|
}
|
|
candidates.forEach((comment) => {
|
|
enqueue(() => saveCommentIfNeeded(comment));
|
|
});
|
|
}
|
|
|
|
function startObserver() {
|
|
if (observer) return;
|
|
let timer = null;
|
|
observer = new MutationObserver(() => {
|
|
clearTimeout(timer);
|
|
timer = setTimeout(scan, 400);
|
|
});
|
|
observer.observe(document.documentElement, { childList: true, subtree: true });
|
|
window.addEventListener("scroll", () => {
|
|
clearTimeout(timer);
|
|
timer = setTimeout(scan, 400);
|
|
}, { passive: true });
|
|
}
|
|
|
|
async function boot() {
|
|
await refreshConfig();
|
|
if (!configured) {
|
|
console.info("[RCS] backend URL not set — open the extension popup to configure.");
|
|
return;
|
|
}
|
|
console.info("[RCS] configured; capturing via background API proxy (no page→localhost fetch).");
|
|
try {
|
|
await rcsHealth();
|
|
console.info("[RCS] backend health OK");
|
|
} catch (err) {
|
|
console.warn("[RCS] backend health failed:", err && err.message ? err.message : err);
|
|
}
|
|
try {
|
|
const list = await rcsPullBlocklist();
|
|
blockedUsers = new Set(list.map(rcsNormalizeUsername));
|
|
console.info("[RCS] blocklist synced:", list.length, "users");
|
|
} catch (err) {
|
|
await refreshBlockedUsers();
|
|
console.warn("[RCS] blocklist pull failed, using cache:", err && err.message ? err.message : err);
|
|
}
|
|
startObserver();
|
|
scan();
|
|
}
|
|
|
|
chrome.storage.onChanged.addListener((changes, area) => {
|
|
if (area !== "sync") return;
|
|
if (changes.blockedUsernames) {
|
|
refreshBlockedUsers();
|
|
}
|
|
if (changes.backendUrl || changes.apiKey) {
|
|
refreshConfig().then(() => {
|
|
if (configured) {
|
|
startObserver();
|
|
scan();
|
|
}
|
|
});
|
|
}
|
|
});
|
|
|
|
if (document.readyState === "loading") {
|
|
document.addEventListener("DOMContentLoaded", boot);
|
|
} else {
|
|
boot();
|
|
}
|
|
})();
|