Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

Improved archiving.

Daniel1c9d626d087d
diff --git a/api/modules/getData.js b/api/modules/getData.js
index dcb65670..2cf047e7 100644
--- a/api/modules/getData.js
+++ b/api/modules/getData.js
@@ -5,7 +5,7 @@ const { screenshotDirectory, pdfDirectory } = require("../config.js");
module.exports = async (link, id) => {
const browser = await puppeteer.launch({
- args: ['--no-sandbox'],
+ args: ["--no-sandbox"],
timeout: 10000,
});
const page = await browser.newPage();
@@ -14,7 +14,14 @@ module.exports = async (link, id) => {
blocker.enableBlockingInPage(page);
});
- await page.goto(link, { waitUntil: "load", timeout: 0 });
+ await page.goto(link, { waitUntil: "load", timeout: 300000 });
+
+ await page.setViewport({
+ width: 1200,
+ height: 800,
+ });
+
+ await autoScroll(page);
await page.screenshot({
path: screenshotDirectory + "/" + id + ".png",
@@ -24,3 +31,24 @@ module.exports = async (link, id) => {
await browser.close();
};
+
+async function autoScroll(page) {
+ await page.evaluate(async () => {
+ await new Promise((resolve, reject) => {
+ let totalHeight = 0;
+ let distance = 100;
+ let timer = setInterval(() => {
+ let scrollHeight = document.body.scrollHeight;
+ window.scrollBy(0, distance);
+ totalHeight += distance;
+
+ if (totalHeight >= scrollHeight - window.innerHeight) {
+ clearInterval(timer);
+ resolve();
+ }
+ }, 100);
+ });
+
+ window.scrollTo(0,0);
+ });
+}