Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

feat(web): add support for http/https proxies

wnor543f83a9e899c74
diff --git a/apps/web/lib/shared/fetchTitleAndHeaders.ts b/apps/web/lib/shared/fetchTitleAndHeaders.ts
index fcfee319..96eb058d 100644
--- a/apps/web/lib/shared/fetchTitleAndHeaders.ts
+++ b/apps/web/lib/shared/fetchTitleAndHeaders.ts
@@ -1,5 +1,6 @@
import fetch from "node-fetch";
import https from "https";
+import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from "socks-proxy-agent";
export default async function fetchTitleAndHeaders(url: string) {
@@ -23,8 +24,10 @@ export default async function fetchTitleAndHeaders(url: string) {
proxy.password = process.env.PROXY_PASSWORD || "";
}
- // add socks5 proxy to fetchOpts
- fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
+ const proxyAgent = proxy.protocol.includes("http") ? HttpsProxyAgent : SocksProxyAgent;
+
+ // add socks5/http/https proxy to fetchOpts
+ fetchOpts = { agent: new proxyAgent(proxy.toString()) };
}
const responsePromise = fetch(url, fetchOpts);
diff --git a/apps/web/package.json b/apps/web/package.json
index 46ef4682..39eaf34d 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -50,6 +50,7 @@
"fuse.js": "^7.0.0",
"handlebars": "^4.7.8",
"himalaya": "^1.1.0",
+ "https-proxy-agent": "^7.0.6",
"i18next": "^23.11.5",
"jimp": "^0.22.10",
"jsdom": "^22.1.0",
diff --git a/apps/worker/lib/fetchHeaders.ts b/apps/worker/lib/fetchHeaders.ts
index f423661a..23416dc8 100644
--- a/apps/worker/lib/fetchHeaders.ts
+++ b/apps/worker/lib/fetchHeaders.ts
@@ -1,5 +1,6 @@
import fetch from "node-fetch";
import https from "https";
+import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from "socks-proxy-agent";
export default async function fetchHeaders(url: string) {
@@ -23,9 +24,11 @@ export default async function fetchHeaders(url: string) {
proxy.password = process.env.PROXY_PASSWORD || "";
}
+ const proxyAgent = proxy.protocol.includes("http") ? HttpsProxyAgent : SocksProxyAgent;
+
fetchOpts = {
method: "HEAD",
- agent: new SocksProxyAgent(proxy.toString()),
+ agent: new proxyAgent(proxy.toString()),
};
}
diff --git a/yarn.lock b/yarn.lock
index c93faad8..3460e4eb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4650,6 +4650,11 @@ agent-base@^7.0.2:
dependencies:
debug "^4.3.4"
+agent-base@^7.1.2:
+ version "7.1.3"
+ resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1"
+ integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==
+
aggregate-error@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a"
@@ -7807,6 +7812,14 @@ https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1:
agent-base "6"
debug "4"
+https-proxy-agent@^7.0.6:
+ version "7.0.6"
+ resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
+ integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==
+ dependencies:
+ agent-base "^7.1.2"
+ debug "4"
+
human-signals@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"