Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

fix(Docs): Remove raw GitHub endpoint calls and move ot Octokit. Create common util for GitHub files. (#44079)

Jeremias Menichellic0e60adf3dec
diff --git a/apps/docs/app/guides/ai/python/[slug]/page.tsx b/apps/docs/app/guides/ai/python/[slug]/page.tsx
index b65b95f86a1..e31bc6b1595 100644
--- a/apps/docs/app/guides/ai/python/[slug]/page.tsx
+++ b/apps/docs/app/guides/ai/python/[slug]/page.tsx
@@ -4,7 +4,7 @@ import rehypeSlug from 'rehype-slug'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
import { genGuideMeta, removeRedundantH1 } from '~/features/docs/GuidesMdx.utils'
-import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
+import { getGitHubFileContents } from '~/lib/octokit'
import { UrlTransformFunction, linkTransform } from '~/lib/mdx/plugins/rehypeLinkTransform'
import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
@@ -83,23 +83,9 @@ const getContent = async ({ slug }: Params) => {
const editLink = newEditLink(`${org}/${repo}/blob/${branch}/${docsDir}/${remoteFile}`)
- let response: Response
- try {
- response = await fetchRevalidatePerDay(
- `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${docsDir}/${remoteFile}`
- )
- } catch (err) {
- throw new Error(`Failed to fetch Python vecs docs from GitHub (network error): ${err}`)
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch Python vecs docs from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- let content = await response.text()
- content = removeRedundantH1(content)
+ const content = removeRedundantH1(
+ await getGitHubFileContents({ org, repo, path: `${docsDir}/${remoteFile}`, branch })
+ )
return {
pathname: `/guides/ai/python/${slug}` satisfies `/${string}`,
diff --git a/apps/docs/app/guides/database/database-advisors/page.tsx b/apps/docs/app/guides/database/database-advisors/page.tsx
index aedf3e3396b..630c7c3ee59 100644
--- a/apps/docs/app/guides/database/database-advisors/page.tsx
+++ b/apps/docs/app/guides/database/database-advisors/page.tsx
@@ -1,4 +1,3 @@
-import { Octokit } from '@octokit/core'
import { capitalize } from 'lodash-es'
import rehypeSlug from 'rehype-slug'
import { Heading } from 'ui'
@@ -7,7 +6,7 @@ import { Admonition } from 'ui-patterns'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
import { genGuideMeta } from '~/features/docs/GuidesMdx.utils'
import { MDXRemoteBase } from '~/features/docs/MdxBase'
-import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
+import { getGitHubFileContents, octokit } from '~/lib/octokit'
import { TabPanel, Tabs } from '~/features/ui/Tabs'
import { UrlTransformFunction, linkTransform } from '~/lib/mdx/plugins/rehypeLinkTransform'
import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
@@ -144,24 +143,15 @@ const urlTransform: (lints: Array<{ path: string }>) => UrlTransformFunction = (
* Fetch lint remediation Markdown from external repo
*/
const getLints = async () => {
- const octokit = new Octokit({ request: { fetch: fetchRevalidatePerDay } })
-
- let response: Awaited<ReturnType<typeof octokit.request>>
- try {
- response = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', {
- owner: org,
- repo: repo,
- path: docsDir,
- ref: branch,
- headers: {
- 'X-GitHub-Api-Version': '2022-11-28',
- },
- })
- } catch (err) {
- throw new Error(
- `Failed to fetch database advisors lint list from GitHub (network error): ${err}`
- )
- }
+ const response = await octokit().request('GET /repos/{owner}/{repo}/contents/{path}', {
+ owner: org,
+ repo: repo,
+ path: docsDir,
+ ref: branch,
+ headers: {
+ 'X-GitHub-Api-Version': '2022-11-28',
+ },
+ })
if (response.status >= 400) {
throw new Error(
@@ -179,24 +169,7 @@ const getLints = async () => {
const lints = await Promise.all(
lintsList.map(async ({ path }) => {
- let fileResponse: Response
- try {
- fileResponse = await fetchRevalidatePerDay(
- `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${path}`
- )
- } catch (err) {
- throw new Error(
- `Failed to fetch database advisors lint file ${path} from GitHub (network error): ${err}`
- )
- }
-
- if (response.status >= 400) {
- throw new Error(
- `Failed to fetch ${org}/${repo}/${branch}/${path} docs from GitHub: ${response.status}`
- )
- }
-
- const content = await fileResponse.text()
+ const content = await getGitHubFileContents({ org, repo, path, branch })
return {
path: getBasename(path),
diff --git a/apps/docs/app/guides/deployment/ci/[slug]/page.tsx b/apps/docs/app/guides/deployment/ci/[slug]/page.tsx
index 9876928553e..76152dc0aa1 100644
--- a/apps/docs/app/guides/deployment/ci/[slug]/page.tsx
+++ b/apps/docs/app/guides/deployment/ci/[slug]/page.tsx
@@ -4,7 +4,7 @@ import rehypeSlug from 'rehype-slug'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
import { genGuideMeta, removeRedundantH1 } from '~/features/docs/GuidesMdx.utils'
-import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
+import { getGitHubFileContents } from '~/lib/octokit'
import { UrlTransformFunction, linkTransform } from '~/lib/mdx/plugins/rehypeLinkTransform'
import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
@@ -82,22 +82,9 @@ const getContent = async ({ slug }: Params) => {
const editLink = newEditLink(`${org}/${repo}/blob/${branch}/${docsDir}/${remoteFile}`)
- let response: Response
- try {
- response = await fetchRevalidatePerDay(
- `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${docsDir}/${remoteFile}`
- )
- } catch (err) {
- throw new Error(`Failed to fetch CI docs from GitHub (network error): ${err}`)
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch CI docs from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- const content = removeRedundantH1(await response.text())
+ const content = removeRedundantH1(
+ await getGitHubFileContents({ org, repo, path: `${docsDir}/${remoteFile}`, branch })
+ )
return {
pathname: `/guides/cli/github-action/${slug}` satisfies `/${string}`,
diff --git a/apps/docs/app/guides/deployment/terraform/[[...slug]]/page.tsx b/apps/docs/app/guides/deployment/terraform/[[...slug]]/page.tsx
index 8fd187bfeb2..c97957170c2 100644
--- a/apps/docs/app/guides/deployment/terraform/[[...slug]]/page.tsx
+++ b/apps/docs/app/guides/deployment/terraform/[[...slug]]/page.tsx
@@ -4,7 +4,7 @@ import rehypeSlug from 'rehype-slug'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
import { genGuideMeta, removeRedundantH1 } from '~/features/docs/GuidesMdx.utils'
-import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
+import { getGitHubFileContents } from '~/lib/octokit'
import { isValidGuideFrontmatter } from '~/lib/docs'
import { UrlTransformFunction, linkTransform } from '~/lib/mdx/plugins/rehypeLinkTransform'
import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
@@ -115,22 +115,12 @@ const getContent = async ({ slug }: Params) => {
`${terraformDocsOrg}/${terraformDocsRepo}/blob/${terraformDocsBranch}/${useRoot ? '' : `${terraformDocsDocsDir}/`}${remoteFile}`
)
- let response: Response
- try {
- response = await fetchRevalidatePerDay(
- `https://raw.githubusercontent.com/${terraformDocsOrg}/${terraformDocsRepo}/${terraformDocsBranch}/${useRoot ? '' : `${terraformDocsDocsDir}/`}${remoteFile}`
- )
- } catch (err) {
- throw new Error(`Failed to fetch Terraform docs from GitHub (network error): ${err}`)
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch Terraform docs from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- let rawContent = await response.text()
+ let rawContent = await getGitHubFileContents({
+ org: terraformDocsOrg,
+ repo: terraformDocsRepo,
+ path: useRoot ? remoteFile : `${terraformDocsDocsDir}/${remoteFile}`,
+ branch: terraformDocsBranch,
+ })
// Strip out HTML comments
rawContent = rawContent.replace(/<!--.*?-->/, '')
let { content, data } = matter(rawContent)
diff --git a/apps/docs/app/guides/deployment/terraform/reference/page.tsx b/apps/docs/app/guides/deployment/terraform/reference/page.tsx
index ae9c9b66daa..b8877c2e92a 100644
--- a/apps/docs/app/guides/deployment/terraform/reference/page.tsx
+++ b/apps/docs/app/guides/deployment/terraform/reference/page.tsx
@@ -13,7 +13,7 @@ import {
import { genGuideMeta } from '~/features/docs/GuidesMdx.utils'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
-import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
+import { getGitHubFileContents } from '~/lib/octokit'
import { TabPanel, Tabs } from '~/features/ui/Tabs'
import {
terraformDocsBranch,
@@ -394,21 +394,14 @@ const TerraformReferencePage = async () => {
* Fetch JSON schema from external repo
*/
const getSchema = async () => {
- let response: Response
- try {
- response = await fetchRevalidatePerDay(
- `https://raw.githubusercontent.com/${terraformDocsOrg}/${terraformDocsRepo}/${terraformDocsBranch}/${terraformDocsDocsDir}/schema.json`
- )
- } catch (err) {
- throw new Error(`Failed to fetch Terraform JSON schema from GitHub (network error): ${err}`)
- }
-
- if (!response.ok)
- throw Error(
- `Failed to fetch Terraform JSON schema from GitHub: ${response.status} ${response.statusText}`
- )
-
- const schema = await response.json()
+ const schema = JSON.parse(
+ await getGitHubFileContents({
+ org: terraformDocsOrg,
+ repo: terraformDocsRepo,
+ path: `${terraformDocsDocsDir}/schema.json`,
+ branch: terraformDocsBranch,
+ })
+ )
return {
schema,
diff --git a/apps/docs/app/guides/getting-started/ai-skills/AiSkills.utils.ts b/apps/docs/app/guides/getting-started/ai-skills/AiSkills.utils.ts
index 1ed1a7ca49b..ca7f7ff1b67 100644
--- a/apps/docs/app/guides/getting-started/ai-skills/AiSkills.utils.ts
+++ b/apps/docs/app/guides/getting-started/ai-skills/AiSkills.utils.ts
@@ -1,6 +1,8 @@
import matter from 'gray-matter'
import { cache } from 'react'
+import { getGitHubFileContents, octokit } from '~/lib/octokit'
+
const SKILLS_REPO = {
org: 'supabase',
repo: 'agent-skills',
@@ -20,73 +22,29 @@ interface SkillSummary {
installCommand: string
}
-interface GitHubContentItem {
- name: string
- path: string
- type: 'file' | 'dir'
-}
-
-async function fetchGitHubDirectory(path: string): Promise<GitHubContentItem[]> {
- const url = `https://api.github.com/repos/${SKILLS_REPO.org}/${SKILLS_REPO.repo}/contents/${path}?ref=${SKILLS_REPO.branch}`
-
- let response: Response
- try {
- response = await fetch(url, {
- headers: {
- Accept: 'application/vnd.github.v3+json',
- 'User-Agent': 'Supabase-Docs',
- },
- next: { revalidate: 3600 },
- })
- } catch (err) {
- throw new Error(`Failed to fetch agent skills directory from GitHub (network error)`, {
- cause: err,
- })
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch agent skills directory from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- const data = await response.json()
- if (!Array.isArray(data)) {
+async function getAiSkillsImpl(): Promise<SkillSummary[]> {
+ const { data: contents } = await octokit().request('GET /repos/{owner}/{repo}/contents/{path}', {
+ owner: SKILLS_REPO.org,
+ repo: SKILLS_REPO.repo,
+ path: SKILLS_REPO.path,
+ ref: SKILLS_REPO.branch,
+ })
+
+ if (!Array.isArray(contents)) {
throw new Error('Expected directory listing from GitHub agent skills repo')
}
- return data
-}
-
-async function fetchGitHubFile(path: string): Promise<string> {
- const url = `https://raw.githubusercontent.com/${SKILLS_REPO.org}/${SKILLS_REPO.repo}/${SKILLS_REPO.branch}/${path}`
-
- let response: Response
- try {
- response = await fetch(url, {
- next: { revalidate: 3600 },
- })
- } catch (err) {
- throw new Error(`Failed to fetch agent skill file from GitHub (network error)`, { cause: err })
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch agent skill file from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- return await response.text()
-}
-
-async function getAiSkillsImpl(): Promise<SkillSummary[]> {
- const directories = await fetchGitHubDirectory(SKILLS_REPO.path)
- const skillDirs = directories.filter((item) => item.type === 'dir')
+ const skillDirs = contents.filter((item) => item.type === 'dir')
const skills = await Promise.all(
skillDirs.map(async (item) => {
const skillPath = `${SKILLS_REPO.path}/${item.name}/SKILL.md`
- const rawContent = await fetchGitHubFile(skillPath)
+ const rawContent = await getGitHubFileContents({
+ org: SKILLS_REPO.org,
+ repo: SKILLS_REPO.repo,
+ branch: SKILLS_REPO.branch,
+ path: skillPath,
+ })
const { data } = matter(rawContent) as { data: SkillMetadata }
return {
diff --git a/apps/docs/app/guides/graphql/[[...slug]]/page.tsx b/apps/docs/app/guides/graphql/[[...slug]]/page.tsx
index 57ca0b26eae..5368816194c 100644
--- a/apps/docs/app/guides/graphql/[[...slug]]/page.tsx
+++ b/apps/docs/app/guides/graphql/[[...slug]]/page.tsx
@@ -4,7 +4,7 @@ import rehypeSlug from 'rehype-slug'
import { GuideTemplate, newEditLink } from '~/features/docs/GuidesMdx.template'
import { genGuideMeta } from '~/features/docs/GuidesMdx.utils'
-import { REVALIDATION_TAGS } from '~/features/helpers.fetch'
+import { getGitHubFileContents } from '~/lib/octokit'
import { UrlTransformFunction, linkTransform } from '~/lib/mdx/plugins/rehypeLinkTransform'
import remarkMkDocsAdmonition from '~/lib/mdx/plugins/remarkAdmonition'
import { removeTitle } from '~/lib/mdx/plugins/remarkRemoveTitle'
@@ -136,23 +136,12 @@ const getContent = async ({ slug }: Params) => {
const editLink = newEditLink(`${org}/${repo}/blob/${branch}/${docsDir}/${remoteFile}`)
- let response: Response
- try {
- response = await fetch(
- `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${docsDir}/${remoteFile}`,
- { cache: 'force-cache', next: { tags: [REVALIDATION_TAGS.GRAPHQL] } }
- )
- } catch (err) {
- throw new Error(`Failed to fetch GraphQL docs from GitHub (network error): ${err}`)
- }
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch GraphQL docs from GitHub: ${response.status} ${response.statusText}`
- )
- }
-
- const content = await response.text()
+ const content = await getGitHubFileContents({
+ org,
+ repo,
+ path: `${docsDir}/${remoteFile}`,
+ branch,
+ })
return {
pathname: `/guides/graphql${slug?.length ? `/${slug.join('/')}` : ''}` satisfies `/${string}`,
diff --git a/apps/docs/lib/octokit.ts b/apps/docs/lib/octokit.ts
index 022af9d655f..4cf5499566c 100644
--- a/apps/docs/lib/octokit.ts
+++ b/apps/docs/lib/octokit.ts
@@ -34,19 +34,13 @@ export function octokit() {
return octokitInstance
}
-async function getGitHubFileContents({
- org,
- repo,
- path,
- branch,
- options: { onError, fetch },
-}: {
+type GithubFileRequest = {
org: string
repo: string
path: string
branch: string
- options: {
- onError: (err?: unknown) => void
+ options?: {
+ onError?: (err?: unknown) => void
/**
*
* A custom fetch implementation to control Next.js caching.
@@ -55,13 +49,25 @@ async function getGitHubFileContents({
*/
fetch?: (info: RequestInfo, init?: RequestInit) => Promise<Response>
}
-}) {
+}
+
+export async function getGitHubFileContents({
+ org,
+ repo,
+ path,
+ branch,
+ options: { onError, fetch } = {},
+}: GithubFileRequest) {
if (path.startsWith('/')) {
path = path.slice(1)
}
+ const client = octokit()
+ let response: Awaited<
+ ReturnType<typeof client.request<'GET /repos/{owner}/{repo}/contents/{path}'>>
+ >
try {
- const response = await octokit().request('GET /repos/{owner}/{repo}/contents/{path}', {
+ response = await client.request('GET /repos/{owner}/{repo}/contents/{path}', {
owner: org,
repo: repo,
path: path,
@@ -70,19 +76,34 @@ async function getGitHubFileContents({
fetch: fetch ?? fetchRevalidatePerDay,
},
})
- if (response.status !== 200 || !response.data) {
- throw Error(`Could not find contents of ${path} in ${org}/${repo}`)
- }
- if (!('type' in response.data) || response.data.type !== 'file') {
- throw Error(`${path} in ${org}/${repo} is not a file`)
- }
- const content = Buffer.from(response.data.content, 'base64').toString('utf-8')
- return content
} catch (err) {
- console.error('Error fetching GitHub file: %o', err)
- onError?.(err)
+ const error = new Error(
+ `getGitHubFileContents: request failed for ${org}/${repo}/${path}@${branch}`,
+ { cause: err }
+ )
+ console.error(error)
+ onError?.(error)
+ return ''
+ }
+
+ if (Array.isArray(response.data)) {
+ const error = new Error(
+ `getGitHubFileContents: ${path} in ${org}/${repo} is a directory, not a file`
+ )
+ console.error(error)
+ onError?.(error)
return ''
}
+ if (!('content' in response.data) || response.data.type !== 'file') {
+ const error = new Error(
+ `getGitHubFileContents: unexpected response for ${path} in ${org}/${repo} (type: ${'type' in response.data ? response.data.type : 'unknown'})`
+ )
+ console.error(error)
+ onError?.(error)
+ return ''
+ }
+
+ return Buffer.from(response.data.content, 'base64').toString('utf-8')
}
export async function getGitHubFileContentsImmutableOnly({
@@ -90,17 +111,8 @@ export async function getGitHubFileContentsImmutableOnly({
repo,
branch,
path,
- options: { onError, fetch },
-}: {
- org: string
- repo: string
- branch: string
- path: string
- options: {
- onError: (error: unknown) => void
- fetch: (url: string) => Promise<Response>
- }
-}): Promise<string> {
+ options: { onError, fetch } = {},
+}: GithubFileRequest): Promise<string> {
const isImmutableCommit = await checkForImmutableCommit({
org,
repo,