A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat(storage): connect GitHub repos from the Add Connection UI (#10604)
Shahmir Varqha289c7ee76722
diff --git a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snapindex 4398d9d9fd..6f52f04ce2 100644--- a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap+++ b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap@@ -117,6 +117,49 @@ exports[`generateStorageCode > GCS > without service account key (default credenstore = GCSStore("my-bucket")"`;+exports[`generateStorageCode > GitHub > public repo 1`] = `+"from fsspec.implementations.github import GithubFileSystem++fs = GithubFileSystem(+ org="marimo-team",+ repo="marimo",+)"+`;++exports[`generateStorageCode > GitHub > with sha 1`] = `+"from fsspec.implementations.github import GithubFileSystem++fs = GithubFileSystem(+ org="marimo-team",+ repo="marimo",+ sha="main",+)"+`;++exports[`generateStorageCode > GitHub > with username and token 1`] = `+"from fsspec.implementations.github import GithubFileSystem++fs = GithubFileSystem(+ org="marimo-team",+ repo="marimo",+ username="octocat",+ token="ghp_example",+)"+`;++exports[`generateStorageCode > GitHub > with username and token from secrets 1`] = `+"from fsspec.implementations.github import GithubFileSystem+import os++_token = os.environ.get("GITHUB_TOKEN")+fs = GithubFileSystem(+ org="marimo-team",+ repo="marimo",+ username="octocat",+ token=_token,+)"+`;+exports[`generateStorageCode > Google Drive > with default auth (no credentials) 1`] = `"from gdrive_fsspec import GoogleDriveFileSystemdiff --git a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.tsindex 58711056e5..4d515fb94c 100644--- a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts+++ b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts@@ -329,6 +329,83 @@ describe("generateStorageCode", () => {});});+ describe("GitHub", () => {+ it("public repo", () => {+ expect(+ generateStorageCode(+ { type: "github", org: "marimo-team", repo: "marimo" },+ { library: "fsspec" },+ ),+ ).toMatchSnapshot();+ });++ it("with sha", () => {+ expect(+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ sha: "main",+ },+ { library: "fsspec" },+ ),+ ).toMatchSnapshot();+ });++ it("escapes quotes and backslashes in sha", () => {+ expect(+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ sha: 'feature/\\"new"',+ },+ { library: "fsspec" },+ ),+ ).toMatchInlineSnapshot(`+ "from fsspec.implementations.github import GithubFileSystem++ fs = GithubFileSystem(+ org="marimo-team",+ repo="marimo",+ sha="feature/\\\\\\"new\\"",+ )"+ `);+ });++ it("with username and token", () => {+ expect(+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ username: "octocat",+ token: "ghp_example",+ },+ { library: "fsspec" },+ ),+ ).toMatchSnapshot();+ });++ it("with username and token from secrets", () => {+ expect(+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ username: "octocat",+ token: prefixSecret("GITHUB_TOKEN"),+ },+ { library: "fsspec" },+ ),+ ).toMatchSnapshot();+ });+ });+describe("invalid cases", () => {it("throws for empty S3 bucket", () => {expect(() =>@@ -397,5 +474,55 @@ describe("generateStorageCode", () => {),).toThrow();});++ it("throws for empty GitHub org", () => {+ expect(() =>+ generateStorageCode(+ { type: "github", org: "", repo: "marimo" } as StorageConnection,+ { library: "fsspec" },+ ),+ ).toThrow();+ });++ it("throws for empty GitHub repo", () => {+ expect(() =>+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "",+ } as StorageConnection,+ { library: "fsspec" },+ ),+ ).toThrow();+ });++ it("throws when GitHub username is set without token", () => {+ expect(() =>+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ username: "octocat",+ },+ { library: "fsspec" },+ ),+ ).toThrow();+ });++ it("throws when GitHub token is set without username", () => {+ expect(() =>+ generateStorageCode(+ {+ type: "github",+ org: "marimo-team",+ repo: "marimo",+ token: "ghp_example",+ },+ { library: "fsspec" },+ ),+ ).toThrow();+ });});});diff --git a/frontend/src/components/editor/connections/storage/add-storage-form.tsx b/frontend/src/components/editor/connections/storage/add-storage-form.tsxindex 4edbbd30c7..bac7674a69 100644--- a/frontend/src/components/editor/connections/storage/add-storage-form.tsx+++ b/frontend/src/components/editor/connections/storage/add-storage-form.tsx@@ -16,6 +16,7 @@ import {AzureStorageSchema,CoreWeaveStorageSchema,GCSStorageSchema,+ GithubStorageSchema,GoogleDriveStorageSchema,HuggingfaceStorageSchema,S3StorageSchema,@@ -89,6 +90,15 @@ const STORAGE_PROVIDERS = [preferred: "huggingface_hub",},},+ {+ name: "GitHub",+ schema: GithubStorageSchema,+ protocol: "github",+ storageLibraries: {+ libraries: ["fsspec"],+ preferred: "fsspec",+ },+ },] satisfies StorageProviderSchema[];const StorageProviderSelector: React.FC<{@@ -105,7 +115,7 @@ const StorageProviderSelector: React.FC<{<ProtocolIconprotocol={protocol as KnownStorageProtocol}forceDark={true}- className="w-7.5 h-7.5"+ className="w-7.5 h-7.5 text-white"/>}onSelect={() => onSelect(schema)}diff --git a/frontend/src/components/editor/connections/storage/as-code.ts b/frontend/src/components/editor/connections/storage/as-code.tsindex 2619dcd5ec..82f6d7caf2 100644--- a/frontend/src/components/editor/connections/storage/as-code.ts+++ b/frontend/src/components/editor/connections/storage/as-code.ts@@ -43,7 +43,7 @@ class SecretContainer {this.secrets[privateVar] = `os.environ.get("${envVar}")`;return privateVar;}- return `"${value}"`;+ return `"${escapePythonString(value)}"`;}formatSecrets(): string {@@ -260,6 +260,32 @@ function generateHuggingfaceCode(return { imports, code: `hf = HfApi(token=${token})` };}+function generateGithubCode(+ connection: Extract<StorageConnection, { type: "github" }>,+ secrets: SecretContainer,+): { imports: Set<string>; code: string } {+ const imports = new Set([+ "from fsspec.implementations.github import GithubFileSystem",+ ]);+ const params: string[] = [+ `org=${secrets.print("org", connection.org)},`,+ `repo=${secrets.print("repo", connection.repo)},`,+ ];++ if (connection.sha) {+ params.push(`sha=${secrets.print("sha", connection.sha)},`);+ }+ if (connection.username && connection.token) {+ params.push(+ `username=${secrets.print("username", connection.username)},`,+ `token=${secrets.print("token", connection.token)},`,+ );+ }++ const paramsStr = params.map((param) => ` ${param}`).join("\n");+ return { imports, code: `fs = GithubFileSystem(\n${paramsStr}\n)` };+}+export function generateStorageCode(connection: StorageConnection,options: StorageCodeOptions,@@ -291,6 +317,9 @@ export function generateStorageCode(case "huggingface":result = generateHuggingfaceCode(connection, secrets);break;+ case "github":+ result = generateGithubCode(connection, secrets);+ break;default:assertNever(connection);}diff --git a/frontend/src/components/editor/connections/storage/schemas.ts b/frontend/src/components/editor/connections/storage/schemas.tsindex b46610ebb5..64bee7fa37 100644--- a/frontend/src/components/editor/connections/storage/schemas.ts+++ b/frontend/src/components/editor/connections/storage/schemas.ts@@ -238,6 +238,68 @@ export const HuggingfaceStorageSchema = z}).describe(FieldOptions.of({ direction: "two-columns" }));+function isFilled(value: string | undefined): boolean {+ return value != null && value !== "";+}++export const GithubStorageSchema = z+ .object({+ type: z.literal("github"),+ org: z+ .string()+ .nonempty()+ .describe(+ FieldOptions.of({+ label: "Organization or user",+ }),+ ),+ repo: z+ .string()+ .nonempty()+ .describe(+ FieldOptions.of({+ label: "Repository",+ }),+ ),+ sha: z+ .string()+ .optional()+ .describe(+ FieldOptions.of({+ label: "Branch, tag, or commit",+ placeholder: "main",+ description: "Leave empty to use the default branch",+ }),+ ),+ username: z+ .string()+ .optional()+ .describe(+ FieldOptions.of({+ label: "Username",+ description: "Set this together with access token",+ optionRegex: "(github.?username|github.?user)",+ }),+ ),+ token: z+ .string()+ .optional()+ .describe(+ FieldOptions.of({+ label: "Access Token",+ description:+ "Set this together with username. Required for private repos and to avoid GitHub's rate limits.",+ inputType: "password",+ optionRegex: "(github.?token|gh.?token|github.?pat)",+ }),+ ),+ })+ .refine((value) => isFilled(value.username) === isFilled(value.token), {+ message: "Username and access token are required together",+ path: ["token"],+ })+ .describe(FieldOptions.of({ direction: "two-columns" }));+export const StorageConnectionSchema = z.discriminatedUnion("type", [S3StorageSchema,GCSStorageSchema,@@ -245,6 +307,7 @@ export const StorageConnectionSchema = z.discriminatedUnion("type", [CoreWeaveStorageSchema,GoogleDriveStorageSchema,HuggingfaceStorageSchema,+ GithubStorageSchema,]);export type StorageConnection = z.infer<typeof StorageConnectionSchema>;