Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(storage): connect GitHub repos from the Add Connection UI (#10604)

Shahmir Varqha289c7ee76722
diff --git a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
index 4398d9d9fd..6f52f04ce2 100644
--- a/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
+++ b/frontend/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap
@@ -117,6 +117,49 @@ exports[`generateStorageCode > GCS > without service account key (default creden
store = GCSStore("my-bucket")"
`;
+exports[`generateStorageCode > GitHub > public repo 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+ org="marimo-team",
+ repo="marimo",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with sha 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+ org="marimo-team",
+ repo="marimo",
+ sha="main",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+ org="marimo-team",
+ repo="marimo",
+ username="octocat",
+ token="ghp_example",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token from secrets 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+import os
+
+_token = os.environ.get("GITHUB_TOKEN")
+fs = GithubFileSystem(
+ org="marimo-team",
+ repo="marimo",
+ username="octocat",
+ token=_token,
+)"
+`;
+
exports[`generateStorageCode > Google Drive > with default auth (no credentials) 1`] = `
"from gdrive_fsspec import GoogleDriveFileSystem
diff --git a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
index 58711056e5..4d515fb94c 100644
--- a/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
+++ b/frontend/src/components/editor/connections/storage/__tests__/as-code.test.ts
@@ -329,6 +329,83 @@ describe("generateStorageCode", () => {
});
});
+ describe("GitHub", () => {
+ it("public repo", () => {
+ expect(
+ generateStorageCode(
+ { type: "github", org: "marimo-team", repo: "marimo" },
+ { library: "fsspec" },
+ ),
+ ).toMatchSnapshot();
+ });
+
+ it("with sha", () => {
+ expect(
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ sha: "main",
+ },
+ { library: "fsspec" },
+ ),
+ ).toMatchSnapshot();
+ });
+
+ it("escapes quotes and backslashes in sha", () => {
+ expect(
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ sha: 'feature/\\"new"',
+ },
+ { library: "fsspec" },
+ ),
+ ).toMatchInlineSnapshot(`
+ "from fsspec.implementations.github import GithubFileSystem
+
+ fs = GithubFileSystem(
+ org="marimo-team",
+ repo="marimo",
+ sha="feature/\\\\\\"new\\"",
+ )"
+ `);
+ });
+
+ it("with username and token", () => {
+ expect(
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ username: "octocat",
+ token: "ghp_example",
+ },
+ { library: "fsspec" },
+ ),
+ ).toMatchSnapshot();
+ });
+
+ it("with username and token from secrets", () => {
+ expect(
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ username: "octocat",
+ token: prefixSecret("GITHUB_TOKEN"),
+ },
+ { library: "fsspec" },
+ ),
+ ).toMatchSnapshot();
+ });
+ });
+
describe("invalid cases", () => {
it("throws for empty S3 bucket", () => {
expect(() =>
@@ -397,5 +474,55 @@ describe("generateStorageCode", () => {
),
).toThrow();
});
+
+ it("throws for empty GitHub org", () => {
+ expect(() =>
+ generateStorageCode(
+ { type: "github", org: "", repo: "marimo" } as StorageConnection,
+ { library: "fsspec" },
+ ),
+ ).toThrow();
+ });
+
+ it("throws for empty GitHub repo", () => {
+ expect(() =>
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "",
+ } as StorageConnection,
+ { library: "fsspec" },
+ ),
+ ).toThrow();
+ });
+
+ it("throws when GitHub username is set without token", () => {
+ expect(() =>
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ username: "octocat",
+ },
+ { library: "fsspec" },
+ ),
+ ).toThrow();
+ });
+
+ it("throws when GitHub token is set without username", () => {
+ expect(() =>
+ generateStorageCode(
+ {
+ type: "github",
+ org: "marimo-team",
+ repo: "marimo",
+ token: "ghp_example",
+ },
+ { library: "fsspec" },
+ ),
+ ).toThrow();
+ });
});
});
diff --git a/frontend/src/components/editor/connections/storage/add-storage-form.tsx b/frontend/src/components/editor/connections/storage/add-storage-form.tsx
index 4edbbd30c7..bac7674a69 100644
--- a/frontend/src/components/editor/connections/storage/add-storage-form.tsx
+++ b/frontend/src/components/editor/connections/storage/add-storage-form.tsx
@@ -16,6 +16,7 @@ import {
AzureStorageSchema,
CoreWeaveStorageSchema,
GCSStorageSchema,
+ GithubStorageSchema,
GoogleDriveStorageSchema,
HuggingfaceStorageSchema,
S3StorageSchema,
@@ -89,6 +90,15 @@ const STORAGE_PROVIDERS = [
preferred: "huggingface_hub",
},
},
+ {
+ name: "GitHub",
+ schema: GithubStorageSchema,
+ protocol: "github",
+ storageLibraries: {
+ libraries: ["fsspec"],
+ preferred: "fsspec",
+ },
+ },
] satisfies StorageProviderSchema[];
const StorageProviderSelector: React.FC<{
@@ -105,7 +115,7 @@ const StorageProviderSelector: React.FC<{
<ProtocolIcon
protocol={protocol as KnownStorageProtocol}
forceDark={true}
- className="w-7.5 h-7.5"
+ className="w-7.5 h-7.5 text-white"
/>
}
onSelect={() => onSelect(schema)}
diff --git a/frontend/src/components/editor/connections/storage/as-code.ts b/frontend/src/components/editor/connections/storage/as-code.ts
index 2619dcd5ec..82f6d7caf2 100644
--- a/frontend/src/components/editor/connections/storage/as-code.ts
+++ b/frontend/src/components/editor/connections/storage/as-code.ts
@@ -43,7 +43,7 @@ class SecretContainer {
this.secrets[privateVar] = `os.environ.get("${envVar}")`;
return privateVar;
}
- return `"${value}"`;
+ return `"${escapePythonString(value)}"`;
}
formatSecrets(): string {
@@ -260,6 +260,32 @@ function generateHuggingfaceCode(
return { imports, code: `hf = HfApi(token=${token})` };
}
+function generateGithubCode(
+ connection: Extract<StorageConnection, { type: "github" }>,
+ secrets: SecretContainer,
+): { imports: Set<string>; code: string } {
+ const imports = new Set([
+ "from fsspec.implementations.github import GithubFileSystem",
+ ]);
+ const params: string[] = [
+ `org=${secrets.print("org", connection.org)},`,
+ `repo=${secrets.print("repo", connection.repo)},`,
+ ];
+
+ if (connection.sha) {
+ params.push(`sha=${secrets.print("sha", connection.sha)},`);
+ }
+ if (connection.username && connection.token) {
+ params.push(
+ `username=${secrets.print("username", connection.username)},`,
+ `token=${secrets.print("token", connection.token)},`,
+ );
+ }
+
+ const paramsStr = params.map((param) => ` ${param}`).join("\n");
+ return { imports, code: `fs = GithubFileSystem(\n${paramsStr}\n)` };
+}
+
export function generateStorageCode(
connection: StorageConnection,
options: StorageCodeOptions,
@@ -291,6 +317,9 @@ export function generateStorageCode(
case "huggingface":
result = generateHuggingfaceCode(connection, secrets);
break;
+ case "github":
+ result = generateGithubCode(connection, secrets);
+ break;
default:
assertNever(connection);
}
diff --git a/frontend/src/components/editor/connections/storage/schemas.ts b/frontend/src/components/editor/connections/storage/schemas.ts
index b46610ebb5..64bee7fa37 100644
--- a/frontend/src/components/editor/connections/storage/schemas.ts
+++ b/frontend/src/components/editor/connections/storage/schemas.ts
@@ -238,6 +238,68 @@ export const HuggingfaceStorageSchema = z
})
.describe(FieldOptions.of({ direction: "two-columns" }));
+function isFilled(value: string | undefined): boolean {
+ return value != null && value !== "";
+}
+
+export const GithubStorageSchema = z
+ .object({
+ type: z.literal("github"),
+ org: z
+ .string()
+ .nonempty()
+ .describe(
+ FieldOptions.of({
+ label: "Organization or user",
+ }),
+ ),
+ repo: z
+ .string()
+ .nonempty()
+ .describe(
+ FieldOptions.of({
+ label: "Repository",
+ }),
+ ),
+ sha: z
+ .string()
+ .optional()
+ .describe(
+ FieldOptions.of({
+ label: "Branch, tag, or commit",
+ placeholder: "main",
+ description: "Leave empty to use the default branch",
+ }),
+ ),
+ username: z
+ .string()
+ .optional()
+ .describe(
+ FieldOptions.of({
+ label: "Username",
+ description: "Set this together with access token",
+ optionRegex: "(github.?username|github.?user)",
+ }),
+ ),
+ token: z
+ .string()
+ .optional()
+ .describe(
+ FieldOptions.of({
+ label: "Access Token",
+ description:
+ "Set this together with username. Required for private repos and to avoid GitHub's rate limits.",
+ inputType: "password",
+ optionRegex: "(github.?token|gh.?token|github.?pat)",
+ }),
+ ),
+ })
+ .refine((value) => isFilled(value.username) === isFilled(value.token), {
+ message: "Username and access token are required together",
+ path: ["token"],
+ })
+ .describe(FieldOptions.of({ direction: "two-columns" }));
+
export const StorageConnectionSchema = z.discriminatedUnion("type", [
S3StorageSchema,
GCSStorageSchema,
@@ -245,6 +307,7 @@ export const StorageConnectionSchema = z.discriminatedUnion("type", [
CoreWeaveStorageSchema,
GoogleDriveStorageSchema,
HuggingfaceStorageSchema,
+ GithubStorageSchema,
]);
export type StorageConnection = z.infer<typeof StorageConnectionSchema>;