Skip to content

armgit / kaneo

Public Imported from GitHub
0

⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.

22.4 KB
Raw Blame
import { confirmedFieldLabels } from "../../plugins/sync/field-labels";
import { acceptsIssue, readSyncRules } from "../../plugins/sync/rules";
import { canSyncTask } from "../../plugins/sync/eligibility";
import { importIssueLabels } from "../../plugins/sync/issue-labels";
import { parseLinkMetadata } from "../../plugins/github/utils/parse-link-metadata";
import { createId } from "@paralleldrive/cuid2";
import { and, asc, eq, isNull } from "drizzle-orm";
import { HTTPException } from "hono/http-exception";
import db from "../../database";
import { publishEvent } from "../../events";
import {
  activityTable,
  columnTable,
  externalLinkTable,
  githubImportTable,
  integrationTable,
  labelTable,
  projectTable,
  taskTable,
} from "../../database/schema";
import {
  type GitHubConfig,
  hasVerifiedGitHubBinding,
} from "../../plugins/github/config";
import { resolvePullRequestTask } from "../../plugins/github/services/resolve-pull-request-task";
import {
  extractIssuePriority,
  extractIssueStatus,
} from "../../plugins/github/utils/extract-priority";
import { formatTaskDescriptionFromIssue } from "../../plugins/github/utils/format";
import { getVerifiedInstallationOctokit } from "../../plugins/github/utils/github-app";
import { claimTaskNumber } from "../../task/controllers/claim-task-numbers";
import { VIRTUAL_STATUSES } from "../../task/validate-task-fields";
import { withGithubImportLock } from "../import-lock";
import {
  commentsPageSchema,
  fetchImportPage,
  IMPORT_PAGES_PER_REQUEST,
  type ImportedComment,
  type ImportedIssue,
  type ImportedLabel,
  type ImportedPull,
  issuesPageSchema,
  labelsPageSchema,
  nextCursor,
  pullsPageSchema,
} from "../import-pages";
import {
  type GitHubImportState,
  importProgress,
  initialImportState,
} from "../import-state";

type ImportEvent = "task.updated" | "task.labels_updated" | "comment.updated";
type Transaction = Parameters<Parameters<typeof db.transaction>[0]>[0];
async function resolveImportedStatus(
  tx: Transaction,
  projectId: string,
  requested: string | null,
  current?: string,
) {
  for (const status of [requested, current, "to-do"]) {
    if (!status) continue;
    const column = await tx.query.columnTable.findFirst({
      where: and(
        eq(columnTable.projectId, projectId),
        eq(columnTable.slug, status),
      ),
      columns: { id: true },
    });
    if (column) return { status, columnId: column.id };
    if ((VIRTUAL_STATUSES as readonly string[]).includes(status))
      return { status, columnId: null };
  }
  const first = await tx.query.columnTable.findFirst({
    where: eq(columnTable.projectId, projectId),
    orderBy: asc(columnTable.position),
    columns: { id: true, slug: true },
  });
  return { status: first?.slug ?? "planned", columnId: first?.id ?? null };
}
function readConfig(integration: typeof integrationTable.$inferSelect) {
  if (!integration.isActive)
    throw new HTTPException(400, {
      message: "GitHub integration is not active",
    });
  let config: GitHubConfig;
  try {
    config = JSON.parse(integration.config);
  } catch {
    throw new HTTPException(400, {
      message: "Invalid GitHub integration configuration",
    });
  }
  if (!hasVerifiedGitHubBinding(config) || !config.installationId)
    throw new HTTPException(400, {
      message:
        "GitHub integration must be reconnected by a repository administrator",
    });
  return config;
}
function conflict() {
  return new HTTPException(409, {
    message: "GitHub integration or import changed; refresh before resuming",
  });
}

export async function importIssues(projectId: string, runId?: string) {
  return withGithubImportLock(projectId, async () => {
    const project = await db.query.projectTable.findFirst({
      where: eq(projectTable.id, projectId),
    });
    if (!project)
      throw new HTTPException(404, { message: "Project not found" });
    const integration = await db.query.integrationTable.findFirst({
      where: and(
        eq(integrationTable.projectId, projectId),
        eq(integrationTable.type, "github"),
      ),
    });
    if (!integration)
      throw new HTTPException(404, { message: "GitHub integration not found" });
    const config = readConfig(integration);
    const repositoryId = config.repositoryId;
    if (!repositoryId) throw conflict();
    let run = await db.query.githubImportTable.findFirst({
      where: eq(githubImportTable.integrationId, integration.id),
    });
    if (
      runId &&
      (!run || run.runId !== runId || run.state.repositoryId !== repositoryId)
    )
      throw conflict();
    if (runId && run?.state.phase === "complete")
      return importProgress(run.runId, run.state);
    if (
      !run ||
      run.state.phase === "complete" ||
      run.state.repositoryId !== repositoryId
    ) {
      const [created] = await db
        .insert(githubImportTable)
        .values({
          integrationId: integration.id,
          state: initialImportState(repositoryId),
        })
        .onConflictDoUpdate({
          target: githubImportTable.integrationId,
          set: {
            runId: createId(),
            state: initialImportState(repositoryId),
            updatedAt: new Date(),
          },
        })
        .returning();
      run = created;
    }
    if (!run) throw new Error("Failed to start import");
    let octokit: Awaited<ReturnType<typeof getVerifiedInstallationOctokit>>;
    try {
      octokit = await getVerifiedInstallationOctokit(config, true);
    } catch {
      throw new HTTPException(502, {
        message:
          "GitHub repository could not be verified; check the integration and resume",
      });
    }
    for (
      let page = 0;
      page < IMPORT_PAGES_PER_REQUEST && run.state.phase !== "complete";
      page++
    ) {
      let payload: unknown;
      try {
        payload = await fetchImportPage(octokit, config, run.state);
      } catch {
        throw new HTTPException(502, {
          message: "GitHub import paused; retry to resume saved progress",
        });
      }
      const rules = readSyncRules(integration.config);
      let admitted = rules !== null;
      let scopeLabels: unknown = [];
      if (
        run.state.phase === "issues" &&
        (rules?.incoming.mode === "labels" || rules?.outgoing.mode === "labels")
      ) {
        const parsed = issuesPageSchema.safeParse(payload);
        const issue = parsed.success
          ? parsed.data.repository.issues.nodes[0]
          : undefined;
        if (issue) {
          try {
            const remote = await octokit.rest.issues.get({
              owner: config.repositoryOwner,
              repo: config.repositoryName,
              issue_number: issue.number,
              request: { timeout: 10_000 },
            });
            scopeLabels = remote.data.labels;
            admitted = acceptsIssue(integration.config, scopeLabels);
          } catch {
            throw new HTTPException(502, {
              message:
                "Issue labels could not be read; resume the import later",
            });
          }
        }
      }
      const currentRun: typeof githubImportTable.$inferSelect = run;
      const notifications = new Map<
        string,
        { type: ImportEvent; taskId: string }
      >();
      run = await db.transaction(async (tx) => {
        const [currentProject] = await tx
          .select()
          .from(projectTable)
          .where(
            and(
              eq(projectTable.id, project.id),
              eq(projectTable.workspaceId, project.workspaceId),
            ),
          )
          .for("key share");
        if (!currentProject) throw conflict();
        // Serialize against integration changes and webhook issue creation. No
        // provider request is made while this transaction holds row locks.
        const [currentIntegration] = await tx
          .select()
          .from(integrationTable)
          .where(eq(integrationTable.id, integration.id))
          .for("update");
        if (
          !currentIntegration ||
          currentIntegration.config !== integration.config ||
          !currentIntegration.isActive
        )
          throw conflict();
        const state = structuredClone(currentRun.state);
        await applyPage(
          tx,
          payload,
          state,
          integration.id,
          project,
          config,
          (taskId, ...types) => {
            for (const type of types)
              notifications.set(`${type}:${taskId}`, { type, taskId });
          },
          admitted,
          scopeLabels,
        );
        const [saved] = await tx
          .update(githubImportTable)
          .set({ state })
          .where(
            and(
              eq(githubImportTable.integrationId, integration.id),
              eq(githubImportTable.runId, currentRun.runId),
            ),
          )
          .returning();
        if (!saved) throw conflict();
        return saved;
      });
      // Each bounded page is durable before other clients refresh, including
      // continuation pages that change labels, comments or linked resources.
      for (const { type, taskId } of notifications.values())
        await publishEvent(type, { projectId, taskId });
      await publishEvent("project.updated", { projectId });
    }
    return importProgress(run.runId, run.state);
  });
}

function finishIssue(state: GitHubImportState) {
  state.currentIssue = null;
  state.phase = state.moreIssues ? "issues" : "pulls";
}
function nextIssuePart(state: GitHubImportState) {
  const current = state.currentIssue;
  if (current && current.labelsRemaining > 0) state.phase = "labels";
  else if (current?.moreComments && current.commentsRemaining > 0)
    state.phase = "comments";
  else finishIssue(state);
}
function repositoryMatches(actual: number, state: GitHubImportState) {
  if (actual !== state.repositoryId) throw conflict();
}
function pageError() {
  return new HTTPException(502, {
    message:
      "GitHub returned an invalid import page; saved progress can be resumed",
  });
}
function cursor(
  info: { hasNextPage: boolean; endCursor: string | null },
  previous: string | null,
) {
  try {
    return nextCursor(info, previous);
  } catch {
    throw pageError();
  }
}
async function applyPage(
  tx: Transaction,
  payload: unknown,
  state: GitHubImportState,
  integrationId: string,
  project: typeof projectTable.$inferSelect,
  config: GitHubConfig,
  announce: (taskId: string, ...types: ImportEvent[]) => void,
  admitted = true,
  scopeLabels: unknown = [],
) {
  if (state.phase === "issues") {
    const parsed = issuesPageSchema.safeParse(payload);
    if (!parsed.success) throw pageError();
    repositoryMatches(parsed.data.repository.databaseId, state);
    const page = parsed.data.repository.issues;
    state.issueCursor = cursor(page.pageInfo, state.issueCursor);
    state.moreIssues = page.pageInfo.hasNextPage;
    const issue = page.nodes[0];
    if (!issue || Date.parse(issue.createdAt) > Date.parse(state.startedAt)) {
      state.moreIssues = false;
      finishIssue(state);
      return;
    }
    if (
      !admitted &&
      !(await findLink(tx, integrationId, "issue", issue.number))
    ) {
      state.skipped++;
      finishIssue(state);
      return;
    }
    const task = await importIssue(tx, issue, integrationId, project.id);
    if (!task) {
      state.skipped++;
      finishIssue(state);
      return;
    }
    state[task.result]++;
    const labelsRemaining = pageRemaining(issue.labels);
    const commentsRemaining = pageRemaining(issue.comments);
    state.currentIssue = {
      number: issue.number,
      taskId: task.id,
      isNewTask: task.result === "imported",
      labelCursor: cursor(issue.labels.pageInfo, null),
      commentCursor: cursor(issue.comments.pageInfo, null),
      labelsRemaining,
      commentsRemaining,
      moreComments: issue.comments.pageInfo.hasNextPage,
      statusSeen: issue.labels.nodes.some((label) =>
        label.name.startsWith("status:"),
      ),
      prioritySeen: issue.labels.nodes.some((label) =>
        label.name.startsWith("priority:"),
      ),
    };
    await importIssueLabels(task.id, project.workspaceId, scopeLabels, tx);
    await importLabels(tx, issue.labels.nodes, task.id, project.workspaceId);
    await canSyncTask(task.id, integrationId, tx);
    await importComments(tx, issue.comments.nodes, task.id, state.startedAt);
    announce(task.id, "task.updated", "task.labels_updated", "comment.updated");
    nextIssuePart(state);
    return;
  }
  if (state.phase === "pulls") {
    const parsed = pullsPageSchema.safeParse(payload);
    if (!parsed.success) throw pageError();
    repositoryMatches(parsed.data.repository.databaseId, state);
    const page = parsed.data.repository.pullRequests;
    state.pullCursor = cursor(page.pageInfo, state.pullCursor);
    for (const pull of page.nodes) {
      if (Date.parse(pull.createdAt) > Date.parse(state.startedAt)) {
        state.phase = "complete";
        return;
      }
      const taskId = await linkPull(tx, pull, integrationId, project, config);
      if (taskId) announce(taskId, "task.updated");
    }
    if (!page.pageInfo.hasNextPage || page.nodes.length === 0)
      state.phase = "complete";
    return;
  }
  const current = state.currentIssue;
  if (!current) throw new Error("Import continuation missing");
  const [task] = await tx
    .select()
    .from(taskTable)
    .where(
      and(
        eq(taskTable.id, current.taskId),
        eq(taskTable.projectId, project.id),
      ),
    )
    .for("no key update");
  const [linked] = await tx
    .select({
      id: externalLinkTable.id,
      metadata: externalLinkTable.metadata,
      createdAt: externalLinkTable.createdAt,
    })
    .from(externalLinkTable)
    .where(
      and(
        eq(externalLinkTable.integrationId, integrationId),
        eq(externalLinkTable.resourceType, "issue"),
        eq(externalLinkTable.externalId, String(current.number)),
        eq(externalLinkTable.taskId, current.taskId),
      ),
    )
    .for("update");
  if (linked && current.isNewTask === undefined) {
    const metadata = parseLinkMetadata<{ createdFrom?: string }>(
      linked.metadata,
      {
        externalLinkId: linked.id,
        source: "github_import_continuation",
      },
    );
    current.isNewTask =
      metadata.createdFrom === "github-import" &&
      linked.createdAt.getTime() >= Date.parse(state.startedAt);
  }
  const eligible =
    task && linked && (await canSyncTask(current.taskId, integrationId, tx));
  // A newly admitted issue's initial history belongs to the explicit import,
  // even when its resulting link is paused for outgoing sync.
  if (!task || !linked || (!eligible && !current.isNewTask)) {
    state.skipped++;
    finishIssue(state);
    return;
  }
  if (state.phase === "labels") {
    const parsed = labelsPageSchema.safeParse(payload);
    if (!parsed.success) throw pageError();
    repositoryMatches(parsed.data.repository.databaseId, state);
    const page = parsed.data.repository.issue?.labels;
    if (!page) {
      state.skipped++;
      finishIssue(state);
      return;
    }
    const labels = page.nodes.slice(0, current.labelsRemaining);
    current.labelCursor = cursor(page.pageInfo, current.labelCursor);
    current.labelsRemaining =
      page.pageInfo.hasNextPage && labels.length
        ? Math.max(0, current.labelsRemaining - labels.length)
        : 0;
    const priority = current.prioritySeen ? null : extractIssuePriority(labels);
    const status = current.statusSeen ? null : extractIssueStatus(labels);
    if (priority || status)
      await tx
        .update(taskTable)
        .set({
          ...(priority ? { priority } : {}),
          ...(status
            ? await resolveImportedStatus(tx, project.id, status, task.status)
            : {}),
        })
        .where(
          and(eq(taskTable.id, task.id), eq(taskTable.projectId, project.id)),
        );
    current.prioritySeen ||= labels.some((label) =>
      label.name.startsWith("priority:"),
    );
    current.statusSeen ||= labels.some((label) =>
      label.name.startsWith("status:"),
    );
    await importLabels(tx, labels, task.id, project.workspaceId);
    announce(task.id, "task.updated", "task.labels_updated");
    nextIssuePart(state);
  } else if (state.phase === "comments") {
    const parsed = commentsPageSchema.safeParse(payload);
    if (!parsed.success) throw pageError();
    repositoryMatches(parsed.data.repository.databaseId, state);
    const page = parsed.data.repository.issue?.comments;
    if (!page) {
      state.skipped++;
      finishIssue(state);
      return;
    }
    const comments = page.nodes.slice(0, current.commentsRemaining);
    current.commentCursor = cursor(page.pageInfo, current.commentCursor);
    current.commentsRemaining = Math.max(
      0,
      current.commentsRemaining - comments.length,
    );
    current.moreComments = page.pageInfo.hasNextPage && comments.length > 0;
    await importComments(tx, comments, task.id, state.startedAt);
    announce(task.id, "comment.updated");
    nextIssuePart(state);
  }
}
function pageRemaining(page: {
  totalCount: number;
  nodes: unknown[];
  pageInfo: { hasNextPage: boolean };
}) {
  return page.pageInfo.hasNextPage
    ? Math.max(0, page.totalCount - page.nodes.length)
    : 0;
}
async function findLink(
  tx: Transaction,
  integrationId: string,
  resourceType: string,
  externalId: number,
) {
  return tx.query.externalLinkTable.findFirst({
    where: and(
      eq(externalLinkTable.integrationId, integrationId),
      eq(externalLinkTable.resourceType, resourceType),
      eq(externalLinkTable.externalId, String(externalId)),
    ),
  });
}
async function importIssue(
  tx: Transaction,
  issue: ImportedIssue,
  integrationId: string,
  projectId: string,
): Promise<{ id: string; result: "imported" | "updated" } | null> {
  const link = await findLink(tx, integrationId, "issue", issue.number);
  const priority = extractIssuePriority(issue.labels.nodes);
  const status = extractIssueStatus(issue.labels.nodes);
  if (link) {
    if (!(await canSyncTask(link.taskId, integrationId, tx))) return null;
    const [task] = await tx
      .select()
      .from(taskTable)
      .where(
        and(eq(taskTable.id, link.taskId), eq(taskTable.projectId, projectId)),
      )
      .for("no key update");
    if (!task) return null;
    const [linked] = await tx
      .select({ id: externalLinkTable.id })
      .from(externalLinkTable)
      .where(eq(externalLinkTable.id, link.id))
      .for("update");
    if (!linked) return null;
    const [updated] = await tx
      .update(taskTable)
      .set({
        title: issue.title,
        description: formatTaskDescriptionFromIssue(issue.body, task.id),
        ...(await resolveImportedStatus(tx, projectId, status, task.status)),
        ...(priority ? { priority } : {}),
      })
      .where(and(eq(taskTable.id, task.id), eq(taskTable.projectId, projectId)))
      .returning({ id: taskTable.id });
    return updated ? { id: updated.id, result: "updated" } : null;
  }
  const number = await claimTaskNumber(projectId, tx);
  const [created] = await tx
    .insert(taskTable)
    .values({
      projectId,
      userId: null,
      title: issue.title,
      description: formatTaskDescriptionFromIssue(issue.body),
      ...(await resolveImportedStatus(tx, projectId, status)),
      priority: priority ?? "low",
      number,
    })
    .returning({ id: taskTable.id });
  if (!created) throw new Error("Failed to create imported task");
  await tx.insert(externalLinkTable).values({
    taskId: created.id,
    integrationId,
    resourceType: "issue",
    externalId: String(issue.number),
    url: issue.url,
    title: issue.title,
    metadata: JSON.stringify({
      state: issue.state.toLowerCase(),
      createdFrom: "github-import",
      author: issue.author?.login,
      ...confirmedFieldLabels(issue.labels.nodes),
    }),
  });
  return { id: created.id, result: "imported" };
}
async function importLabels(
  tx: Transaction,
  labels: ImportedLabel[],
  taskId: string,
  workspaceId: string,
) {
  for (const label of labels) {
    if (label.name.startsWith("priority:") || label.name.startsWith("status:"))
      continue;
    const root = await tx.query.labelTable.findFirst({
      where: and(
        eq(labelTable.workspaceId, workspaceId),
        eq(labelTable.name, label.name),
        isNull(labelTable.taskId),
      ),
    });
    await tx
      .insert(labelTable)
      .values({
        name: label.name,
        color: root?.color || `#${label.color}`,
        taskId,
        workspaceId,
      })
      .onConflictDoNothing({ target: [labelTable.taskId, labelTable.name] });
  }
}
async function importComments(
  tx: Transaction,
  comments: ImportedComment[],
  taskId: string,
  startedAt: string,
) {
  const values = comments
    .filter(
      (comment) =>
        comment.author?.__typename !== "Bot" &&
        !comment.author?.login.endsWith("[bot]") &&
        Date.parse(comment.createdAt) <= Date.parse(startedAt),
    )
    .map((comment) => ({
      taskId,
      type: "comment",
      content: comment.body,
      externalUserName: comment.author?.login ?? "Unknown",
      externalUserAvatar: comment.author?.avatarUrl ?? null,
      externalSource: "github",
      externalUrl: comment.url,
    }));
  if (values.length)
    await tx
      .insert(activityTable)
      .values(values)
      .onConflictDoNothing({
        target: [
          activityTable.taskId,
          activityTable.externalSource,
          activityTable.externalUrl,
        ],
      });
}
async function linkPull(
  tx: Transaction,
  pull: ImportedPull,
  integrationId: string,
  project: typeof projectTable.$inferSelect,
  config: GitHubConfig,
) {
  if (await findLink(tx, integrationId, "pull_request", pull.number)) return;
  const task = await resolvePullRequestTask({
    integrationId,
    projectId: project.id,
    projectSlug: project.slug,
    config,
    repositoryUrl: `https://github.com/${config.repositoryOwner}/${config.repositoryName}`,
    pullRequest: {
      number: pull.number,
      title: pull.title,
      body: pull.body,
      head: { ref: pull.headRefName },
    },
    database: tx,
  });
  if (!task) return;
  const [scopedTask] = await tx
    .select({ id: taskTable.id })
    .from(taskTable)
    .where(and(eq(taskTable.id, task.id), eq(taskTable.projectId, project.id)))
    .for("share");
  if (!scopedTask) return;
  await tx.insert(externalLinkTable).values({
    taskId: task.id,
    integrationId,
    resourceType: "pull_request",
    externalId: String(pull.number),
    url: pull.url,
    title: pull.title,
    metadata: JSON.stringify({
      state: pull.state.toLowerCase(),
      branch: pull.headRefName,
      author: pull.author?.login,
    }),
  });
  return task.id;
}