Skip to content

⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.

fix(worker): resume incomplete import Create batches (#654)

I'm Groot 🌳d2ad9d67dddb
diff --git a/apps/worker/src/__tests__/integration/import-create.test.ts b/apps/worker/src/__tests__/integration/import-create.test.ts
index 5d68bdfd..b4778a12 100644
--- a/apps/worker/src/__tests__/integration/import-create.test.ts
+++ b/apps/worker/src/__tests__/integration/import-create.test.ts
@@ -9,10 +9,19 @@ import {
issue,
issueActivity,
importJob,
+ importRecord,
+ issueLabel,
user,
} from '@repo/db';
import { and, eq } from 'drizzle-orm';
-import { insertDiscoveredIds, type ClaimedImportJob } from '../../import-store';
+import {
+ insertDiscoveredIds,
+ createLocalIssueAndRecord,
+ createLocalComment,
+ upsertImportRecord,
+ findImportRecord,
+ type ClaimedImportJob,
+} from '../../import-store';
import { runCreate } from '../../import-worker';
import type { SourceReader } from '../../reader';
import type { CanonicalComment, CanonicalIssue } from '../../canonical';
@@ -133,6 +142,41 @@ function readerWithFlakyComments(): SourceReader {
return reader as SourceReader;
}
+function readerForIssues(issues: CanonicalIssue[]): SourceReader {
+ return {
+ listStates: async () => [],
+ listLabels: async () => [],
+ listCycles: async () => [],
+ listIssues: async () => ({ items: issues, cursor: null }),
+ getIssue: async (id) => issues.find((item) => item.sourceId === id)!,
+ listIssueRelations: async () => [],
+ listIssueComments: async () => COMMENTS,
+ listIssueAttachments: async () => [],
+ resolveAttachmentDownload: async () => {
+ throw new Error('Create must not download attachments');
+ },
+ };
+}
+
+async function seedMappedIssue(job: ClaimedImportJob): Promise<number> {
+ const [column] = await db
+ .select({ id: projectColumn.id })
+ .from(projectColumn)
+ .where(eq(projectColumn.projectId, job.projectId));
+ return createLocalIssueAndRecord(job.id, ISSUE.sourceId, String(ISSUE.sequenceId), {
+ projectId: job.projectId,
+ columnId: column!.id,
+ cycleId: null,
+ parentId: null,
+ assigneeUserId: null,
+ title: 'Locally edited title',
+ description: 'Keep local description',
+ priority: null,
+ startDate: null,
+ dueDate: null,
+ });
+}
+
describe('runCreate', () => {
it("creates an issue's comments on the retry after its comment read failed", async () => {
const job = await makeCreateJob();
@@ -153,4 +197,135 @@ describe('runCreate', () => {
.where(and(eq(issueActivity.issueId, issues[0]!.id), eq(issueActivity.kind, 'comment')));
expect(comments.map((c) => c.body).sort()).toEqual(['First', 'Second']);
});
+
+ it('finishes labels, comments and attachment discovery after an issue mapping committed', async () => {
+ const job = await makeCreateJob();
+ await insertDiscoveredIds(job.id, 'issue', [ISSUE.sourceId]);
+ await insertDiscoveredIds(job.id, 'label', ['label-1']);
+ const localId = await seedMappedIssue(job);
+ const reader = readerForIssues([{ ...ISSUE, labelSourceIds: ['label-1'] }]);
+ reader.listLabels = async () => [{ sourceId: 'label-1', name: 'Imported label' }];
+ reader.listIssueAttachments = async () => [
+ {
+ sourceId: 'attachment-1',
+ filename: 'note.txt',
+ contentType: 'text/plain',
+ sizeBytes: 4,
+ },
+ ];
+
+ await runCreate(job, reader);
+ const [savedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ await runCreate({ ...job, cursor: savedJob!.cursor as ClaimedImportJob['cursor'] }, reader);
+
+ const issues = await db.select().from(issue).where(eq(issue.projectId, job.projectId));
+ expect(issues).toHaveLength(1);
+ expect(issues[0]!.id).toBe(localId);
+ expect(issues[0]!.title).toBe('Locally edited title');
+ expect(issues[0]!.description).toBe('Keep local description');
+ const labels = await db.select().from(issueLabel).where(eq(issueLabel.issueId, localId));
+ expect(labels).toHaveLength(1);
+ const comments = await db
+ .select()
+ .from(issueActivity)
+ .where(and(eq(issueActivity.issueId, localId), eq(issueActivity.kind, 'comment')));
+ expect(comments.map((comment) => comment.body).sort()).toEqual(['First', 'Second']);
+ expect(await findImportRecord(job.id, 'attachment', 'attachment-1')).toEqual({ localId: null });
+ });
+
+ it('reuses an unmapped comment after a crash and keeps identical source comments distinct', async () => {
+ const job = await makeCreateJob();
+ await insertDiscoveredIds(job.id, 'issue', [ISSUE.sourceId]);
+ const localId = await seedMappedIssue(job);
+ const first = COMMENTS[0]!;
+ const firstId = await createLocalComment(
+ job.id,
+ localId,
+ null,
+ first.authorName,
+ first.bodyMarkdown,
+ new Date(first.createdAt),
+ null,
+ );
+ await upsertImportRecord(job.id, 'comment', first.sourceId, 'comment', firstId);
+ const unmappedId = await createLocalComment(
+ job.id,
+ localId,
+ null,
+ first.authorName,
+ first.bodyMarkdown,
+ new Date(first.createdAt),
+ null,
+ );
+ const reader = readerForIssues([ISSUE]);
+ reader.listIssueComments = async () => [
+ first,
+ { ...first, sourceId: 'identical-comment' },
+ { ...COMMENTS[1]!, replyToSourceId: 'identical-comment' },
+ ];
+
+ await runCreate(job, reader);
+ const [savedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ await runCreate({ ...job, cursor: savedJob!.cursor as ClaimedImportJob['cursor'] }, reader);
+
+ expect(await findImportRecord(job.id, 'comment', first.sourceId)).toEqual({ localId: firstId });
+ expect(await findImportRecord(job.id, 'comment', 'identical-comment')).toEqual({
+ localId: unmappedId,
+ });
+ const comments = await db
+ .select()
+ .from(issueActivity)
+ .where(and(eq(issueActivity.issueId, localId), eq(issueActivity.kind, 'comment')));
+ expect(comments).toHaveLength(3);
+ expect(comments.find((comment) => comment.body === 'Second')!.replyToId).toBe(unmappedId);
+ });
+
+ it('replays an unfinished batch and advances only beyond fully completed batches', async () => {
+ const job = await makeCreateJob();
+ const issues = Array.from({ length: 16 }, (_, index) => ({
+ ...ISSUE,
+ sourceId: `batch-${index}`,
+ sequenceId: index + 1,
+ }));
+ await insertDiscoveredIds(
+ job.id,
+ 'issue',
+ issues.map((item) => item.sourceId),
+ );
+ const reader = readerForIssues(issues);
+ reader.listIssueComments = async () => [];
+ let fail = true;
+ const reads: string[] = [];
+ reader.getIssue = async (id) => {
+ reads.push(id);
+ if (id === 'batch-1' && fail) throw new Error('interrupted batch');
+ return issues.find((item) => item.sourceId === id)!;
+ };
+ await expect(runCreate(job, reader)).rejects.toThrow('interrupted batch');
+ const [failedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ expect(failedJob!.cursor).toEqual({});
+ fail = false;
+ reads.length = 0;
+ await runCreate(job, reader);
+ expect(reads).toHaveLength(15);
+ expect(reads[0]).toBe('batch-0');
+ const [saved] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ const records = await db
+ .select()
+ .from(importRecord)
+ .where(and(eq(importRecord.importJobId, job.id), eq(importRecord.sourceEntityType, 'issue')))
+ .orderBy(importRecord.id);
+ expect(saved!.cursor).toEqual({ lastRecordId: records[14]!.id });
+ job.cursor = saved!.cursor as ClaimedImportJob['cursor'];
+ reads.length = 0;
+ await runCreate(job, reader);
+ expect(reads).toEqual(['batch-15']);
+ const [last] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ job.cursor = last!.cursor as ClaimedImportJob['cursor'];
+ await runCreate(job, reader);
+ const [finished] = await db.select().from(importJob).where(eq(importJob.id, job.id));
+ expect(finished!.phase).toBe('link');
+ const imported = await db.select().from(issue).where(eq(issue.projectId, job.projectId));
+ expect(imported).toHaveLength(16);
+ });
});
diff --git a/apps/worker/src/import-store.ts b/apps/worker/src/import-store.ts
index e2004c6e..5f349667 100644
--- a/apps/worker/src/import-store.ts
+++ b/apps/worker/src/import-store.ts
@@ -320,6 +320,26 @@ export interface ImportRecordRow {
sourceId: string;
}
+export async function listImportRecords(
+ jobId: number,
+ entityType: ImportEntityType,
+ afterId: number,
+ limit: number,
+): Promise<ImportRecordRow[]> {
+ return db
+ .select({ id: importRecord.id, sourceId: importRecord.sourceId })
+ .from(importRecord)
+ .where(
+ and(
+ eq(importRecord.importJobId, jobId),
+ eq(importRecord.sourceEntityType, entityType),
+ gt(importRecord.id, afterId),
+ ),
+ )
+ .orderBy(importRecord.id)
+ .limit(limit);
+}
+
// The next chunk of this job's snapshot that has not been materialized yet,
// keyset-paginated on import_record's own id — itsaplan's own position in the
// snapshot list, not the source's pagination cursor (see the notes file). A
diff --git a/apps/worker/src/import-worker.ts b/apps/worker/src/import-worker.ts
index d8f78d27..7b49f59a 100644
--- a/apps/worker/src/import-worker.ts
+++ b/apps/worker/src/import-worker.ts
@@ -19,6 +19,7 @@ import {
upsertImportRecord,
findImportRecord,
findImportRecordByDisplayId,
+ listImportRecords,
listUncreatedImportRecords,
listCreatedImportRecords,
firstProjectColumnId,
@@ -152,7 +153,7 @@ async function runDiscover(job: ClaimedImportJob, reader: SourceReader): Promise
}
// --- Create: materialize states/labels/cycles once (cheap, unpaginated),
-// then create one chunk of not-yet-created issues per tick, with their
+// then finish one chunk of issues per tick, with their
// comments and their attachment metadata (byte download is a later phase).
interface RecordCursor {
@@ -166,7 +167,7 @@ export async function runCreate(job: ClaimedImportJob, reader: SourceReader): Pr
const cursor = job.cursor as Partial<RecordCursor>;
const afterId = cursor.lastRecordId ?? 0;
- const pending = await listUncreatedImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
+ const pending = await listImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
if (pending.length === 0) {
await advanceImportJobPhase(job.id, 'link', {});
return;
@@ -227,46 +228,48 @@ async function createOneIssue(
sourceId: string,
): Promise<void> {
const canonical = await reader.getIssue(sourceId);
- // Read before the issue and its mapping are written: a failed read then leaves
- // the issue unmapped, and the next tick retries the whole issue.
+ // Fetch source data before writing local rows; a failed read leaves no partial writes.
const comments = await reader.listIssueComments(sourceId);
const attachments = await reader.listIssueAttachments(sourceId);
- const stateRecord = await findImportRecord(job.id, 'state', canonical.stateSourceId);
- const columnId = stateRecord?.localId ?? (await firstProjectColumnId(job.projectId));
- if (columnId == null)
- throw new Error(`project ${job.projectId} has no workflow column to import into`);
-
- const cycleRecord = canonical.cycleSourceId
- ? await findImportRecord(job.id, 'cycle', canonical.cycleSourceId)
- : null;
- const parentRecord = canonical.parentSourceId
- ? await findImportRecord(job.id, 'issue', canonical.parentSourceId)
- : null;
- const assigneeUserId = canonical.assigneeEmail
- ? await findProjectMemberUserId(job.projectId, canonical.assigneeEmail)
- : null;
+ const existing = await findImportRecord(job.id, 'issue', sourceId);
+ let localId = existing?.localId;
+ if (localId == null) {
+ const stateRecord = await findImportRecord(job.id, 'state', canonical.stateSourceId);
+ const columnId = stateRecord?.localId ?? (await firstProjectColumnId(job.projectId));
+ if (columnId == null)
+ throw new Error(`project ${job.projectId} has no workflow column to import into`);
+ const cycleRecord = canonical.cycleSourceId
+ ? await findImportRecord(job.id, 'cycle', canonical.cycleSourceId)
+ : null;
+ const parentRecord = canonical.parentSourceId
+ ? await findImportRecord(job.id, 'issue', canonical.parentSourceId)
+ : null;
+ const assigneeUserId = canonical.assigneeEmail
+ ? await findProjectMemberUserId(job.projectId, canonical.assigneeEmail)
+ : null;
+
+ localId = await createLocalIssueAndRecord(job.id, sourceId, String(canonical.sequenceId), {
+ projectId: job.projectId,
+ columnId,
+ cycleId: cycleRecord?.localId ?? null,
+ parentId: parentRecord?.localId ?? null,
+ assigneeUserId,
+ title: canonical.title,
+ description: canonical.descriptionMarkdown,
+ priority: canonical.priority,
+ startDate: canonical.startDate,
+ dueDate: canonical.dueDate,
+ createdAt: canonical.createdAt,
+ updatedAt: canonical.updatedAt,
+ });
+ }
const labelIds: number[] = [];
for (const labelSourceId of canonical.labelSourceIds) {
const record = await findImportRecord(job.id, 'label', labelSourceId);
if (record?.localId != null) labelIds.push(record.localId);
}
-
- const localId = await createLocalIssueAndRecord(job.id, sourceId, String(canonical.sequenceId), {
- projectId: job.projectId,
- columnId,
- cycleId: cycleRecord?.localId ?? null,
- parentId: parentRecord?.localId ?? null,
- assigneeUserId,
- title: canonical.title,
- description: canonical.descriptionMarkdown,
- priority: canonical.priority,
- startDate: canonical.startDate,
- dueDate: canonical.dueDate,
- createdAt: canonical.createdAt,
- updatedAt: canonical.updatedAt,
- });
if (labelIds.length) await setIssueLabels(localId, labelIds);
await createComments(job, localId, comments);
diff --git a/docs/dev/plane-import.md b/docs/dev/plane-import.md
index 6f8b13cf..781f02e8 100644
--- a/docs/dev/plane-import.md
+++ b/docs/dev/plane-import.md
@@ -130,6 +130,10 @@ attachment) are a distinct path, not covered by this — see "Inline images" in
(`plane-adapter.ts`) has no entry for `start_before`/`start_after`/`finish_before`/
`finish_after` — itsaplan's `issue_link.kind` has no scheduling-dependency concept, so
these are filtered out, not a bug.
+- Create advances its local record cursor after the entire batch finishes, including labels,
+ comments and attachment discovery. A retry revisits every issue after that cursor and reuses
+ its existing job/source mapping, then finishes its child records without replacing local
+ issue fields. Completed batches are not fetched again.
- **Re-running an import against the same source project reuses what a previous run already
created, by name/content match, not by any record of which import created what.**
`import_record` makes one job's own retries and resumes safe; across two separate jobs