⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
fix(worker): resume incomplete import Create batches (#654)
I'm Groot 🌳d2ad9d67dddb
diff --git a/apps/worker/src/__tests__/integration/import-create.test.ts b/apps/worker/src/__tests__/integration/import-create.test.tsindex 5d68bdfd..b4778a12 100644--- a/apps/worker/src/__tests__/integration/import-create.test.ts+++ b/apps/worker/src/__tests__/integration/import-create.test.ts@@ -9,10 +9,19 @@ import {issue,issueActivity,importJob,+ importRecord,+ issueLabel,user,} from '@repo/db';import { and, eq } from 'drizzle-orm';-import { insertDiscoveredIds, type ClaimedImportJob } from '../../import-store';+import {+ insertDiscoveredIds,+ createLocalIssueAndRecord,+ createLocalComment,+ upsertImportRecord,+ findImportRecord,+ type ClaimedImportJob,+} from '../../import-store';import { runCreate } from '../../import-worker';import type { SourceReader } from '../../reader';import type { CanonicalComment, CanonicalIssue } from '../../canonical';@@ -133,6 +142,41 @@ function readerWithFlakyComments(): SourceReader {return reader as SourceReader;}+function readerForIssues(issues: CanonicalIssue[]): SourceReader {+ return {+ listStates: async () => [],+ listLabels: async () => [],+ listCycles: async () => [],+ listIssues: async () => ({ items: issues, cursor: null }),+ getIssue: async (id) => issues.find((item) => item.sourceId === id)!,+ listIssueRelations: async () => [],+ listIssueComments: async () => COMMENTS,+ listIssueAttachments: async () => [],+ resolveAttachmentDownload: async () => {+ throw new Error('Create must not download attachments');+ },+ };+}++async function seedMappedIssue(job: ClaimedImportJob): Promise<number> {+ const [column] = await db+ .select({ id: projectColumn.id })+ .from(projectColumn)+ .where(eq(projectColumn.projectId, job.projectId));+ return createLocalIssueAndRecord(job.id, ISSUE.sourceId, String(ISSUE.sequenceId), {+ projectId: job.projectId,+ columnId: column!.id,+ cycleId: null,+ parentId: null,+ assigneeUserId: null,+ title: 'Locally edited title',+ description: 'Keep local description',+ priority: null,+ startDate: null,+ dueDate: null,+ });+}+describe('runCreate', () => {it("creates an issue's comments on the retry after its comment read failed", async () => {const job = await makeCreateJob();@@ -153,4 +197,135 @@ describe('runCreate', () => {.where(and(eq(issueActivity.issueId, issues[0]!.id), eq(issueActivity.kind, 'comment')));expect(comments.map((c) => c.body).sort()).toEqual(['First', 'Second']);});++ it('finishes labels, comments and attachment discovery after an issue mapping committed', async () => {+ const job = await makeCreateJob();+ await insertDiscoveredIds(job.id, 'issue', [ISSUE.sourceId]);+ await insertDiscoveredIds(job.id, 'label', ['label-1']);+ const localId = await seedMappedIssue(job);+ const reader = readerForIssues([{ ...ISSUE, labelSourceIds: ['label-1'] }]);+ reader.listLabels = async () => [{ sourceId: 'label-1', name: 'Imported label' }];+ reader.listIssueAttachments = async () => [+ {+ sourceId: 'attachment-1',+ filename: 'note.txt',+ contentType: 'text/plain',+ sizeBytes: 4,+ },+ ];++ await runCreate(job, reader);+ const [savedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ await runCreate({ ...job, cursor: savedJob!.cursor as ClaimedImportJob['cursor'] }, reader);++ const issues = await db.select().from(issue).where(eq(issue.projectId, job.projectId));+ expect(issues).toHaveLength(1);+ expect(issues[0]!.id).toBe(localId);+ expect(issues[0]!.title).toBe('Locally edited title');+ expect(issues[0]!.description).toBe('Keep local description');+ const labels = await db.select().from(issueLabel).where(eq(issueLabel.issueId, localId));+ expect(labels).toHaveLength(1);+ const comments = await db+ .select()+ .from(issueActivity)+ .where(and(eq(issueActivity.issueId, localId), eq(issueActivity.kind, 'comment')));+ expect(comments.map((comment) => comment.body).sort()).toEqual(['First', 'Second']);+ expect(await findImportRecord(job.id, 'attachment', 'attachment-1')).toEqual({ localId: null });+ });++ it('reuses an unmapped comment after a crash and keeps identical source comments distinct', async () => {+ const job = await makeCreateJob();+ await insertDiscoveredIds(job.id, 'issue', [ISSUE.sourceId]);+ const localId = await seedMappedIssue(job);+ const first = COMMENTS[0]!;+ const firstId = await createLocalComment(+ job.id,+ localId,+ null,+ first.authorName,+ first.bodyMarkdown,+ new Date(first.createdAt),+ null,+ );+ await upsertImportRecord(job.id, 'comment', first.sourceId, 'comment', firstId);+ const unmappedId = await createLocalComment(+ job.id,+ localId,+ null,+ first.authorName,+ first.bodyMarkdown,+ new Date(first.createdAt),+ null,+ );+ const reader = readerForIssues([ISSUE]);+ reader.listIssueComments = async () => [+ first,+ { ...first, sourceId: 'identical-comment' },+ { ...COMMENTS[1]!, replyToSourceId: 'identical-comment' },+ ];++ await runCreate(job, reader);+ const [savedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ await runCreate({ ...job, cursor: savedJob!.cursor as ClaimedImportJob['cursor'] }, reader);++ expect(await findImportRecord(job.id, 'comment', first.sourceId)).toEqual({ localId: firstId });+ expect(await findImportRecord(job.id, 'comment', 'identical-comment')).toEqual({+ localId: unmappedId,+ });+ const comments = await db+ .select()+ .from(issueActivity)+ .where(and(eq(issueActivity.issueId, localId), eq(issueActivity.kind, 'comment')));+ expect(comments).toHaveLength(3);+ expect(comments.find((comment) => comment.body === 'Second')!.replyToId).toBe(unmappedId);+ });++ it('replays an unfinished batch and advances only beyond fully completed batches', async () => {+ const job = await makeCreateJob();+ const issues = Array.from({ length: 16 }, (_, index) => ({+ ...ISSUE,+ sourceId: `batch-${index}`,+ sequenceId: index + 1,+ }));+ await insertDiscoveredIds(+ job.id,+ 'issue',+ issues.map((item) => item.sourceId),+ );+ const reader = readerForIssues(issues);+ reader.listIssueComments = async () => [];+ let fail = true;+ const reads: string[] = [];+ reader.getIssue = async (id) => {+ reads.push(id);+ if (id === 'batch-1' && fail) throw new Error('interrupted batch');+ return issues.find((item) => item.sourceId === id)!;+ };+ await expect(runCreate(job, reader)).rejects.toThrow('interrupted batch');+ const [failedJob] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ expect(failedJob!.cursor).toEqual({});+ fail = false;+ reads.length = 0;+ await runCreate(job, reader);+ expect(reads).toHaveLength(15);+ expect(reads[0]).toBe('batch-0');+ const [saved] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ const records = await db+ .select()+ .from(importRecord)+ .where(and(eq(importRecord.importJobId, job.id), eq(importRecord.sourceEntityType, 'issue')))+ .orderBy(importRecord.id);+ expect(saved!.cursor).toEqual({ lastRecordId: records[14]!.id });+ job.cursor = saved!.cursor as ClaimedImportJob['cursor'];+ reads.length = 0;+ await runCreate(job, reader);+ expect(reads).toEqual(['batch-15']);+ const [last] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ job.cursor = last!.cursor as ClaimedImportJob['cursor'];+ await runCreate(job, reader);+ const [finished] = await db.select().from(importJob).where(eq(importJob.id, job.id));+ expect(finished!.phase).toBe('link');+ const imported = await db.select().from(issue).where(eq(issue.projectId, job.projectId));+ expect(imported).toHaveLength(16);+ });});diff --git a/apps/worker/src/import-store.ts b/apps/worker/src/import-store.tsindex e2004c6e..5f349667 100644--- a/apps/worker/src/import-store.ts+++ b/apps/worker/src/import-store.ts@@ -320,6 +320,26 @@ export interface ImportRecordRow {sourceId: string;}+export async function listImportRecords(+ jobId: number,+ entityType: ImportEntityType,+ afterId: number,+ limit: number,+): Promise<ImportRecordRow[]> {+ return db+ .select({ id: importRecord.id, sourceId: importRecord.sourceId })+ .from(importRecord)+ .where(+ and(+ eq(importRecord.importJobId, jobId),+ eq(importRecord.sourceEntityType, entityType),+ gt(importRecord.id, afterId),+ ),+ )+ .orderBy(importRecord.id)+ .limit(limit);+}+// The next chunk of this job's snapshot that has not been materialized yet,// keyset-paginated on import_record's own id — itsaplan's own position in the// snapshot list, not the source's pagination cursor (see the notes file). Adiff --git a/apps/worker/src/import-worker.ts b/apps/worker/src/import-worker.tsindex d8f78d27..7b49f59a 100644--- a/apps/worker/src/import-worker.ts+++ b/apps/worker/src/import-worker.ts@@ -19,6 +19,7 @@ import {upsertImportRecord,findImportRecord,findImportRecordByDisplayId,+ listImportRecords,listUncreatedImportRecords,listCreatedImportRecords,firstProjectColumnId,@@ -152,7 +153,7 @@ async function runDiscover(job: ClaimedImportJob, reader: SourceReader): Promise}// --- Create: materialize states/labels/cycles once (cheap, unpaginated),-// then create one chunk of not-yet-created issues per tick, with their+// then finish one chunk of issues per tick, with their// comments and their attachment metadata (byte download is a later phase).interface RecordCursor {@@ -166,7 +167,7 @@ export async function runCreate(job: ClaimedImportJob, reader: SourceReader): Prconst cursor = job.cursor as Partial<RecordCursor>;const afterId = cursor.lastRecordId ?? 0;- const pending = await listUncreatedImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);+ const pending = await listImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);if (pending.length === 0) {await advanceImportJobPhase(job.id, 'link', {});return;@@ -227,46 +228,48 @@ async function createOneIssue(sourceId: string,): Promise<void> {const canonical = await reader.getIssue(sourceId);- // Read before the issue and its mapping are written: a failed read then leaves- // the issue unmapped, and the next tick retries the whole issue.+ // Fetch source data before writing local rows; a failed read leaves no partial writes.const comments = await reader.listIssueComments(sourceId);const attachments = await reader.listIssueAttachments(sourceId);- const stateRecord = await findImportRecord(job.id, 'state', canonical.stateSourceId);- const columnId = stateRecord?.localId ?? (await firstProjectColumnId(job.projectId));- if (columnId == null)- throw new Error(`project ${job.projectId} has no workflow column to import into`);-- const cycleRecord = canonical.cycleSourceId- ? await findImportRecord(job.id, 'cycle', canonical.cycleSourceId)- : null;- const parentRecord = canonical.parentSourceId- ? await findImportRecord(job.id, 'issue', canonical.parentSourceId)- : null;- const assigneeUserId = canonical.assigneeEmail- ? await findProjectMemberUserId(job.projectId, canonical.assigneeEmail)- : null;+ const existing = await findImportRecord(job.id, 'issue', sourceId);+ let localId = existing?.localId;+ if (localId == null) {+ const stateRecord = await findImportRecord(job.id, 'state', canonical.stateSourceId);+ const columnId = stateRecord?.localId ?? (await firstProjectColumnId(job.projectId));+ if (columnId == null)+ throw new Error(`project ${job.projectId} has no workflow column to import into`);+ const cycleRecord = canonical.cycleSourceId+ ? await findImportRecord(job.id, 'cycle', canonical.cycleSourceId)+ : null;+ const parentRecord = canonical.parentSourceId+ ? await findImportRecord(job.id, 'issue', canonical.parentSourceId)+ : null;+ const assigneeUserId = canonical.assigneeEmail+ ? await findProjectMemberUserId(job.projectId, canonical.assigneeEmail)+ : null;++ localId = await createLocalIssueAndRecord(job.id, sourceId, String(canonical.sequenceId), {+ projectId: job.projectId,+ columnId,+ cycleId: cycleRecord?.localId ?? null,+ parentId: parentRecord?.localId ?? null,+ assigneeUserId,+ title: canonical.title,+ description: canonical.descriptionMarkdown,+ priority: canonical.priority,+ startDate: canonical.startDate,+ dueDate: canonical.dueDate,+ createdAt: canonical.createdAt,+ updatedAt: canonical.updatedAt,+ });+ }const labelIds: number[] = [];for (const labelSourceId of canonical.labelSourceIds) {const record = await findImportRecord(job.id, 'label', labelSourceId);if (record?.localId != null) labelIds.push(record.localId);}-- const localId = await createLocalIssueAndRecord(job.id, sourceId, String(canonical.sequenceId), {- projectId: job.projectId,- columnId,- cycleId: cycleRecord?.localId ?? null,- parentId: parentRecord?.localId ?? null,- assigneeUserId,- title: canonical.title,- description: canonical.descriptionMarkdown,- priority: canonical.priority,- startDate: canonical.startDate,- dueDate: canonical.dueDate,- createdAt: canonical.createdAt,- updatedAt: canonical.updatedAt,- });if (labelIds.length) await setIssueLabels(localId, labelIds);await createComments(job, localId, comments);diff --git a/docs/dev/plane-import.md b/docs/dev/plane-import.mdindex 6f8b13cf..781f02e8 100644--- a/docs/dev/plane-import.md+++ b/docs/dev/plane-import.md@@ -130,6 +130,10 @@ attachment) are a distinct path, not covered by this — see "Inline images" in(`plane-adapter.ts`) has no entry for `start_before`/`start_after`/`finish_before`/`finish_after` — itsaplan's `issue_link.kind` has no scheduling-dependency concept, sothese are filtered out, not a bug.+- Create advances its local record cursor after the entire batch finishes, including labels,+ comments and attachment discovery. A retry revisits every issue after that cursor and reuses+ its existing job/source mapping, then finishes its child records without replacing local+ issue fields. Completed batches are not fetched again.- **Re-running an import against the same source project reuses what a previous run alreadycreated, by name/content match, not by any record of which import created what.**`import_record` makes one job's own retries and resumes safe; across two separate jobs