⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
fix(worker): keep distinct source issues with the same title apart (#474)
Thomas Gallawaya481f602b687
diff --git a/apps/worker/src/__tests__/integration/import-dedupe.test.ts b/apps/worker/src/__tests__/integration/import-dedupe.test.tsnew file mode 100644index 00000000..c9e8d506--- /dev/null+++ b/apps/worker/src/__tests__/integration/import-dedupe.test.ts@@ -0,0 +1,146 @@+import { describe, it, expect } from 'bun:test';+import { randomUUID } from 'node:crypto';+import {+ db,+ workspace,+ team,+ project,+ projectColumn,+ issue,+ issueActivity,+ importJob,+ user,+} from '@repo/db';+import { eq } from 'drizzle-orm';+import {+ createLocalIssueAndRecord,+ createLocalComment,+ findImportRecord,+ upsertImportRecord,+} from '../../import-store';++// Each test makes its own team, project and user, like import-store.test.ts. Jobs are+// 'paused' so no other test's claimDueImportJobs picks them up.++async function makeProject(): Promise<{ projectId: number; columnId: number; userId: string }> {+ const [ws] = await db+ .insert(workspace)+ .values({ name: 'Importers' })+ .returning({ id: workspace.id });+ const [teamRow] = await db+ .insert(team)+ .values({ workspaceId: ws!.id, name: 'Importers' })+ .returning({ id: team.id });+ const [projectRow] = await db+ .insert(project)+ .values({ teamId: teamRow!.id, key: randomUUID().slice(0, 8), name: 'Imported' })+ .returning({ id: project.id });+ const [columnRow] = await db+ .insert(projectColumn)+ .values({ projectId: projectRow!.id, name: 'Backlog', position: 1 })+ .returning({ id: projectColumn.id });+ const userId = randomUUID();+ await db.insert(user).values({ id: userId, name: 'Importer', email: `${userId}@example.test` });+ return { projectId: projectRow!.id, columnId: columnRow!.id, userId };+}++async function makeJob(projectId: number, userId: string): Promise<number> {+ const [row] = await db+ .insert(importJob)+ .values({ projectId, createdByUserId: userId, source: 'plane', status: 'paused' })+ .returning({ id: importJob.id });+ return row!.id;+}++function importIssue(+ jobId: number,+ sourceId: string,+ number: number,+ target: { projectId: number; columnId: number },+): Promise<number> {+ return createLocalIssueAndRecord(jobId, sourceId, String(number), {+ projectId: target.projectId,+ columnId: target.columnId,+ cycleId: null,+ parentId: null,+ assigneeUserId: null,+ title: 'Fix the login page',+ description: '',+ priority: null,+ startDate: null,+ dueDate: null,+ });+}++// What the Create phase does for one source comment: create or reuse, then map it.+async function importComment(jobId: number, sourceId: string, issueId: number): Promise<number> {+ const localId = await createLocalComment(+ jobId,+ issueId,+ null,+ 'Bot',+ 'Deployed.',+ new Date('2025-01-05T10:00:00.000Z'),+ null,+ );+ await upsertImportRecord(jobId, 'comment', sourceId, 'comment', localId);+ return localId;+}++describe('same-titled source issues', () => {+ it('creates one issue per source issue within a job, even when titles match', async () => {+ const target = await makeProject();+ const jobId = await makeJob(target.projectId, target.userId);++ const first = await importIssue(jobId, 'src-1', 1, target);+ const second = await importIssue(jobId, 'src-2', 2, target);++ expect(second).not.toBe(first);+ const rows = await db.select().from(issue).where(eq(issue.projectId, target.projectId));+ expect(rows).toHaveLength(2);+ });++ it('reuses each one in a later job, matched in import order', async () => {+ const target = await makeProject();+ const firstJob = await makeJob(target.projectId, target.userId);+ const first = await importIssue(firstJob, 'src-1', 1, target);+ const second = await importIssue(firstJob, 'src-2', 2, target);+ const rerun = await makeJob(target.projectId, target.userId);++ expect(await importIssue(rerun, 'src-1', 1, target)).toBe(first);+ expect(await importIssue(rerun, 'src-2', 2, target)).toBe(second);+ expect(await findImportRecord(rerun, 'issue', 'src-2')).toEqual({ localId: second });+ const rows = await db.select().from(issue).where(eq(issue.projectId, target.projectId));+ expect(rows).toHaveLength(2);+ });+});++describe('identical source comments', () => {+ it('creates one comment per source comment within a job, even with the same body and time', async () => {+ const target = await makeProject();+ const jobId = await makeJob(target.projectId, target.userId);+ const issueId = await importIssue(jobId, 'src-1', 1, target);++ const first = await importComment(jobId, 'comment-1', issueId);+ const second = await importComment(jobId, 'comment-2', issueId);++ expect(second).not.toBe(first);+ const rows = await db.select().from(issueActivity).where(eq(issueActivity.issueId, issueId));+ expect(rows).toHaveLength(2);+ });++ it('reuses each one in a later job', async () => {+ const target = await makeProject();+ const firstJob = await makeJob(target.projectId, target.userId);+ const issueId = await importIssue(firstJob, 'src-1', 1, target);+ const first = await importComment(firstJob, 'comment-1', issueId);+ const second = await importComment(firstJob, 'comment-2', issueId);+ const rerun = await makeJob(target.projectId, target.userId);+ expect(await importIssue(rerun, 'src-1', 1, target)).toBe(issueId);++ expect(await importComment(rerun, 'comment-1', issueId)).toBe(first);+ expect(await importComment(rerun, 'comment-2', issueId)).toBe(second);+ const rows = await db.select().from(issueActivity).where(eq(issueActivity.issueId, issueId));+ expect(rows).toHaveLength(2);+ });+});diff --git a/apps/worker/src/import-store.ts b/apps/worker/src/import-store.tsindex e3a86eec..59257949 100644--- a/apps/worker/src/import-store.ts+++ b/apps/worker/src/import-store.ts@@ -20,7 +20,7 @@ import {lockAttachmentStorage,type ImportSource,} from '@repo/db';-import { and, eq, gt, isNull, isNotNull, sql } from 'drizzle-orm';+import { and, eq, gt, isNull, isNotNull, sql, type Column } from 'drizzle-orm';import { decryptSecret, type EncryptedSecret } from '@repo/crypto';import {putObject,@@ -248,6 +248,19 @@ async function upsertImportRecordWith(});}+// True for a local row this job has not mapped yet. A row the same job already+// mapped belongs to another source record, so it is never that record's duplicate:+// two source issues may share a title, two comments a body and a timestamp. A row an+// earlier job mapped stays reusable, which is what makes a re-run not duplicate.+function notMappedByJob(jobId: number, entityType: ImportEntityType, localId: Column) {+ return sql`NOT EXISTS (+ SELECT 1 FROM ${importRecord}+ WHERE ${importRecord.importJobId} = ${jobId}+ AND ${importRecord.sourceEntityType} = ${entityType}+ AND ${importRecord.localId} = ${localId}+ )`;+}+export async function upsertImportRecord(jobId: number,entityType: ImportEntityType,@@ -524,8 +537,9 @@ export interface NewLocalIssue {}// An issue whose title matches one already in the project (case- and-// whitespace-insensitive, the same comparison the file-based importer uses)-// is reused instead of duplicated — the same protection createLocalLabel and+// whitespace-insensitive, the same comparison the file-based importer uses), and+// that this job has not already mapped to another source issue, is reused+// instead of duplicated — the same protection createLocalLabel and// createLocalStateAndRecord give their own entities. Comments and labels the// import attaches afterward land on that existing issue rather than being// orphaned. Skipped for a blank title: input.title falls back to@@ -556,8 +570,11 @@ export async function createLocalIssueAndRecord(and(eq(issue.projectId, input.projectId),sql`lower(btrim(${issue.title})) = lower(btrim(${input.title}))`,+ notMappedByJob(jobId, 'issue', issue.id),),- );+ )+ .orderBy(issue.id)+ .limit(1);if (existing) {await upsertImportRecordWith(tx,@@ -611,12 +628,13 @@ export async function setIssueLabels(issueId: number, labelIds: number[]): Promi}// A comment already on the issue with the same body and createdAt (the two-// fields Plane's own record carries verbatim) is reused instead of-// duplicated — the same reuse-by-content protection issues, states, and-// cycles get, needed here because a second import job resolves the parent-// issue to the same, already-created row and would otherwise re-post every-// comment on it.+// fields Plane's own record carries verbatim), and not already mapped by this+// job, is reused instead of duplicated — the same reuse-by-content protection+// issues, states, and cycles get, needed here because a second import job+// resolves the parent issue to the same, already-created row and would otherwise+// re-post every comment on it.export async function createLocalComment(+ jobId: number,issueId: number,authorUserId: string | null,authorName: string,@@ -633,8 +651,11 @@ export async function createLocalComment(eq(issueActivity.kind, 'comment'),eq(issueActivity.body, bodyMarkdown),eq(issueActivity.createdAt, createdAt),+ notMappedByJob(jobId, 'comment', issueActivity.id),),- );+ )+ .orderBy(issueActivity.id)+ .limit(1);if (existing) return existing.id;const [row] = await dbdiff --git a/apps/worker/src/import-worker.ts b/apps/worker/src/import-worker.tsindex 900e0dd6..81899b9c 100644--- a/apps/worker/src/import-worker.ts+++ b/apps/worker/src/import-worker.ts@@ -330,6 +330,7 @@ async function createComments(? (localIdBySourceId.get(comment.replyToSourceId) ?? null): null;const localId = await createLocalComment(+ job.id,issueLocalId,authorUserId,comment.authorName,diff --git a/docs/dev/plane-import.md b/docs/dev/plane-import.mdindex 2e8a8f35..040c2476 100644--- a/docs/dev/plane-import.md+++ b/docs/dev/plane-import.md@@ -140,7 +140,10 @@ attachment) are a distinct path, not covered by this — see "Inline images" invia its own `(project_id, name)` unique constraint. A match is reused, never overwritten:an existing column's `stateType` is left as Plane's category disagrees with it, forinstance. This is a content match, not provenance tracking, so it also matches content a- user created by hand before importing, not only a previous import's output. `issue_link`+ user created by hand before importing, not only a previous import's output. An issue or+ comment the same job already mapped is excluded from the lookup (`notMappedByJob`): it+ belongs to another source record, so two source issues with one title, or two comments+ with one body and timestamp, stay apart within a job. `issue_link`needs no equivalent logic — its `(pair, kind)` unique index rejects the duplicate insertoutright, caught by `isUniqueViolation` in `createIssueLink`.State/cycle name matching is exact (case-sensitive); issue title matching is not. Nothingdiff --git a/docs/importing-from-plane.md b/docs/importing-from-plane.mdindex 03e688cd..7bcbe62a 100644--- a/docs/importing-from-plane.md+++ b/docs/importing-from-plane.md@@ -105,7 +105,10 @@ categorizes it differently.This is a name/content match, not a record of which import created what, so it also meansan issue you created by hand before importing — one that happens to share an exact titlewith something in Plane — is treated as the same issue and gets Plane's labels, comments,-and other fields attached to it rather than getting a second copy.+and other fields attached to it rather than getting a second copy. Within one import, two+different Plane issues with the same title stay two issues, and two comments with the same+text and time stay two comments: what an import has already matched is never matched again+by that same import.## Exporting your project's data