⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
import { getStorageSettings, mimeAllowed, MB } from '@repo/db';
import { startPollLoop, type WorkerHandle } from './poll-loop';
import { intEnv } from './env';
import { tickErrorOutcome } from './import-retry';
import { readerForJob, sourceProjectKeyForJob, type CommonImportConfig } from './import-sources';
import type { SourceReader } from './reader';
import type { CanonicalChecklist, CanonicalComment, CanonicalIssue } from './canonical';
import { extractCrossReferences, applyCrossReferenceReplacements } from './cross-reference';
import { downloadAttachment } from './attachment-download';
import {
claimDueImportJobs,
decryptImportCredential,
saveImportJobCursor,
advanceImportJobPhase,
completeImportJob,
failImportJob,
retryImportJobLater,
insertDiscoveredIds,
upsertImportRecord,
findImportRecord,
findImportRecordByDisplayId,
listImportRecords,
listUncreatedImportRecords,
listCreatedImportRecords,
firstProjectColumnId,
getProjectKey,
getIssueSequenceNumber,
getIssueTextForRewrite,
updateIssueDescription,
updateCommentBody,
createLocalStateAndRecord,
createLocalLabel,
createLocalCycleAndRecord,
createLocalIssueAndRecord,
setIssueLabels,
addIssueWatchers,
createLocalCustomFieldAndRecord,
setIssueCustomFieldValue,
createLocalChecklistAndRecord,
setIssueParentIfUnset,
createLocalComment,
createIssueLink,
findProjectMemberUserId,
createLocalAttachmentAndRecord,
AttachmentRejectedError,
type ClaimedImportJob,
} from './import-store';
// The poll loop that drives a source import through its phases: discover ->
// create -> link -> rewrite -> attachments -> done. Follows the same claim-a-lease,
// bounded-chunk-per-tick shape as worker.ts and agent-worker.ts. Each tick
// claims at most one due import_job and advances it by one bounded unit of
// work, then reschedules — a large import interleaves across many ticks
// rather than running to completion in one.
export function startImportWorker(): WorkerHandle {
return startPollLoop('import-worker', tick, () => intEnv('IMPORT_POLL_INTERVAL_MS', 3000));
}
// How many issues Create/Link process per tick. Each issue costs several
// additional Plane requests (comments, attachments, relations) against a
// ~60 requests/minute budget, so this stays well under the discover page size.
const ISSUES_PER_TICK = 15;
async function tick(): Promise<void> {
const [job] = await claimDueImportJobs();
if (!job) return;
try {
const reader = buildReader(job);
switch (job.phase) {
case 'discover':
await runDiscover(job, reader);
break;
case 'create':
await runCreate(job, reader);
break;
case 'link':
await runLink(job, reader);
break;
case 'rewrite':
await runRewrite(job);
break;
case 'attachments':
await runAttachments(job, reader);
break;
default:
await completeImportJob(job.id);
}
} catch (error) {
await handleTickError(job, error);
}
}
async function handleTickError(job: ClaimedImportJob, error: unknown): Promise<void> {
const outcome = tickErrorOutcome(error, job.attempts);
if (outcome.action === 'fail') {
await failImportJob(job.id, outcome.lastError);
return;
}
await retryImportJobLater(job.id, outcome.delayMs, outcome.lastError, outcome.countsAsAttempt);
}
function buildReader(job: ClaimedImportJob): SourceReader {
return readerForJob(job, decryptImportCredential(job));
}
// --- Discover: snapshot every source id up front, writing import_record rows
// with no local mapping yet. Plane's own pagination cursor is not a safe
// resumability anchor across a job that can span many ticks (see "Pagination"
// in docs/dev/plane-import-source-notes.md) — Create resumes from this
// snapshot instead, keyed on import_record's own id.
interface DiscoverCursor {
subphase: 'meta' | 'issues';
planeCursor: string | null;
}
async function runDiscover(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const cursor = job.cursor as Partial<DiscoverCursor>;
if (cursor.subphase !== 'issues') {
const [states, labels, cycles, customFields] = await Promise.all([
reader.listStates(),
reader.listLabels(),
reader.listCycles(),
reader.listCustomFields(),
]);
await insertDiscoveredIds(
job.id,
'state',
states.map((s) => s.sourceId),
);
await insertDiscoveredIds(
job.id,
'label',
labels.map((l) => l.sourceId),
);
await insertDiscoveredIds(
job.id,
'cycle',
cycles.map((c) => c.sourceId),
);
await insertDiscoveredIds(
job.id,
'custom_field',
customFields.map((f) => f.sourceId),
);
const next: DiscoverCursor = { subphase: 'issues', planeCursor: null };
await saveImportJobCursor(job.id, next);
return;
}
const page = await reader.listIssues(cursor.planeCursor ?? null);
await insertDiscoveredIds(
job.id,
'issue',
page.items.map((i) => i.sourceId),
);
if (page.cursor === null) {
await advanceImportJobPhase(job.id, 'create', {});
return;
}
const next: DiscoverCursor = { subphase: 'issues', planeCursor: page.cursor };
await saveImportJobCursor(job.id, next);
}
// --- Create: materialize states/labels/cycles/custom fields once (cheap, unpaginated),
// then finish one chunk of issues per tick, with their
// comments and their attachment metadata (byte download is a later phase).
interface RecordCursor {
lastRecordId: number;
}
export async function runCreate(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
await materializeStates(job, reader);
await materializeLabels(job, reader);
await materializeCycles(job, reader);
await materializeCustomFields(job, reader);
const cursor = job.cursor as Partial<RecordCursor>;
const afterId = cursor.lastRecordId ?? 0;
const pending = await listImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
if (pending.length === 0) {
await advanceImportJobPhase(job.id, 'link', {});
return;
}
for (const record of pending) {
await createOneIssue(job, reader, record.sourceId);
}
const next: RecordCursor = { lastRecordId: pending[pending.length - 1]!.id };
await saveImportJobCursor(job.id, next);
}
// stateOverrides (Plane state id -> itsaplan category) comes from the mapping review
// step at job creation; only newly-created columns are affected, never a dedup-matched
// existing one - createLocalStateAndRecord already never touches an existing match's
// stateType.
async function materializeStates(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const pending = await listUncreatedImportRecords(job.id, 'state', 0, 1000);
if (pending.length === 0) return;
const overrides = (job.config as CommonImportConfig).stateOverrides ?? {};
const states = new Map((await reader.listStates()).map((s) => [s.sourceId, s]));
for (const record of pending) {
const state = states.get(record.sourceId);
if (!state) continue;
const category = overrides[record.sourceId] ?? state.category;
await createLocalStateAndRecord(job.id, record.sourceId, job.projectId, {
...state,
category,
});
}
}
async function materializeLabels(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const pending = await listUncreatedImportRecords(job.id, 'label', 0, 1000);
if (pending.length === 0) return;
const labels = new Map((await reader.listLabels()).map((l) => [l.sourceId, l]));
for (const record of pending) {
const label = labels.get(record.sourceId);
if (!label) continue;
const localId = await createLocalLabel(job.projectId, label);
await upsertImportRecord(job.id, 'label', record.sourceId, 'label', localId);
}
}
async function materializeCycles(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const pending = await listUncreatedImportRecords(job.id, 'cycle', 0, 1000);
if (pending.length === 0) return;
const cycles = new Map((await reader.listCycles()).map((c) => [c.sourceId, c]));
for (const record of pending) {
const cycle = cycles.get(record.sourceId);
if (!cycle) continue;
await createLocalCycleAndRecord(job.id, record.sourceId, job.projectId, cycle);
}
}
async function materializeCustomFields(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const pending = await listUncreatedImportRecords(job.id, 'custom_field', 0, 1000);
if (pending.length === 0) return;
const fields = new Map((await reader.listCustomFields()).map((f) => [f.sourceId, f]));
for (const record of pending) {
const field = fields.get(record.sourceId);
if (!field) continue;
await createLocalCustomFieldAndRecord(job.id, record.sourceId, job.projectId, field);
}
}
async function createOneIssue(
job: ClaimedImportJob,
reader: SourceReader,
sourceId: string,
): Promise<void> {
const canonical = await reader.getIssue(sourceId);
// Fetch source data before writing local rows; a failed read leaves no partial writes.
const comments = await reader.listIssueComments(sourceId);
const attachments = await reader.listIssueAttachments(sourceId);
const checklists = await reader.listIssueChecklists(sourceId);
const existing = await findImportRecord(job.id, 'issue', sourceId);
let localId = existing?.localId;
if (localId == null) {
const stateRecord = await findImportRecord(job.id, 'state', canonical.stateSourceId);
const columnId = stateRecord?.localId ?? (await firstProjectColumnId(job.projectId));
if (columnId == null)
throw new Error(`project ${job.projectId} has no workflow column to import into`);
const cycleRecord = canonical.cycleSourceId
? await findImportRecord(job.id, 'cycle', canonical.cycleSourceId)
: null;
const parentRecord = canonical.parentSourceId
? await findImportRecord(job.id, 'issue', canonical.parentSourceId)
: null;
const assigneeUserId = canonical.assigneeEmail
? await findProjectMemberUserId(job.projectId, canonical.assigneeEmail)
: null;
localId = await createLocalIssueAndRecord(job.id, sourceId, String(canonical.sequenceId), {
projectId: job.projectId,
columnId,
cycleId: cycleRecord?.localId ?? null,
parentId: parentRecord?.localId ?? null,
assigneeUserId,
title: canonical.title,
description: canonical.descriptionMarkdown,
priority: canonical.priority,
startDate: canonical.startDate,
dueDate: canonical.dueDate,
createdAt: canonical.createdAt,
updatedAt: canonical.updatedAt,
archivedAt: canonical.archivedAt,
});
}
const labelIds: number[] = [];
for (const labelSourceId of canonical.labelSourceIds) {
const record = await findImportRecord(job.id, 'label', labelSourceId);
if (record?.localId != null) labelIds.push(record.localId);
}
if (labelIds.length) await setIssueLabels(localId, labelIds);
await addWatchers(job, localId, canonical.watcherEmails);
await setCustomFieldValues(job, localId, canonical.customFields);
await createChecklists(job, localId, checklists);
await createComments(job, localId, comments);
if (attachments.length) {
await insertDiscoveredIds(
job.id,
'attachment',
attachments.map((a) => a.sourceId),
);
}
}
async function addWatchers(
job: ClaimedImportJob,
issueLocalId: number,
emails: string[],
): Promise<void> {
const userIds: string[] = [];
for (const email of emails) {
const userId = await findProjectMemberUserId(job.projectId, email);
if (userId) userIds.push(userId);
}
await addIssueWatchers(issueLocalId, userIds);
}
async function setCustomFieldValues(
job: ClaimedImportJob,
issueLocalId: number,
values: CanonicalIssue['customFields'],
): Promise<void> {
for (const { fieldSourceId, value } of values) {
if (value === null || value === '') continue;
const record = await findImportRecord(job.id, 'custom_field', fieldSourceId);
if (record?.localId == null) continue;
await setIssueCustomFieldValue(issueLocalId, record.localId, value);
}
}
async function createChecklists(
job: ClaimedImportJob,
issueLocalId: number,
checklists: CanonicalChecklist[],
): Promise<void> {
for (const checklist of checklists) {
if ((await findImportRecord(job.id, 'checklist', checklist.sourceId))?.localId != null)
continue;
await createLocalChecklistAndRecord(job.id, issueLocalId, checklist);
}
}
// Comments are recorded in import_record too (entity type 'comment'), so a
// crashed-and-retried tick does not insert the same comment twice — issue_
// activity carries no unique constraint of its own to fall back on.
// Processed in the order Plane returned them, which is assumed chronological,
// so a reply's parent is created before the reply itself.
async function createComments(
job: ClaimedImportJob,
issueLocalId: number,
comments: CanonicalComment[],
): Promise<void> {
const unmatchedUserPolicy = (job.config as CommonImportConfig).unmatchedUserPolicy;
const localIdBySourceId = new Map<string, number>();
for (const comment of comments) {
const existing = await findImportRecord(job.id, 'comment', comment.sourceId);
if (existing?.localId != null) {
localIdBySourceId.set(comment.sourceId, existing.localId);
continue;
}
const authorUserId = comment.authorEmail
? await findProjectMemberUserId(job.projectId, comment.authorEmail)
: null;
// 'skip' drops a comment whose author matched no project member, rather than
// creating it unattributed. A comment with no authorEmail at all (Plane gave no
// address to match) is unaffected - there was never a member to match against.
if (comment.authorEmail && !authorUserId && unmatchedUserPolicy === 'skip') continue;
const replyToId = comment.replyToSourceId
? (localIdBySourceId.get(comment.replyToSourceId) ?? null)
: null;
const localId = await createLocalComment(
job.id,
issueLocalId,
authorUserId,
comment.authorName,
comment.bodyMarkdown,
new Date(comment.createdAt),
replyToId,
);
await upsertImportRecord(job.id, 'comment', comment.sourceId, 'comment', localId);
localIdBySourceId.set(comment.sourceId, localId);
}
}
// --- Link: for each created issue, fetch its relations and create the ones
// with a destination in itsaplan's issue_link.kind (see "Relations vs.
// dependencies" in the notes file). Also gives every issue's parent link a
// second chance to resolve: Create only knew what import_record held at the
// moment it ran, so a sub-issue processed before its parent existed was left
// with no parent — by Link, every issue in the job has been created.
async function runLink(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const cursor = job.cursor as Partial<RecordCursor>;
const afterId = cursor.lastRecordId ?? 0;
const pending = await listCreatedImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
if (pending.length === 0) {
await advanceImportJobPhase(job.id, 'rewrite', {});
return;
}
for (const record of pending) {
const canonical = await reader.getIssue(record.sourceId);
if (canonical.parentSourceId) {
const parentRecord = await findImportRecord(job.id, 'issue', canonical.parentSourceId);
if (parentRecord?.localId != null) {
await setIssueParentIfUnset(record.localId, parentRecord.localId);
}
}
const relations = await reader.listIssueRelations(record.sourceId);
for (const relation of relations) {
const targetRecord = await findImportRecord(job.id, 'issue', relation.targetIssueSourceId);
if (targetRecord?.localId == null) continue;
await createIssueLink(record.localId, targetRecord.localId, relation.kind);
}
}
const next: RecordCursor = { lastRecordId: pending[pending.length - 1]!.id };
await saveImportJobCursor(job.id, next);
}
// --- Rewrite: a description or comment that mentions the source's own issue
// number ("ROOMS-524") still says that after Create — this phase resolves any
// such mention that lands inside the imported set and rewrites it to this
// project's own identifier. Purely local: everything it needs was captured
// during Create/Link, so it makes no further Plane requests.
// A mention of the issue whose own text this is stays as written: it is how a
// source records the issue's own identifier ("Imported from Linear: ATO-505").
async function rewriteCrossReferences(
jobId: number,
sourceProjectKey: string,
localProjectKey: string,
issueLocalId: number,
text: string,
): Promise<string> {
const references = extractCrossReferences(text, sourceProjectKey);
if (references.length === 0) return text;
const replacements = new Map<string, string>();
for (const { reference, sequenceId } of references) {
const record = await findImportRecordByDisplayId(jobId, 'issue', sequenceId);
if (record?.localId == null || record.localId === issueLocalId) continue;
const targetSequence = await getIssueSequenceNumber(record.localId);
if (targetSequence == null) continue;
replacements.set(reference, `${localProjectKey}-${targetSequence}`);
}
return applyCrossReferenceReplacements(text, sourceProjectKey, replacements);
}
export async function runRewrite(job: ClaimedImportJob): Promise<void> {
const cursor = job.cursor as Partial<RecordCursor>;
const afterId = cursor.lastRecordId ?? 0;
const pending = await listCreatedImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
if (pending.length === 0) {
await advanceImportJobPhase(job.id, 'attachments', {});
return;
}
// A job created before this phase existed has no source project key — its
// description/comment text is left exactly as Create wrote it.
const sourceProjectKey = sourceProjectKeyForJob(job);
if (sourceProjectKey) {
const localProjectKey = await getProjectKey(job.projectId);
for (const record of pending) {
const { description, comments } = await getIssueTextForRewrite(record.localId);
const nextDescription = await rewriteCrossReferences(
job.id,
sourceProjectKey,
localProjectKey,
record.localId,
description,
);
if (nextDescription !== description) {
await updateIssueDescription(record.localId, nextDescription);
}
for (const comment of comments) {
const nextBody = await rewriteCrossReferences(
job.id,
sourceProjectKey,
localProjectKey,
record.localId,
comment.body,
);
if (nextBody !== comment.body) await updateCommentBody(comment.id, nextBody);
}
}
}
const next: RecordCursor = { lastRecordId: pending[pending.length - 1]!.id };
await saveImportJobCursor(job.id, next);
}
// --- Attachments: metadata (filename, content type, size) was already listed
// per-issue during Create, but never the bytes — those need a fresh
// resolve-then-fetch of the two-hop, hour-lived S3 redirect right before each
// download (see "Attachments" in the notes file), so this phase re-lists each
// issue's attachments rather than reusing anything captured earlier.
async function runAttachments(job: ClaimedImportJob, reader: SourceReader): Promise<void> {
const cursor = job.cursor as Partial<RecordCursor>;
const afterId = cursor.lastRecordId ?? 0;
const pending = await listCreatedImportRecords(job.id, 'issue', afterId, ISSUES_PER_TICK);
if (pending.length === 0) {
await completeImportJob(job.id);
return;
}
const limits = await getStorageSettings();
for (const record of pending) {
const attachments = await reader.listIssueAttachments(record.sourceId);
for (const attachment of attachments) {
if ((await findImportRecord(job.id, 'attachment', attachment.sourceId))?.localId != null) {
continue;
}
try {
// Plane's own advertised size/type skip an obviously-too-large or
// disallowed file before spending a download on it; the authoritative
// check is still the real downloaded byte count, in
// createLocalAttachmentAndRecord.
if (attachment.sizeBytes > limits.maxAttachmentMb * MB) {
throw new AttachmentRejectedError(
`"${attachment.filename}" is ${Math.ceil(attachment.sizeBytes / MB)} MB, over the ${limits.maxAttachmentMb} MB limit`,
);
}
if (!mimeAllowed(attachment.contentType, limits.attachmentMimeTypes)) {
throw new AttachmentRejectedError(
`"${attachment.filename}" is type "${attachment.contentType}", not accepted on this instance`,
);
}
const download = await reader.resolveAttachmentDownload(
record.sourceId,
attachment.sourceId,
);
const bytes = await downloadAttachment(download, limits.maxAttachmentMb * MB);
await createLocalAttachmentAndRecord(job.id, attachment.sourceId, {
projectId: job.projectId,
issueId: record.localId,
filename: attachment.filename,
contentType: attachment.contentType,
bytes,
});
} catch (error) {
if (!(error instanceof AttachmentRejectedError)) throw error;
console.error(
`[import ${job.id}] attachment ${attachment.sourceId} rejected:`,
error.message,
);
}
}
}
const next: RecordCursor = { lastRecordId: pending[pending.length - 1]!.id };
await saveImportJobCursor(job.id, next);
}