Skip to content

πŸ“¨ The ultimate agentic social media scheduling tool πŸ€–

feat: pending + finalize for more platforms

Nevo Davide287a14dd812
diff --git a/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts b/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts
index e5b02faed..33866c6c7 100644
--- a/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts
@@ -1,5 +1,6 @@
import {
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -65,10 +66,14 @@ async function reduceImageBySize(url: string, maxSizeKB = 976) {
}
}
-async function uploadVideo(
+// Streams the video bytes to the Bluesky video service and returns the job
+// status - it does NOT wait for the processing. The blob only exists once the
+// job completes; a BlobRef can't survive JSON serialization through the
+// workflow history, so callers re-read it from getVideoJobStatus when needed.
+async function startVideoUpload(
agent: AtpAgent,
videoPath: string
-): Promise<AppBskyEmbedVideo.Main> {
+): Promise<AppBskyVideoDefs.JobStatus> {
const { data: serviceAuth } = await agent.com.atproto.server.getServiceAuth({
aud: `did:web:${agent.dispatchUrl.host}`,
lxm: 'com.atproto.repo.uploadBlob',
@@ -125,8 +130,39 @@ async function uploadVideo(
const jobStatus = (await uploadResponse.json()) as AppBskyVideoDefs.JobStatus;
console.log('JobId:', jobStatus.jobId);
- let blob: BlobRef | undefined = jobStatus.blob;
+
+ if (jobStatus.state === 'JOB_STATE_FAILED') {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify({}),
+ {} as any,
+ 'Could not upload video, job failed'
+ );
+ }
+
+ return jobStatus;
+}
+
+// Single read of the processing job - the endpoint is public (no login), so
+// status checks never spend Bluesky's strict createSession rate limits.
+async function getVideoJobStatus(
+ jobId: string
+): Promise<AppBskyVideoDefs.JobStatus> {
const videoAgent = new AtpAgent({ service: 'https://video.bsky.app' });
+ const { data: status } = await videoAgent.app.bsky.video.getJobStatus({
+ jobId,
+ });
+ return status.jobStatus;
+}
+
+// Blocking upload+wait, used by the paths that still resolve everything inside
+// one activity (comments, and post() for pre-v1.0.6 workflows).
+async function uploadVideo(
+ agent: AtpAgent,
+ videoPath: string
+): Promise<AppBskyEmbedVideo.Main> {
+ const jobStatus = await startVideoUpload(agent, videoPath);
+ let blob: BlobRef | undefined = jobStatus.blob;
let attempts = 0;
const maxAttempts = 18; // ~9 minutes at 30s interval
@@ -140,19 +176,13 @@ async function uploadVideo(
);
}
- const { data: status } = await videoAgent.app.bsky.video.getJobStatus({
- jobId: jobStatus.jobId,
- });
- console.log(
- 'Status:',
- status.jobStatus.state,
- status.jobStatus.progress || ''
- );
- if (status.jobStatus.blob) {
- blob = status.jobStatus.blob;
+ const status = await getVideoJobStatus(jobStatus.jobId);
+ console.log('Status:', status.state, status.progress || '');
+ if (status.blob) {
+ blob = status.blob;
}
- if (status.jobStatus.state === 'JOB_STATE_FAILED') {
+ if (status.state === 'JOB_STATE_FAILED') {
throw new BadBody(
'bluesky',
JSON.stringify({}),
@@ -172,6 +202,25 @@ async function uploadVideo(
} satisfies AppBskyEmbedVideo.Main;
}
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost - keep it small JSON: the video job id and the post
+// content, never a BlobRef (it does not survive JSON serialization).
+type BlueskyPendingData = {
+ jobId: string; // empty for image-only posts (no asynchronous processing)
+ message: string;
+ media: { path: string; alt?: string }[];
+ // Arm -> confirm -> publish handshake (same as the Facebook story flow):
+ // finalizePost arms without mutating, checkPostStatus witnesses, and only a
+ // witnessed attempt runs the create - so a create that dies with an unknown
+ // outcome is detected instead of run again.
+ attempting?: boolean;
+ confirmed?: boolean;
+ // Consecutive finalize preparation failures (login, image upload): bounded
+ // so a dead integration fails with an accurate error instead of burning the
+ // whole check budget and ending in a misleading "couldn't confirm" warning.
+ prepFailures?: number;
+};
+
@Rules(
'Bluesky can have maximum 1 video or 4 pictures in one post, it can also be without attachments'
)
@@ -369,41 +418,322 @@ export class BlueskyProvider extends SocialAbstract implements SocialProvider {
return { embed, images };
}
- async post(
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails[],
integration: Integration
): Promise<PostResponse[]> {
- const agent = await this.getAgent(integration);
const [firstPost] = postDetails;
-
- const { embed } = await this.uploadMediaForPost(agent, firstPost);
-
- const rt = new RichText({
- text: firstPost.message,
- });
-
- await rt.detectFacets(agent);
-
- // @ts-ignore
- const { cid, uri, commit } = await agent.post({
- text: rt.text,
- facets: rt.facets,
- createdAt: new Date().toISOString(),
- ...(Object.keys(embed).length > 0 ? { embed } : {}),
- });
+ const videoMedia =
+ firstPost.media?.filter((p) => hasExtension(p.path, 'mp4')) || [];
+
+ // Only the video has an asynchronous processing step: start its upload job
+ // now, the wait moves to checkPostStatus. Images are uploaded by
+ // finalizePost (their BlobRefs can't survive JSON serialization through
+ // the workflow history). Nothing here is irreversible - an unused video
+ // job leaves no post behind.
+ let jobId = '';
+ if (videoMedia.length > 0) {
+ const agent = await this.getAgent(integration);
+ const jobStatus = await startVideoUpload(agent, videoMedia[0].path);
+ jobId = jobStatus.jobId;
+ }
return [
{
id: firstPost.id,
- postId: uri,
- status: 'completed',
- releaseURL: `https://bsky.app/profile/${id}/post/${uri.split('/').pop()}`,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ jobId,
+ message: firstPost.message,
+ media: (firstPost.media || []).map((m) => ({
+ path: m.path,
+ alt: m.alt || '',
+ })),
+ } as BlueskyPendingData,
},
];
}
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: BlueskyPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // A confirmed create attempt died without reporting its result: never run
+ // the create again - stop with an explicit warning instead.
+ if (pendingData.attempting && pendingData.confirmed) {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify({}),
+ {} as any,
+ 'Bluesky may have already published this post, please check your account before posting again to avoid duplicates'
+ );
+ }
+
+ // witness the armed create so finalizePost knows the attempt is uniquely
+ // accounted for before it mutates anything
+ const witness = (): PendingCheckResponse =>
+ pendingData.attempting && !pendingData.confirmed
+ ? {
+ status: 'ready',
+ pendingData: { ...pendingData, confirmed: true },
+ }
+ : { status: 'ready', pendingData };
+
+ // Image-only posts have no asynchronous processing step.
+ if (!pendingData.jobId) {
+ return witness();
+ }
+
+ let status: AppBskyVideoDefs.JobStatus;
+ try {
+ status = await getVideoJobStatus(pendingData.jobId);
+ } catch (err) {
+ if (err instanceof RefreshToken) {
+ throw err;
+ }
+
+ // Transient status-check error: the job may finish processing just
+ // fine, keep polling - if the video service stays broken the workflow
+ // exhausts its checks and warns the user properly.
+ return { status: 'pending', pendingData };
+ }
+
+ if (status.state === 'JOB_STATE_FAILED') {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify(status),
+ {} as any,
+ 'Could not upload video, job failed'
+ );
+ }
+
+ if (status.blob) {
+ return witness();
+ }
+
+ return { status: 'pending', pendingData };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: BlueskyPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Create with an arm -> confirm -> publish handshake: the create only runs
+ // after checkPostStatus witnessed the intent, so a run that dies
+ // mid-create is detectable and the post is never published twice.
+ if (!pendingData.attempting || !pendingData.confirmed) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, attempting: true, confirmed: false },
+ };
+ }
+
+ // Everything up to the create itself is preparation: none of it mutates
+ // the account, so a failure here must NOT leave the confirmed handshake
+ // armed - the next check would report a false "may already be published"
+ // warning. Disarm and hand back to the workflow to try again instead; the
+ // only unknown-outcome window left is agent.post below.
+ let agent: BskyAgent;
+ let embed: any = {};
+ let rt: RichText;
+ try {
+ agent = await this.getAgent(integration);
+
+ if (pendingData.jobId) {
+ // Re-read the blob from the job (a BlobRef never travels through the
+ // workflow history).
+ const status = await getVideoJobStatus(pendingData.jobId);
+
+ if (status.state === 'JOB_STATE_FAILED') {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify(status),
+ {} as any,
+ 'Could not upload video, job failed'
+ );
+ }
+
+ if (!status.blob) {
+ // The blob disappeared between the check and now (or the check was
+ // skipped): hand back to the workflow to keep polling.
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, attempting: false, confirmed: false },
+ };
+ }
+
+ embed = {
+ $type: 'app.bsky.embed.video',
+ video: status.blob,
+ } satisfies AppBskyEmbedVideo.Main;
+ } else {
+ const built = await this.uploadMediaForPost(agent, {
+ media: pendingData.media,
+ } as PostDetails);
+ embed = built.embed;
+ }
+
+ rt = new RichText({
+ text: pendingData.message,
+ });
+
+ await rt.detectFacets(agent);
+ } catch (err) {
+ // Explicit verdicts keep their meaning: a failed video job fails the
+ // post, an invalid login goes through the refresh-token flow.
+ if (err instanceof BadBody || err instanceof RefreshToken) {
+ throw err;
+ }
+
+ // Repeated preparation failures mean the integration is broken, not
+ // flaky: fail with an accurate message - nothing was ever created, so
+ // this is safe and beats exhausting the check budget into a misleading
+ // "check your account" warning.
+ if ((pendingData.prepFailures || 0) >= 4) {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify({}),
+ {} as any,
+ 'Could not prepare the post for Bluesky, nothing was published, please try again'
+ );
+ }
+
+ // Transient preparation error, nothing was created: disarm so the
+ // workflow keeps polling and a later cycle re-arms.
+ return {
+ status: 'pending',
+ pendingData: {
+ ...pendingData,
+ attempting: false,
+ confirmed: false,
+ prepFailures: (pendingData.prepFailures || 0) + 1,
+ },
+ };
+ }
+
+ let uri: string;
+ try {
+ // @ts-ignore
+ const created = await agent.post({
+ text: rt.text,
+ facets: rt.facets,
+ createdAt: new Date().toISOString(),
+ ...(Object.keys(embed).length > 0 ? { embed } : {}),
+ });
+ uri = created.uri;
+ } catch (err: any) {
+ // A definite 4xx rejection (invalid record, payload too large) means
+ // the post was NOT created: fail it properly instead of leaving the
+ // armed handshake to report a false "may already be published" warning.
+ // Ambiguous answers (network errors, 5xx, 429) stay on the conservative
+ // unknown-outcome path. No 408 exclusion: the xrpc client maps every
+ // status it does not know - including 408 - to 400 before we see it.
+ const status = err?.status;
+ if (
+ typeof status === 'number' &&
+ status >= 400 &&
+ status < 500 &&
+ status !== 429
+ ) {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify(err?.error || err?.message || {}),
+ {} as any,
+ 'Bluesky rejected the post, nothing was published, please check the content and try again'
+ );
+ }
+
+ throw err;
+ }
+
+ return {
+ status: 'completed',
+ postId: uri,
+ releaseURL: `https://bsky.app/profile/${
+ integration.internalId
+ }/post/${uri.split('/').pop()}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they wait for the
+ // processing and create the post inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (the post is only created once the
+ // video is processed), timing the activity out is not - a retried
+ // activity would upload and publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ 'bluesky',
+ JSON.stringify({}),
+ {} as any,
+ 'Video upload timed out, job did not complete'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(20000);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: result.postId,
+ status: 'completed',
+ releaseURL: result.releaseURL,
+ },
+ ];
+ }
+
+ pendingData = result.pendingData;
+
+ // An armed pending just loops into the witnessing check; a de-armed one
+ // is a transient preparation failure (login, image upload) - back off
+ // before retrying instead of hammering the login endpoint.
+ if (!(result.pendingData as BlueskyPendingData)?.attempting) {
+ await timer(20000);
+ }
+ }
+ }
+
async comment(
id: string,
postId: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/linkedin.page.provider.ts b/libraries/nestjs-libraries/src/integrations/social/linkedin.page.provider.ts
index 3c37b67a3..288e36daa 100644
--- a/libraries/nestjs-libraries/src/integrations/social/linkedin.page.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/linkedin.page.provider.ts
@@ -273,6 +273,23 @@ export class LinkedinPageProvider
return super.post(id, accessToken, postDetails, integration, 'company');
}
+ // checkPostStatus / finalizePost are inherited as-is: the company context
+ // travels inside pendingData (postType), set here once.
+ override async postPending(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ return super.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration,
+ 'company'
+ );
+ }
+
override async comment(
id: string,
postId: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/linkedin.provider.ts b/libraries/nestjs-libraries/src/integrations/social/linkedin.provider.ts
index 7816f70c6..958605a50 100644
--- a/libraries/nestjs-libraries/src/integrations/social/linkedin.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/linkedin.provider.ts
@@ -1,5 +1,6 @@
import {
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -12,6 +13,7 @@ import { hasExtension } from '@gitroom/helpers/utils/has.extension';
import { timer } from '@gitroom/helpers/utils/timer';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
@@ -22,6 +24,34 @@ import imageToPDF from 'image-to-pdf';
import { Readable } from 'stream';
import { Rules } from '@gitroom/nestjs-libraries/chat/rules.description.decorator';
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost - keep it small JSON (media urns and the post content, never
+// buffers).
+type LinkedinPendingData = {
+ authorId: string;
+ postType: 'company' | 'personal';
+ message: string;
+ isPdf: boolean;
+ pdfTitle?: string;
+ mediaIds: string[];
+ // Media whose processing status can be read via GET: videos for every token,
+ // images/documents only for organization tokens.
+ poll: { urn: string; endpoint: 'videos' | 'images' | 'documents' }[];
+ // Check cycles to wait for unpollable media (personal images/documents).
+ graceChecks: number;
+ // Consecutive status reads that failed or answered without a status:
+ // bounded so a broken media endpoint fails with an accurate error instead
+ // of burning the whole check budget into a misleading "couldn't confirm"
+ // warning (nothing is ever published before the media is ready).
+ statusStalls?: number;
+ // Arm -> confirm -> publish handshake (same as the Facebook story flow):
+ // finalizePost arms without mutating, checkPostStatus witnesses, and only a
+ // witnessed attempt runs the create - so a create that dies with an unknown
+ // outcome is detected instead of run again (LinkedIn has no idempotency key).
+ attempting?: boolean;
+ confirmed?: boolean;
+};
+
@Rules(
'LinkedIn can have maximum one attachment when selecting video, when choosing a carousel on LinkedIn minimum amount of attachment must be two, and only pictures, if uploading a video, LinkedIn can have only one attachment'
)
@@ -267,7 +297,11 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
// instead of being held in memory.
picture: Buffer | { path: string },
type = 'personal' as 'company' | 'personal'
- ) {
+ ): Promise<{
+ id: string;
+ poll?: { urn: string; endpoint: 'videos' | 'images' | 'documents' };
+ grace?: boolean;
+ }> {
// Determine the appropriate endpoint based on file type
const isVideo = hasExtension(fileName, 'mp4');
const isPdf = hasExtension(fileName, 'pdf');
@@ -401,99 +435,60 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
},
}
);
+ }
- // After finalizing, the video is processed asynchronously. We have to
- // wait until it reaches the AVAILABLE status before attaching it to a
- // post, otherwise LinkedIn rejects the post with a processing error.
- await this.waitForMediaToBeReady(video, accessToken, 'videos');
- } else if (isPdf) {
- // Documents (PDF carousels) are processed asynchronously too and must be
- // AVAILABLE before being attached to a post, otherwise LinkedIn rejects
- // the post or publishes a broken carousel.
- if (type === 'company') {
- // Organization-owned documents can be polled for their status.
- await this.waitForMediaToBeReady(document, accessToken, 'documents');
- } else {
- // A w_member_social (personal) token is write-only for rest/documents
- // and can't perform the GET, so polling would always be forbidden.
- // Give LinkedIn a moment to finish processing before attaching it.
- await timer(10000);
- }
- } else {
- // Images are also processed asynchronously and must be AVAILABLE before
- // being attached to a post.
- if (type === 'company') {
- // Organization-owned images can be polled for their processing status.
- await this.waitForMediaToBeReady(image, accessToken, 'images');
- } else {
- // A w_member_social (personal) token is write-only for rest/images and
- // can't perform the GET, so polling would always be forbidden. Images
- // are quick to process, so we just give LinkedIn a moment before
- // attaching it to the post.
- await timer(10000);
- }
+ // Every media type is processed asynchronously after the upload and must
+ // reach AVAILABLE before it is attached to a post, otherwise LinkedIn
+ // rejects the post with a processing error. The waiting itself happens in
+ // checkPostStatus, this method only describes what to wait for: videos can
+ // be polled by any token, images and documents only by organization
+ // tokens - a w_member_social (personal) token is write-only for those
+ // endpoints, so the caller waits a grace period instead of polling.
+ if (isVideo) {
+ return {
+ id: finalOutput,
+ poll: { urn: finalOutput, endpoint: 'videos' as const },
+ };
}
- return finalOutput;
+ if (type === 'company') {
+ return {
+ id: finalOutput,
+ poll: {
+ urn: finalOutput,
+ endpoint: (isPdf ? 'documents' : 'images') as 'documents' | 'images',
+ },
+ };
+ }
+
+ return { id: finalOutput, grace: true };
}
- // Polls the "Get a Video"/"Get an Image" API until the media finishes
- // processing (status === AVAILABLE).
+ // Single read of the media processing status, no loops and no timers - the
+ // polling loop lives in the post workflow (checkPostStatus).
// videos: https://learn.microsoft.com/en-us/linkedin/marketing/community-management/shares/videos-api#get-a-video
// images: https://learn.microsoft.com/en-us/linkedin/marketing/community-management/shares/images-api#get-a-single-image
- private async waitForMediaToBeReady(
- urn: string,
+ private async mediaProcessingStatus(
accessToken: string,
- type: 'videos' | 'images' | 'documents',
- maxAttempts = 20,
- intervalMs = 30000
- ): Promise<void> {
- const label =
- type === 'videos' ? 'video' : type === 'documents' ? 'document' : 'image';
-
- for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
- const json = await (
- await this.fetch(
- `https://api.linkedin.com/rest/${type}/${encodeURIComponent(urn)}`,
- {
- method: 'GET',
- headers: {
- 'X-Restli-Protocol-Version': '2.0.0',
- 'LinkedIn-Version': '202601',
- 'Content-Type': 'application/json',
- Authorization: `Bearer ${accessToken}`,
- },
- }
- )
- ).json();
-
- const { status, processingFailureReason } = json;
-
- if (status === 'AVAILABLE') {
- return;
- }
-
- if (status === 'PROCESSING_FAILED') {
- throw new BadBody(
- this.identifier,
- JSON.stringify(json),
- '{}',
- `LinkedIn ${label} processing failed${
- processingFailureReason ? `: ${processingFailureReason}` : ''
- }`
- );
- }
-
- // status is PROCESSING or WAITING_UPLOAD, keep polling.
- await timer(intervalMs);
- }
-
- throw new BadBody(
- this.identifier,
- '{}',
- '{}',
- `Timed out waiting for LinkedIn ${label} to be ready`
- );
+ urn: string,
+ endpoint: 'videos' | 'images' | 'documents'
+ ): Promise<{ status?: string; processingFailureReason?: string }> {
+ // The full response body is returned (not just the two fields) so a
+ // processing failure can attach LinkedIn's complete answer to the error.
+ return await (
+ await this.fetch(
+ `https://api.linkedin.com/rest/${endpoint}/${encodeURIComponent(urn)}`,
+ {
+ method: 'GET',
+ headers: {
+ 'X-Restli-Protocol-Version': '2.0.0',
+ 'LinkedIn-Version': '202601',
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${accessToken}`,
+ },
+ }
+ )
+ ).json();
}
protected fixText(text: string) {
@@ -596,10 +591,17 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
accessToken: string,
personId: string,
type: 'company' | 'personal'
- ): Promise<Record<string, string[]>> {
+ ): Promise<{
+ mediaUploads: Record<string, string[]>;
+ poll: { urn: string; endpoint: 'videos' | 'images' | 'documents' }[];
+ graceChecks: number;
+ }> {
// Media is uploaded sequentially on purpose: uploading everything with
// Promise.all holds every file in memory at the same time.
const mediaUploads: Record<string, string[]> = {};
+ const poll: { urn: string; endpoint: 'videos' | 'images' | 'documents' }[] =
+ [];
+ let graceChecks = 0;
for (const post of postDetails) {
for (const media of post.media || []) {
let mediaBuffer: Buffer | { path: string };
@@ -620,7 +622,7 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
mediaBuffer = await this.prepareMediaBuffer(media.path);
}
- const uploadedMediaId = await this.uploadPicture(
+ const uploaded = await this.uploadPicture(
media.path,
accessToken,
personId,
@@ -628,16 +630,26 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
type
);
- if (!uploadedMediaId) {
+ if (!uploaded?.id) {
continue;
}
mediaUploads[post.id] = mediaUploads[post.id] || [];
- mediaUploads[post.id].push(uploadedMediaId);
+ mediaUploads[post.id].push(uploaded.id);
+
+ if (uploaded.poll) {
+ poll.push(uploaded.poll);
+ }
+
+ // Unpollable media (personal images/documents): wait one durable check
+ // cycle before attaching it, replacing the old inline timer.
+ if (uploaded.grace) {
+ graceChecks = 1;
+ }
}
}
- return mediaUploads;
+ return { mediaUploads, poll, graceChecks };
}
private async prepareMediaBuffer(mediaUrl: string): Promise<Buffer> {
@@ -725,21 +737,18 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
}
private async createMainPost(
- id: string,
+ authorId: string,
accessToken: string,
- firstPost: PostDetails<LinkedinDto>,
+ message: string,
mediaIds: string[],
type: 'company' | 'personal',
- isPdf: boolean
+ isPdf: boolean,
+ pdfTitle?: string
): Promise<string> {
- const pdfTitle = isPdf
- ? firstPost.settings?.carousel_name || 'slides'
- : undefined;
-
const postPayload = this.createLinkedInPostPayload(
- id,
+ authorId,
type,
- firstPost.message,
+ message,
mediaIds,
isPdf,
pdfTitle
@@ -821,7 +830,7 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
};
}
- async post(
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails<LinkedinDto>[],
@@ -830,9 +839,10 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
): Promise<PostResponse[]> {
let processedPostDetails = postDetails;
const [firstPost] = postDetails;
+ const isPdf = this.assetBoolean(firstPost.settings?.post_as_images_carousel);
// Check if we should convert images to PDF carousel
- if (this.assetBoolean(firstPost.settings?.post_as_images_carousel)) {
+ if (isPdf) {
processedPostDetails = await this.convertImagesToPdfCarousel(
postDetails,
firstPost
@@ -841,31 +851,251 @@ export class LinkedinProvider extends SocialAbstract implements SocialProvider {
const [processedFirstPost] = processedPostDetails;
- // Process and upload media for the first post only
- const uploadedMedia = await this.processMediaForPosts(
+ // Upload the media now; the asynchronous processing wait moves to
+ // checkPostStatus and the post itself is only created by finalizePost, so
+ // nothing here is irreversible - a failure leaves only orphaned media.
+ const { mediaUploads, poll, graceChecks } = await this.processMediaForPosts(
[processedFirstPost],
accessToken,
id,
type
);
- // Get media IDs for the main post
- const mainPostMediaIds = (
- uploadedMedia[processedFirstPost.id] || []
- ).filter(Boolean);
+ return [
+ {
+ id: processedFirstPost.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ authorId: id,
+ postType: type,
+ message: processedFirstPost.message,
+ isPdf,
+ ...(isPdf
+ ? { pdfTitle: firstPost.settings?.carousel_name || 'slides' }
+ : {}),
+ mediaIds: (mediaUploads[processedFirstPost.id] || []).filter(Boolean),
+ poll,
+ graceChecks,
+ } as LinkedinPendingData,
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: LinkedinPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // A confirmed create attempt died without reporting its result: LinkedIn
+ // gives no way to ask whether that post was created, so never run the
+ // create again - stop with an explicit warning instead.
+ if (pendingData.attempting && pendingData.confirmed) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'LinkedIn may have already published this post, please check your account before posting again to avoid duplicates'
+ );
+ }
+
+ // Check every media that is still processing; keep the ones that have not
+ // reached AVAILABLE yet for the next cycle.
+ const stillProcessing: LinkedinPendingData['poll'] = [];
+ for (const media of pendingData.poll || []) {
+ let status: { status?: string; processingFailureReason?: string };
+ try {
+ status = await this.mediaProcessingStatus(
+ accessToken,
+ media.urn,
+ media.endpoint
+ );
+
+ // A 200 without a status field can never reach AVAILABLE: count it
+ // like a failed read instead of polling it forever.
+ if (!status.status) {
+ throw new Error('LinkedIn answered without a media status');
+ }
+ } catch (err) {
+ if (err instanceof RefreshToken) {
+ throw err;
+ }
+
+ // Persistent failures mean the media will never be confirmed ready:
+ // fail with an accurate message - the post is only created after the
+ // media is ready, so nothing was published.
+ if ((pendingData.statusStalls || 0) >= 10) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'LinkedIn kept failing the media status check, nothing was published, please try again'
+ );
+ }
+
+ // Transient status-check error: the media may finish processing just
+ // fine, keep polling.
+ return {
+ status: 'pending',
+ pendingData: {
+ ...pendingData,
+ statusStalls: (pendingData.statusStalls || 0) + 1,
+ },
+ };
+ }
+
+ if (status.status === 'PROCESSING_FAILED') {
+ const label =
+ media.endpoint === 'videos'
+ ? 'video'
+ : media.endpoint === 'documents'
+ ? 'document'
+ : 'image';
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify(status),
+ '{}',
+ `LinkedIn ${label} processing failed${
+ status.processingFailureReason
+ ? `: ${status.processingFailureReason}`
+ : ''
+ }`
+ );
+ }
+
+ if (status.status !== 'AVAILABLE') {
+ stillProcessing.push(media);
+ }
+ }
+
+ if (stillProcessing.length) {
+ // every media answered with a real status: the endpoint works, reset
+ // the stall counter
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, poll: stillProcessing, statusStalls: 0 },
+ };
+ }
+
+ // Unpollable media (personal images/documents are write-only endpoints):
+ // wait the remaining grace cycles before attaching them to the post.
+ if (pendingData.graceChecks > 0) {
+ return {
+ status: 'pending',
+ pendingData: {
+ ...pendingData,
+ poll: [],
+ graceChecks: pendingData.graceChecks - 1,
+ },
+ };
+ }
+
+ // witness the armed create so finalizePost knows the attempt is uniquely
+ // accounted for before it mutates anything
+ if (pendingData.attempting && !pendingData.confirmed) {
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, poll: [], confirmed: true },
+ };
+ }
+
+ return { status: 'ready', pendingData: { ...pendingData, poll: [] } };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: LinkedinPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Create with an arm -> confirm -> publish handshake: the create only runs
+ // after checkPostStatus witnessed the intent, so a run that dies
+ // mid-create is detectable and the post is never published twice.
+ if (!pendingData.attempting || !pendingData.confirmed) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, attempting: true, confirmed: false },
+ };
+ }
- // Create the main LinkedIn post
const mainPostId = await this.createMainPost(
+ pendingData.authorId,
+ accessToken,
+ pendingData.message,
+ (pendingData.mediaIds || []).filter(Boolean),
+ pendingData.postType,
+ pendingData.isPdf,
+ pendingData.pdfTitle
+ );
+
+ return {
+ status: 'completed',
+ postId: mainPostId,
+ releaseURL: `https://www.linkedin.com/feed/update/${mainPostId}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they wait for the
+ // media processing and create the post inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<LinkedinDto>[],
+ integration: Integration,
+ type = 'personal' as 'company' | 'personal'
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(
id,
accessToken,
- processedFirstPost,
- mainPostMediaIds,
- type,
- this.assetBoolean(firstPost.settings?.post_as_images_carousel)
+ postDetails,
+ integration,
+ type
);
- // Return response for main post only
- return [this.createPostResponse(mainPostId, processedFirstPost.id, true)];
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (the post is only created once all
+ // the media is ready), timing the activity out is not - a retried
+ // activity would upload and publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'LinkedIn took too long to process the media, please try again'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(20000);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [this.createPostResponse(result.postId, response.id, true)];
+ }
+
+ // finalize only armed the handshake (nothing to wait for), loop straight
+ // into the witnessing check
+ pendingData = result.pendingData;
+ }
}
async comment(
diff --git a/libraries/nestjs-libraries/src/integrations/social/mastodon.custom.provider.ts b/libraries/nestjs-libraries/src/integrations/social/mastodon.custom.provider.ts
index 968008840..40b1c281c 100644
--- a/libraries/nestjs-libraries/src/integrations/social/mastodon.custom.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/mastodon.custom.provider.ts
@@ -5,6 +5,7 @@ import {
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
import { MastodonProvider } from '@gitroom/nestjs-libraries/integrations/social/mastodon.provider';
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
+import { AuthService } from '@gitroom/helpers/auth/auth.service';
import { getSsrfSafeDispatcher } from '@gitroom/nestjs-libraries/dtos/webhooks/ssrf.safe.dispatcher';
import { Integration } from '@prisma/client';
@@ -73,15 +74,45 @@ export class MastodonCustomProvider extends MastodonProvider {
);
}
+ // The user's instance URL, saved encrypted at connection time. Falls back to
+ // the default instance for legacy integrations that predate storing it.
+ private instanceUrl(integration?: Integration) {
+ try {
+ const { instanceUrl } = JSON.parse(
+ AuthService.fixedDecryption(integration?.customInstanceDetails || '')
+ );
+ return instanceUrl || process.env.MASTODON_URL || 'https://mastodon.social';
+ } catch (err) {
+ return process.env.MASTODON_URL || 'https://mastodon.social';
+ }
+ }
+
override async post(
id: string,
accessToken: string,
- postDetails: PostDetails[]
+ postDetails: PostDetails[],
+ integration: Integration
): Promise<PostResponse[]> {
return this.dynamicPost(
id,
accessToken,
- process.env.MASTODON_URL || 'https://mastodon.social',
+ this.instanceUrl(integration),
+ postDetails
+ );
+ }
+
+ // checkPostStatus / finalizePost are inherited as-is: the instance url
+ // travels inside pendingData, set here once.
+ override async postPending(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ return this.dynamicPostPending(
+ id,
+ accessToken,
+ this.instanceUrl(integration),
postDetails
);
}
@@ -99,7 +130,7 @@ export class MastodonCustomProvider extends MastodonProvider {
postId,
lastCommentId,
accessToken,
- process.env.MASTODON_URL || 'https://mastodon.social',
+ this.instanceUrl(integration),
postDetails
);
}
diff --git a/libraries/nestjs-libraries/src/integrations/social/mastodon.provider.ts b/libraries/nestjs-libraries/src/integrations/social/mastodon.provider.ts
index e1f0c108e..ee97692c9 100644
--- a/libraries/nestjs-libraries/src/integrations/social/mastodon.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/mastodon.provider.ts
@@ -1,18 +1,35 @@
import {
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
-import { SocialAbstract } from '@gitroom/nestjs-libraries/integrations/social.abstract';
+import {
+ BadBody,
+ RefreshToken,
+ SocialAbstract,
+} from '@gitroom/nestjs-libraries/integrations/social.abstract';
import { getSsrfSafeDispatcher } from '@gitroom/nestjs-libraries/dtos/webhooks/ssrf.safe.dispatcher';
+import { timer } from '@gitroom/helpers/utils/timer';
import dayjs from 'dayjs';
import { Integration } from '@prisma/client';
import { number, string } from 'yup';
import FormDataUpload from 'form-data';
import { PassThrough, Readable } from 'stream';
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost - keep it small JSON (the instance url, media ids and the
+// status content). The url makes checkPostStatus / finalizePost work the same
+// for the default instance and custom instances.
+type MastodonPendingData = {
+ url: string;
+ message: string;
+ mediaIds: string[];
+ idempotencyKey: string;
+};
+
export class MastodonProvider extends SocialAbstract implements SocialProvider {
override maxConcurrentJob = 5; // Mastodon instances typically have generous limits
identifier = 'mastodon';
@@ -37,6 +54,22 @@ export class MastodonProvider extends SocialAbstract implements SocialProvider {
};
}
+ if (body.includes('not finished processing')) {
+ return {
+ type: 'bad-body',
+ value:
+ 'The media was still processing when the post was published, please try again',
+ };
+ }
+
+ if (body.includes('could not be found')) {
+ return {
+ type: 'bad-body',
+ value:
+ 'The uploaded media expired before the post was published, please post again',
+ };
+ }
+
return undefined;
}
@@ -206,7 +239,7 @@ export class MastodonProvider extends SocialAbstract implements SocialProvider {
return media.id;
}
- async dynamicPost(
+ protected async dynamicPostPending(
id: string,
accessToken: string,
url: string,
@@ -214,39 +247,227 @@ export class MastodonProvider extends SocialAbstract implements SocialProvider {
): Promise<PostResponse[]> {
const [firstPost] = postDetails;
+ // Upload the media now; a 202 means the instance is still processing it
+ // (videos, large GIFs) and attaching it to a status too early fails with
+ // 422 - the wait moves to checkPostStatus. The status itself is only
+ // created by finalizePost, so nothing here is irreversible.
const uploadFiles = await Promise.all(
firstPost?.media?.map((media) =>
this.uploadFile(url, media.path, accessToken, media.alt)
) || []
);
+ return [
+ {
+ id: firstPost.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ url,
+ message: firstPost.message,
+ mediaIds: uploadFiles.filter(Boolean),
+ // Statuses POSTs are deduplicated by Mastodon via Idempotency-Key.
+ // Generated once per publish run: it rides pendingData through the
+ // workflow history, so finalize retries within this run dedupe -
+ // but an edited post re-published in a fresh run gets a new key
+ // (the db post id alone would silently return the old status).
+ idempotencyKey: `${firstPost.id}-${makeId(5)}`,
+ } as MastodonPendingData,
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: MastodonPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Text-only statuses have no asynchronous processing step.
+ if (!pendingData.mediaIds?.length) {
+ return { status: 'ready', pendingData };
+ }
+
+ // GET /api/v1/media/:id answers 206 while the media is still processing
+ // and 200 once it is ready - exactly the pending/ready split. this.fetch
+ // treats 206 as an error, so use the raw fetch with the SSRF-safe
+ // dispatcher (the instance URL is user-influenced for custom instances).
+ for (const mediaId of pendingData.mediaIds) {
+ let response: Response;
+ try {
+ response = await fetch(`${pendingData.url}/api/v1/media/${mediaId}`, {
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ },
+ // @ts-ignore - undici-only option; blocks SSRF to internal IPs
+ dispatcher: getSsrfSafeDispatcher(),
+ } as any);
+ } catch (err) {
+ // Transient status-check error: the media may finish processing just
+ // fine, keep polling - if the instance stays broken the workflow
+ // exhausts its checks and warns the user properly.
+ return { status: 'pending', pendingData };
+ }
+
+ if (response.status === 206) {
+ // consume the body so the undici socket is released between polls
+ await response.text().catch(() => '');
+ return { status: 'pending', pendingData };
+ }
+
+ if (response.status === 401) {
+ throw new RefreshToken(
+ this.identifier,
+ await response.text().catch(() => '{}'),
+ '{}'
+ );
+ }
+
+ if (response.status === 404 || response.status === 410) {
+ // The media GET only sees UNATTACHED media: after a finalize whose
+ // outcome was lost, an already-published status makes its media 404
+ // here. Never conclude "expired, post again" (that instruction is the
+ // duplicate path) - keep checking the remaining media (a later 206
+ // still wins) and let the idempotent finalize either dedupe into the
+ // existing status or get Mastodon's verdict.
+ await response.text().catch(() => '');
+ continue;
+ }
+
+ if (response.status === 422) {
+ throw new BadBody(
+ this.identifier,
+ await response.text().catch(() => '{}'),
+ '{}',
+ 'The uploaded media expired before the post was published, please post again'
+ );
+ }
+
+ await response.text().catch(() => '');
+
+ if (response.status !== 200) {
+ // Unknown answer on a read-only check: treat as transient.
+ return { status: 'pending', pendingData };
+ }
+ }
+
+ return { status: 'ready', pendingData };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: MastodonPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
const form = new FormData();
- form.append('status', firstPost.message);
+ form.append('status', pendingData.message);
form.append('visibility', 'public');
- if (uploadFiles.length) {
- for (const file of uploadFiles) {
- form.append('media_ids[]', file);
- }
+ for (const file of pendingData.mediaIds || []) {
+ form.append('media_ids[]', file);
}
const post = await (
- await this.fetch(`${url}/api/v1/statuses`, {
+ await this.fetch(`${pendingData.url}/api/v1/statuses`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
+ // Mastodon deduplicates statuses POSTs on this key (kept for about
+ // an hour): a finalize retry after an unknown outcome returns the
+ // already-created status instead of publishing twice. Guarded so a
+ // missing key never serializes as the literal "undefined" header,
+ // which would dedupe against an unrelated post.
+ ...(pendingData.idempotencyKey
+ ? { 'Idempotency-Key': pendingData.idempotencyKey }
+ : {}),
},
body: form,
})
).json();
- return [
- {
- id: firstPost.id,
- postId: post.id,
- releaseURL: `${url}/statuses/${post.id}`,
- status: 'completed',
- },
- ];
+ return {
+ status: 'completed',
+ postId: post.id,
+ releaseURL: `${pendingData.url}/statuses/${post.id}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they wait for the
+ // media processing and create the status inside the activity like before.
+ async dynamicPost(
+ id: string,
+ accessToken: string,
+ url: string,
+ postDetails: PostDetails[]
+ ): Promise<PostResponse[]> {
+ const [response] = await this.dynamicPostPending(
+ id,
+ accessToken,
+ url,
+ postDetails
+ );
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // {} as Integration is safe here: checkPostStatus and finalizePost read
+ // everything from pendingData (including the instance url) - keep it
+ // that way if either method ever grows an integration dependency.
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ {} as Integration
+ );
+
+ if (check.status === 'ready') {
+ const finalize = await this.finalizePost(
+ accessToken,
+ check.pendingData,
+ {} as Integration
+ );
+
+ if (finalize.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: finalize.postId,
+ releaseURL: finalize.releaseURL,
+ status: 'completed',
+ },
+ ];
+ }
+
+ pendingData = finalize.pendingData;
+ } else if (check.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: check.postId,
+ releaseURL: check.releaseURL,
+ status: 'completed',
+ },
+ ];
+ } else {
+ pendingData = check.pendingData;
+ }
+
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (the status is only created once the
+ // media is ready), timing the activity out is not - a retried activity
+ // would upload and publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'The media took too long to process, please try again'
+ );
+ }
+
+ await timer(20000);
+ }
}
async dynamicComment(
@@ -299,7 +520,8 @@ export class MastodonProvider extends SocialAbstract implements SocialProvider {
async post(
id: string,
accessToken: string,
- postDetails: PostDetails[]
+ postDetails: PostDetails[],
+ integration: Integration
): Promise<PostResponse[]> {
return this.dynamicPost(
id,
@@ -309,6 +531,20 @@ export class MastodonProvider extends SocialAbstract implements SocialProvider {
);
}
+ async postPending(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ return this.dynamicPostPending(
+ id,
+ accessToken,
+ process.env.MASTODON_URL || 'https://mastodon.social',
+ postDetails
+ );
+ }
+
async comment(
id: string,
postId: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/pinterest.provider.ts b/libraries/nestjs-libraries/src/integrations/social/pinterest.provider.ts
index bc201cbbc..98100cbf2 100644
--- a/libraries/nestjs-libraries/src/integrations/social/pinterest.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/pinterest.provider.ts
@@ -1,10 +1,12 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
+import { Integration } from '@prisma/client';
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import { PinterestSettingsDto } from '@gitroom/nestjs-libraries/dtos/posts/providers-settings/pinterest.dto';
import axios from 'axios';
@@ -12,6 +14,7 @@ import FormData from 'form-data';
import { timer } from '@gitroom/helpers/utils/timer';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
@@ -20,6 +23,28 @@ import { Tool } from '@gitroom/nestjs-libraries/integrations/tool.decorator';
import { Rules } from '@gitroom/nestjs-libraries/chat/rules.description.decorator';
import { hasExtension } from '@gitroom/helpers/utils/has.extension';
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost - keep it small JSON (the media id and the pin content).
+type PinterestPendingData = {
+ mediaId: string; // empty for image-only pins (no asynchronous processing)
+ message: string;
+ settings: {
+ link?: string;
+ title?: string;
+ dominant_color?: string;
+ board: string;
+ };
+ imagePaths: string[];
+ coverPath?: string;
+ // Arm -> confirm -> publish handshake (same as the Facebook story flow):
+ // finalizePost arms without mutating, checkPostStatus witnesses, and only a
+ // witnessed attempt runs the create - so a create that dies with an unknown
+ // outcome is detected instead of run again (Pinterest has no idempotency
+ // key).
+ attempting?: boolean;
+ confirmed?: boolean;
+};
+
@Rules(
'Pinterest requires at least one media, if posting a video, you must have two attachment, one for video, one for the cover picture, When posting a video, there can be only one, if posting images, there can be maximum 5'
)
@@ -243,7 +268,7 @@ export class PinterestProvider
);
}
- async post(
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails<PinterestSettingsDto>[]
@@ -256,6 +281,9 @@ export class PinterestProvider
(p) => !hasExtension(p.path, 'mp4')
);
+ // Upload the video now; the processing wait moves to checkPostStatus and
+ // the pin itself is only created by finalizePost, so nothing here is
+ // irreversible - a failure leaves only an orphaned media upload.
if (findMp4) {
const { upload_url, media_id, upload_parameters } = await (
await this.fetch('https://api.pinterest.com/v5/media', {
@@ -284,53 +312,122 @@ export class PinterestProvider
formData.append('file', data);
await axios.post(upload_url, formData);
- let statusCode = '';
- let attempts = 0;
- const maxAttempts = 18; // ~9 minutes at 30s interval
- while (statusCode !== 'succeeded') {
- if (attempts++ >= maxAttempts) {
- throw new BadBody(
- 'pinterest',
- JSON.stringify({}),
- {} as any,
- 'The file took too long to process, please try again'
- );
- }
+ mediaId = media_id;
+ }
- const mediafile = await (
- await this.fetch(
- 'https://api.pinterest.com/v5/media/' + media_id,
- {
- method: 'GET',
- headers: {
- Authorization: `Bearer ${accessToken}`,
- },
- },
- '',
- 0,
- true
- )
- ).json();
-
- if (mediafile.status === 'failed') {
- throw new BadBody(
- 'pinterest',
- JSON.stringify({}),
- {} as any,
- 'The file is corrupted and cannot be uploaded'
- );
- }
+ return [
+ {
+ id: postDetails?.[0]?.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ mediaId,
+ message: postDetails?.[0]?.message,
+ settings: {
+ link: postDetails?.[0]?.settings.link,
+ title: postDetails?.[0]?.settings.title,
+ dominant_color: postDetails?.[0]?.settings.dominant_color,
+ board: postDetails?.[0]?.settings.board,
+ },
+ imagePaths: (postDetails?.[0]?.media || []).map((m) => m.path),
+ coverPath: picture?.path,
+ } as PinterestPendingData,
+ },
+ ];
+ }
- await timer(30000);
- statusCode = mediafile.status;
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: PinterestPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // A confirmed create attempt died without reporting its result: Pinterest
+ // gives no way to ask whether that pin was created, so never run the
+ // create again - stop with an explicit warning instead.
+ if (pendingData.attempting && pendingData.confirmed) {
+ throw new BadBody(
+ 'pinterest',
+ JSON.stringify({}),
+ {} as any,
+ 'Pinterest may have already published this pin, please check your account before posting again to avoid duplicates'
+ );
+ }
+
+ // witness the armed create so finalizePost knows the attempt is uniquely
+ // accounted for before it mutates anything
+ const witness = (): PendingCheckResponse =>
+ pendingData.attempting && !pendingData.confirmed
+ ? {
+ status: 'ready',
+ pendingData: { ...pendingData, confirmed: true },
+ }
+ : { status: 'ready', pendingData };
+
+ // Image-only pins have no asynchronous processing step.
+ if (!pendingData.mediaId) {
+ return witness();
+ }
+
+ let mediafile: { status?: string };
+ try {
+ mediafile = await (
+ await this.fetch(
+ 'https://api.pinterest.com/v5/media/' + pendingData.mediaId,
+ {
+ method: 'GET',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ },
+ },
+ '',
+ 0,
+ true
+ )
+ ).json();
+ } catch (err) {
+ if (err instanceof RefreshToken) {
+ throw err;
}
- mediaId = media_id;
+ // Transient status-check error: the media may finish processing just
+ // fine, keep polling - if Pinterest stays broken the workflow exhausts
+ // its checks and warns the user properly.
+ return { status: 'pending', pendingData };
+ }
+
+ if (mediafile.status === 'failed') {
+ throw new BadBody(
+ 'pinterest',
+ JSON.stringify({}),
+ {} as any,
+ 'The file is corrupted and cannot be uploaded'
+ );
+ }
+
+ if (mediafile.status !== 'succeeded') {
+ return { status: 'pending', pendingData };
+ }
+
+ return witness();
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: PinterestPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Create with an arm -> confirm -> publish handshake: the create only runs
+ // after checkPostStatus witnessed the intent, so a run that dies
+ // mid-create is detectable and the pin is never published twice.
+ if (!pendingData.attempting || !pendingData.confirmed) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, attempting: true, confirmed: false },
+ };
}
- const mapImages = postDetails?.[0]?.media?.map((m) => ({
- path: m.path,
- }));
+ const mapImages = (pendingData.imagePaths || []).map((path) => ({ path }));
const { id: pId } = await (
await this.fetch('https://api.pinterest.com/v5/pins', {
@@ -340,22 +437,22 @@ export class PinterestProvider
'Content-Type': 'application/json',
},
body: JSON.stringify({
- ...(postDetails?.[0]?.settings.link
- ? { link: postDetails?.[0]?.settings.link }
+ ...(pendingData.settings.link
+ ? { link: pendingData.settings.link }
: {}),
- ...(postDetails?.[0]?.settings.title
- ? { title: postDetails?.[0]?.settings.title }
+ ...(pendingData.settings.title
+ ? { title: pendingData.settings.title }
: {}),
- description: postDetails?.[0]?.message,
- ...(postDetails?.[0]?.settings.dominant_color
- ? { dominant_color: postDetails?.[0]?.settings.dominant_color }
+ description: pendingData.message,
+ ...(pendingData.settings.dominant_color
+ ? { dominant_color: pendingData.settings.dominant_color }
: {}),
- board_id: postDetails?.[0]?.settings.board,
- media_source: mediaId
+ board_id: pendingData.settings.board,
+ media_source: pendingData.mediaId
? {
source_type: 'video_id',
- media_id: mediaId,
- cover_image_url: picture?.path,
+ media_id: pendingData.mediaId,
+ cover_image_url: pendingData.coverPath,
}
: mapImages?.length === 1
? {
@@ -372,14 +469,74 @@ export class PinterestProvider
})
).json();
- return [
- {
- id: postDetails?.[0]?.id,
- postId: pId,
- releaseURL: `https://www.pinterest.com/pin/${pId}`,
- status: 'success',
- },
- ];
+ return {
+ status: 'completed',
+ postId: pId,
+ releaseURL: `https://www.pinterest.com/pin/${pId}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they wait for the
+ // processing and create the pin inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<PinterestSettingsDto>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(id, accessToken, postDetails);
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (the pin is only created once the
+ // media is ready), timing the activity out is not - a retried activity
+ // would upload and publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ 'pinterest',
+ JSON.stringify({}),
+ {} as any,
+ 'The file took too long to process, please try again'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(20000);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: result.postId,
+ releaseURL: result.releaseURL,
+ status: 'success',
+ },
+ ];
+ }
+
+ // finalize only armed the handshake (nothing to wait for), loop straight
+ // into the witnessing check
+ pendingData = result.pendingData;
+ }
}
async analytics(
diff --git a/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts b/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts
index 0c224281f..25b843a4f 100644
--- a/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts
@@ -1,5 +1,6 @@
import {
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -7,22 +8,50 @@ import {
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import { RedditSettingsDto } from '@gitroom/nestjs-libraries/dtos/posts/providers-settings/reddit.dto';
import { timer } from '@gitroom/helpers/utils/timer';
-import { groupBy } from 'lodash';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
import { lookup } from 'mime-types';
import axios from 'axios';
import FormDataUpload from 'form-data';
-import WebSocket from 'ws';
import { Tool } from '@gitroom/nestjs-libraries/integrations/tool.decorator';
import { Integration } from '@prisma/client';
import { hasExtension } from '@gitroom/helpers/utils/has.extension';
-// @ts-ignore
-global.WebSocket = WebSocket;
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost. The cursor makes every subreddit submit its own
+// finalizePost call, and `armed` is the submit-in-flight handshake: set by
+// checkPostStatus BEFORE finalizePost runs (the workflow commits it first), so
+// an interrupted submit is detected by the next check instead of re-run.
+type RedditPendingData = {
+ subreddits: RedditSettingsDto['subreddit'];
+ message: string;
+ mediaPath?: string;
+ mediaThumbnail?: string;
+ cursor: number;
+ results: { postId: string; releaseURL: string }[];
+ armed?: {
+ sr: string;
+ title: string;
+ // When this attempt was armed: bounds the listing lookup (so an older
+ // post with the same title is never mistaken for this one) and the
+ // resubmit guard (a timed-out finalize may still be running for up to
+ // the 10-minute activity ceiling).
+ armedAt: number;
+ // Media submissions appear in the listing minutes after the submit
+ // (Reddit creates them asynchronously), so their recovery lookups get a
+ // longer grace before concluding the submit never landed.
+ media: boolean;
+ // true once Reddit accepted the submit (media posts answer with only a
+ // websocket url) - the id is then recovered from the submitted listing,
+ // and the submit must NEVER run again.
+ submitted: boolean;
+ lookups: number;
+ };
+};
export class RedditProvider extends SocialAbstract implements SocialProvider {
override maxConcurrentJob = 1; // Reddit has strict rate limits (1 request per second)
@@ -220,121 +249,380 @@ export class RedditProvider extends SocialAbstract implements SocialProvider {
return location;
}
- async post(
+ // Read-only recovery of a submission: the submit endpoint answers media
+ // posts with only a websocket url, and an interrupted activity may have
+ // submitted without us seeing the response - the user's newest submissions
+ // answer both cases without ever re-submitting.
+ private async findSubmittedPost(
+ accessToken: string,
+ username: string,
+ sr: string,
+ title: string,
+ armedAt: number
+ ): Promise<{ id: string; url: string } | undefined> {
+ const { data } = await (
+ await this.fetch(
+ `https://oauth.reddit.com/user/${username}/submitted?limit=25&sort=new`,
+ {
+ method: 'GET',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ },
+ },
+ 'reddit',
+ 0,
+ true
+ )
+ ).json();
+
+ // Only submissions created after this attempt was armed (with a minute of
+ // clock slack): an older post with the same title in the same subreddit -
+ // an earlier schedule, a template repost - must never be mistaken for
+ // this one. Fall back to a one-hour window for pendingData that predates
+ // the armedAt field.
+ const cutoff = armedAt
+ ? armedAt - 60 * 1000
+ : Date.now() - 60 * 60 * 1000;
+
+ const found = (data?.children || []).find(
+ (c: any) =>
+ (c?.data?.subreddit || '').toLowerCase() === sr &&
+ (!title || (c?.data?.title || '') === title) &&
+ (c?.data?.created_utc || 0) * 1000 > cutoff
+ );
+
+ if (!found?.data?.id) {
+ return undefined;
+ }
+
+ return {
+ id: found.data.id,
+ url: found.data.permalink
+ ? `https://www.reddit.com${found.data.permalink}`
+ : `https://www.reddit.com/r/${sr}`,
+ };
+ }
+
+ private completedResponse(
+ results: RedditPendingData['results']
+ ): PendingCheckResponse {
+ return {
+ status: 'completed',
+ postId: results.map((p) => p.postId).filter(Boolean).join(','),
+ releaseURL: results.map((p) => p.releaseURL).filter(Boolean).join(','),
+ };
+ }
+
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails<RedditSettingsDto>[]
): Promise<PostResponse[]> {
const [post] = postDetails;
- const valueArray: PostResponse[] = [];
- for (const firstPostSettings of post.settings.subreddit) {
- const kind =
- firstPostSettings.value.type === 'media'
- ? hasExtension(post.media[0].path, 'mp4')
- ? 'video'
- : 'image'
- : firstPostSettings.value.type;
- const postData = {
- api_type: 'json',
- title: firstPostSettings.value.title || '',
- kind:
- ['link', 'self', 'image', 'video', 'videogif'].indexOf(kind) > -1
- ? kind
- : 'self',
- ...(firstPostSettings.value.flair
- ? { flair_id: firstPostSettings.value.flair.id }
- : {}),
- ...(firstPostSettings.value.type === 'link'
- ? {
- url: firstPostSettings.value.url,
- }
- : {}),
- ...(firstPostSettings.value.type === 'media'
- ? {
- url: await this.uploadFileToReddit(
- accessToken,
- post.media[0].path
- ),
- ...(hasExtension(post.media[0].path, 'mp4')
- ? {
- video_poster_url: await this.uploadFileToReddit(
- accessToken,
- post.media[0].thumbnail
- ),
- }
- : {}),
- }
- : {}),
- text: post.message,
- sr: firstPostSettings.value.subreddit.replace('/r/', '').toLowerCase(),
- };
-
- const all = await (
- await this.fetch('https://oauth.reddit.com/api/submit', {
- method: 'POST',
- headers: {
- Authorization: `Bearer ${accessToken}`,
- 'Content-Type': 'application/x-www-form-urlencoded',
- },
- body: new URLSearchParams(postData),
- })
- ).json();
-
- const {
- id: redditId,
- name,
- url,
- } = await new Promise<{
- id: string;
- name: string;
- url: string;
- }>((res) => {
- if (all?.json?.data?.id) {
- res(all.json.data);
+ // Nothing is submitted here: finalizePost publishes ONE subreddit per call
+ // (advancing the cursor between calls), so a crashed activity can never
+ // replay the whole loop - the old code restarted from the first subreddit
+ // and duplicated every already-published one.
+ return [
+ {
+ id: post.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ subreddits: post.settings.subreddit,
+ message: post.message,
+ mediaPath: post.media?.[0]?.path,
+ mediaThumbnail: post.media?.[0]?.thumbnail,
+ cursor: 0,
+ results: [],
+ } as RedditPendingData,
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: RedditPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ const data = { ...pendingData };
+
+ // A previous finalizePost may have died mid-submit (or submitted a media
+ // post that only answers over a websocket): ask Reddit what actually
+ // happened before authorizing anything else.
+ if (data.armed) {
+ let found: { id: string; url: string } | undefined;
+
+ // Legacy rows may have no username stored: the listing can never be
+ // read, so fall straight through to the grace budget below.
+ if (integration.profile) {
+ try {
+ found = await this.findSubmittedPost(
+ accessToken,
+ integration.profile,
+ data.armed.sr,
+ data.armed.title,
+ data.armed.armedAt
+ );
+ } catch (err) {
+ if (err instanceof RefreshToken) {
+ throw err;
+ }
+
+ // A failed lookup still consumes the grace budget below: a
+ // permanently broken listing must never wedge the post into
+ // pending until the whole check budget burns down.
}
+ }
- const ws = new WebSocket(all.json.data.websocket_url);
- ws.on('message', (data: any) => {
- setTimeout(() => {
- res({ id: '', name: '', url: '' });
- ws.close();
- }, 30_000);
- try {
- const parsedData = JSON.parse(data.toString());
- if (parsedData?.payload?.redirect) {
- const onlyId = parsedData?.payload?.redirect.replace(
- /https:\/\/www\.reddit\.com\/r\/.*?\/comments\/(.*?)\/.*/g,
- '$1'
- );
- res({
- id: onlyId,
- name: `t3_${onlyId}`,
- url: parsedData?.payload?.redirect,
- });
- }
- } catch (err) {}
- });
- });
-
- valueArray.push({
- postId: redditId,
- releaseURL: url,
- id: post.id,
- status: 'published',
- });
+ if (found) {
+ data.results = [
+ ...data.results,
+ { postId: found.id, releaseURL: found.url },
+ ];
+ data.cursor += 1;
+ data.armed = undefined;
+ } else if (data.armed.submitted) {
+ // Reddit accepted this submit; media posts are created asynchronously
+ // and can lag in the listing for minutes. Keep looking for the real
+ // id (comments and webhooks need it), then accept without one rather
+ // than ever re-submitting (which would duplicate the post).
+ if (data.armed.lookups < 9) {
+ data.armed = { ...data.armed, lookups: data.armed.lookups + 1 };
+ return { status: 'pending', pendingData: data };
+ }
+
+ data.results = [
+ ...data.results,
+ {
+ postId: '',
+ releaseURL: `https://www.reddit.com/r/${data.armed.sr}`,
+ },
+ ];
+ data.cursor += 1;
+ data.armed = undefined;
+ } else if (
+ data.armed.media &&
+ (Date.now() - data.armed.armedAt < 10 * 60 * 1000 ||
+ data.armed.lookups < 9)
+ ) {
+ // A media finalize that timed out may STILL be running on a worker
+ // (activities are not cancelled) for up to its 10-minute
+ // startToCloseTimeout, and its accepted submit can appear in the
+ // listing minutes later - never authorize a resubmit inside that
+ // window. The wall-clock window alone is not enough: a real timeout
+ // consumes it entirely before control returns, so the lookups floor
+ // guarantees several minutes of listing checks AFTER the failure
+ // regardless of how long the dead activity ran.
+ data.armed = { ...data.armed, lookups: data.armed.lookups + 1 };
+ return { status: 'pending', pendingData: data };
+ } else if (!data.armed.media && data.armed.lookups < 1) {
+ // Self/link posts appear in the listing instantly: one grace miss
+ // covers listing lag.
+ data.armed = { ...data.armed, lookups: data.armed.lookups + 1 };
+ return { status: 'pending', pendingData: data };
+ } else {
+ // Armed but no accepted response was ever seen and the in-flight
+ // window has passed: conclude the submit never landed and let
+ // finalizePost run it again.
+ data.armed = undefined;
+ }
+ }
+
+ if (data.cursor >= (data.subreddits?.length || 0)) {
+ return this.completedResponse(data.results);
+ }
+
+ // Arm the next subreddit BEFORE finalizePost submits it: the workflow
+ // commits this state first, so if finalize dies mid-submit the next check
+ // finds the armed marker and asks Reddit instead of resubmitting blindly.
+ const value = data.subreddits[data.cursor].value;
+ data.armed = {
+ sr: value.subreddit.replace('/r/', '').toLowerCase(),
+ title: value.title || '',
+ armedAt: Date.now(),
+ media: value.type === 'media',
+ submitted: false,
+ lookups: 0,
+ };
+
+ return { status: 'ready', pendingData: data };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: RedditPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ const data = { ...pendingData };
+ const entry = data.subreddits?.[data.cursor];
+
+ if (!entry || !data.armed) {
+ // Nothing armed for this cursor: let the check re-derive the state.
+ return { status: 'pending', pendingData: data };
+ }
- if (post.settings.subreddit.length > 1) {
- await timer(5000);
+ const value = entry.value;
+ const kind =
+ value.type === 'media'
+ ? hasExtension(data.mediaPath || '', 'mp4')
+ ? 'video'
+ : 'image'
+ : value.type;
+
+ const postData = {
+ api_type: 'json',
+ title: value.title || '',
+ kind:
+ ['link', 'self', 'image', 'video', 'videogif'].indexOf(kind) > -1
+ ? kind
+ : 'self',
+ ...(value.flair ? { flair_id: value.flair.id } : {}),
+ ...(value.type === 'link'
+ ? {
+ url: value.url,
+ }
+ : {}),
+ ...(value.type === 'media'
+ ? {
+ url: await this.uploadFileToReddit(accessToken, data.mediaPath!),
+ ...(hasExtension(data.mediaPath || '', 'mp4')
+ ? {
+ video_poster_url: await this.uploadFileToReddit(
+ accessToken,
+ data.mediaThumbnail!
+ ),
+ }
+ : {}),
+ }
+ : {}),
+ text: data.message,
+ sr: value.subreddit.replace('/r/', '').toLowerCase(),
+ };
+
+ const all = await (
+ await this.fetch('https://oauth.reddit.com/api/submit', {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ },
+ body: new URLSearchParams(postData),
+ })
+ ).json();
+
+ // Reddit rejects submissions with a 200 and an errors array: surface the
+ // real reason instead of failing later with an unknown outcome.
+ if (all?.json?.errors?.length) {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify(all),
+ Buffer.from('{}'),
+ `Reddit rejected the post to r/${postData.sr}: ${all.json.errors
+ .map((e: any[]) => e?.[1] || e?.[0] || '')
+ .join(', ')}`
+ );
+ }
+
+ // Self/link posts answer with the created post directly.
+ if (all?.json?.data?.id) {
+ data.results = [
+ ...data.results,
+ {
+ postId: all.json.data.id,
+ releaseURL:
+ all.json.data.url || `https://www.reddit.com/r/${postData.sr}`,
+ },
+ ];
+ data.cursor += 1;
+ data.armed = undefined;
+
+ if (data.cursor >= data.subreddits.length) {
+ return this.completedResponse(data.results);
}
+
+ return { status: 'pending', pendingData: data };
}
- return Object.values(groupBy(valueArray, (p) => p.id)).map((p) => ({
- id: p[0].id,
- postId: p.map((p) => p.postId).join(','),
- releaseURL: p.map((p) => p.releaseURL).join(','),
- status: 'published',
- }));
+ // Media posts only answer with a websocket url: mark the submit as
+ // accepted, the next check recovers the post id from the submitted
+ // listing - a held-open websocket has no place inside an activity.
+ data.armed = { ...data.armed, submitted: true };
+ return { status: 'pending', pendingData: data };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they submit every
+ // subreddit inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<RedditSettingsDto>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(id, accessToken, postDetails);
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'ready') {
+ const finalize = await this.finalizePost(
+ accessToken,
+ check.pendingData,
+ integration
+ );
+
+ if (finalize.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: finalize.postId,
+ releaseURL: finalize.releaseURL,
+ status: 'published',
+ },
+ ];
+ }
+
+ pendingData = finalize.pendingData;
+ } else if (check.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ postId: check.postId,
+ releaseURL: check.releaseURL,
+ status: 'published',
+ },
+ ];
+ } else {
+ pendingData = check.pendingData;
+ }
+
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: some subreddits may already be live at this point, so
+ // timing the activity out (which retries and re-submits) is far worse
+ // than failing here - the old websocket wait could hang forever.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ Buffer.from('{}'),
+ 'Reddit took too long to answer, please check your account before posting again'
+ );
+ }
+
+ // Keeps the old 5-second spacing between subreddit submissions.
+ await timer(5000);
+ }
}
async comment(
diff --git a/libraries/nestjs-libraries/src/integrations/social/x.provider.ts b/libraries/nestjs-libraries/src/integrations/social/x.provider.ts
index b908485ee..e3bd9b3cb 100644
--- a/libraries/nestjs-libraries/src/integrations/social/x.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/x.provider.ts
@@ -3,6 +3,7 @@ import { createHmac, randomBytes } from 'crypto';
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -12,6 +13,7 @@ import sharp from 'sharp';
import { readOrFetch } from '@gitroom/helpers/utils/read.or.fetch';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
import { Plug } from '@gitroom/helpers/decorators/plug.decorator';
@@ -26,6 +28,32 @@ import { XDto } from '@gitroom/nestjs-libraries/dtos/posts/providers-settings/x.
import { Rules } from '@gitroom/nestjs-libraries/chat/rules.description.decorator';
import { hasExtension } from '@gitroom/helpers/utils/has.extension';
+// Travels through the workflow history between postPending, checkPostStatus
+// and finalizePost - keep it small JSON (media ids and the tweet content).
+type XPendingData = {
+ message: string;
+ settings: {
+ who_can_reply_post?:
+ | 'everyone'
+ | 'following'
+ | 'mentionedUsers'
+ | 'subscribers'
+ | 'verified';
+ community?: string;
+ made_with_ai?: boolean;
+ paid_partnership?: boolean;
+ };
+ mediaIds: string[];
+ // Media still transcoding on X's side, waiting for STATUS = succeeded.
+ processingIds: string[];
+ // Arm -> confirm -> publish handshake (same as the Facebook story flow):
+ // finalizePost arms without mutating, checkPostStatus witnesses, and only a
+ // witnessed attempt runs the create - so a create that dies with an unknown
+ // outcome is detected instead of run again (X has no idempotency key).
+ attempting?: boolean;
+ confirmed?: boolean;
+};
+
@Rules(
`X can have maximum 4 pictures, or maximum one video, it can also be without attachments ${
process.env.STRIP_LINKS_FROM_X_POSTS
@@ -39,7 +67,12 @@ export class XProvider extends SocialAbstract implements SocialProvider {
isBetweenSteps = false;
scopes = [] as string[];
stripLinks = () => !!process.env.STRIP_LINKS_FROM_X_POSTS;
- override maxConcurrentJob = 1; // X has strict rate limits (300 posts per 3 hours)
+ // X rate limits are per user (300 posts / 3 hours), not per app, so the cap
+ // only needs to keep bursts polite. With the pending flow the slot is held
+ // for actual API work only (processing waits live in workflow timers), so a
+ // single slot is no longer required - it would serialize every customer's
+ // status checks behind uploads.
+ override maxConcurrentJob = 10;
toolTip =
'You will be logged in into your current account, if you would like a different account, change it first on X';
@@ -463,12 +496,58 @@ export class XProvider extends SocialAbstract implements SocialProvider {
};
}>(`media/upload/${mediaId}/finalize`);
- let processing = finalize.data.processing_info;
+ const processing = finalize.data.processing_info;
+
+ // An explicit rejection right at finalize: the video will never process.
+ if (processing?.state === 'failed') {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify(processing),
+ Buffer.from('{}'),
+ `X failed to process the uploaded video${
+ (processing as any)?.error?.message
+ ? `: ${(processing as any).error.message}`
+ : ''
+ }`
+ );
+ }
+
+ // Per the docs a missing processing_info means the media is ready to use;
+ // anything else keeps transcoding asynchronously and must reach
+ // `succeeded` before the media_id can be attached to a post.
+ return {
+ mediaId,
+ processing: !!processing && processing.state !== 'succeeded',
+ };
+ }
+
+ // Single STATUS read for a media_id, no loops and no timers - the polling
+ // loop lives in the post workflow (checkPostStatus) or, for the legacy
+ // paths, in waitForMediaProcessing.
+ private async mediaProcessingStatus(client: TwitterApi, mediaId: string) {
+ const status = await client.v2.get<{
+ data: {
+ processing_info?: {
+ state: string;
+ check_after_secs?: number;
+ error?: { message?: string };
+ };
+ };
+ }>('media/upload', { command: 'STATUS', media_id: mediaId });
+
+ return status.data.processing_info;
+ }
+
+ // Blocking processing wait, used by the paths that still resolve everything
+ // inside one activity (comments, and post() for pre-v1.0.6 workflows).
+ private async waitForMediaProcessing(client: TwitterApi, mediaId: string) {
// X drives the pace via check_after_secs; cap on accumulated wait time
// (long videos can legitimately process for many minutes) instead of an
// attempt count, but never poll forever.
let waitedMs = 0;
- const maxWaitMs = 30 * 60 * 1000;
+ const maxWaitMs = 7 * 60 * 1000;
+ let processing = await this.mediaProcessingStatus(client, mediaId);
+
while (processing && processing.state !== 'succeeded') {
if (processing.state === 'failed' || waitedMs >= maxWaitMs) {
throw new BadBody(
@@ -476,9 +555,7 @@ export class XProvider extends SocialAbstract implements SocialProvider {
JSON.stringify(processing),
Buffer.from('{}'),
`X failed to process the uploaded video${
- (processing as any)?.error?.message
- ? `: ${(processing as any).error.message}`
- : ''
+ processing?.error?.message ? `: ${processing.error.message}` : ''
}`
);
}
@@ -486,59 +563,103 @@ export class XProvider extends SocialAbstract implements SocialProvider {
const waitMs = (processing.check_after_secs || 1) * 1000;
await timer(waitMs);
waitedMs += waitMs;
- const status = await client.v2.get<{
- data: {
- processing_info?: { state: string; check_after_secs?: number };
- };
- }>('media/upload', { command: 'STATUS', media_id: mediaId });
- processing = status.data.processing_info;
+ processing = await this.mediaProcessingStatus(client, mediaId);
}
+ }
- return mediaId;
+ // With ten X activities running concurrently (maxConcurrentJob), a user
+ // publishing several posts at the same minute can 429 on the upload
+ // endpoints; twitter-api-v2 errors never pass through this.fetch's backoff,
+ // so retry them here instead of hard-failing the post. Nothing is published
+ // at upload time, so a retried upload can never duplicate a post.
+ private async uploadWithRateLimitRetry<T>(
+ func: () => Promise<T>,
+ totalRetries = 0
+ ): Promise<T> {
+ try {
+ return await func();
+ } catch (err: any) {
+ if (totalRetries <= 2 && (err?.code === 429 || err?.rateLimitError)) {
+ await timer(5000 * (totalRetries + 1));
+ return this.uploadWithRateLimitRetry(func, totalRetries + 1);
+ }
+
+ throw err;
+ }
}
- private async uploadMedia(
+ private async uploadMediaEntries(
client: TwitterApi,
postDetails: PostDetails<any>[]
) {
// Media is uploaded sequentially on purpose: uploading everything with
// Promise.all holds every file in memory at the same time.
const media = {} as Record<string, string[]>;
+ const processingIds: string[] = [];
for (const p of postDetails) {
for (const m of p?.media || []) {
- const id = await this.runInConcurrent(
+ const uploaded = await this.runInConcurrent(
async () =>
hasExtension(m.path, 'mp4')
- ? this.uploadVideoInChunks(client, m.path)
- : client.v2.uploadMedia(
- await sharp(await readOrFetch(m.path), {
- animated: lookup(m.path) === 'image/gif',
- })
- .resize({
- width: 1000,
- })
- .gif()
- .toBuffer(),
- {
- media_type: (lookup(m.path) || '') as any,
- }
- ),
+ ? this.uploadWithRateLimitRetry(() =>
+ this.uploadVideoInChunks(client, m.path)
+ )
+ : {
+ mediaId: await this.uploadWithRateLimitRetry(async () =>
+ client.v2.uploadMedia(
+ await sharp(await readOrFetch(m.path), {
+ animated: lookup(m.path) === 'image/gif',
+ })
+ .resize({
+ width: 1000,
+ })
+ .gif()
+ .toBuffer(),
+ {
+ media_type: (lookup(m.path) || '') as any,
+ }
+ )
+ ),
+ processing: false,
+ },
true
);
- if (!id) {
+ if (!uploaded?.mediaId) {
continue;
}
media[p.id] = media[p.id] || [];
- media[p.id].push(id);
+ media[p.id].push(uploaded.mediaId);
+
+ if (uploaded.processing) {
+ processingIds.push(uploaded.mediaId);
+ }
}
}
+ return { media, processingIds };
+ }
+
+ // Legacy blocking upload (comments, and pre-v1.0.6 workflows through
+ // post()): waits for the processing inside the activity like before.
+ private async uploadMedia(
+ client: TwitterApi,
+ postDetails: PostDetails<any>[]
+ ) {
+ const { media, processingIds } = await this.uploadMediaEntries(
+ client,
+ postDetails
+ );
+
+ for (const mediaId of processingIds) {
+ await this.waitForMediaProcessing(client, mediaId);
+ }
+
return media;
}
- async post(
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails<{
@@ -556,37 +677,169 @@ export class XProvider extends SocialAbstract implements SocialProvider {
}>[],
integration: Integration
): Promise<PostResponse[]> {
- const [accessTokenSplit, accessSecretSplit] = accessToken.split(':');
const client = await this.getClient(accessToken);
-
const [firstPost] = postDetails;
- // upload media for the first post
- const uploadAll = await this.uploadMedia(client, [firstPost]);
+ // Upload the media now; the transcoding wait moves to checkPostStatus and
+ // the tweet itself is only created by finalizePost, so nothing here is
+ // irreversible - a failure leaves only orphaned media.
+ const { media, processingIds } = await this.uploadMediaEntries(client, [
+ firstPost,
+ ]);
+
+ return [
+ {
+ id: firstPost.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ message: firstPost.message,
+ settings: {
+ who_can_reply_post: firstPost?.settings?.who_can_reply_post,
+ community: firstPost?.settings?.community,
+ made_with_ai: firstPost?.settings?.made_with_ai,
+ paid_partnership: firstPost?.settings?.paid_partnership,
+ },
+ mediaIds: (media[firstPost.id] || []).filter((f) => f),
+ processingIds,
+ } as XPendingData,
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: XPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // A confirmed create attempt died without reporting its result: X gives
+ // no cheap way to ask whether that tweet was created, so never run the
+ // create again - stop with an explicit warning instead.
+ if (pendingData.attempting && pendingData.confirmed) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ Buffer.from('{}'),
+ 'X may have already published this post, please check your account before posting again to avoid duplicates'
+ );
+ }
+
+ const client = await this.getClient(accessToken);
+
+ // Check every media still transcoding; keep the ones not succeeded yet.
+ const stillProcessing: string[] = [];
+ for (const mediaId of pendingData.processingIds || []) {
+ let processing:
+ | { state: string; check_after_secs?: number; error?: { message?: string } }
+ | undefined;
+ try {
+ processing = await this.mediaProcessingStatus(client, mediaId);
+ } catch (err: any) {
+ // twitter-api-v2 throws ApiResponseError, which never passes through
+ // this.fetch/handleErrors: classify it here so revoked tokens and
+ // suspended accounts fail properly instead of burning the whole check
+ // budget as "transient".
+ const body = JSON.stringify(err?.data || {});
+ const handleError = this.handleErrors(body);
+
+ if (err?.code === 401 || handleError?.type === 'refresh-token') {
+ throw new RefreshToken(this.identifier, body, Buffer.from('{}'));
+ }
+
+ if (handleError?.type === 'bad-body') {
+ throw new BadBody(
+ this.identifier,
+ body,
+ Buffer.from('{}'),
+ handleError.value
+ );
+ }
+
+ // Transient status-check error: the media may finish transcoding just
+ // fine, keep polling - if X stays broken the workflow exhausts its
+ // checks and warns the user properly.
+ return { status: 'pending', pendingData };
+ }
+
+ if (processing?.state === 'failed') {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify(processing),
+ Buffer.from('{}'),
+ `X failed to process the uploaded video${
+ processing?.error?.message ? `: ${processing.error.message}` : ''
+ }`
+ );
+ }
+
+ // A missing processing_info means the media is ready to use.
+ if (processing && processing.state !== 'succeeded') {
+ stillProcessing.push(mediaId);
+ }
+ }
+
+ if (stillProcessing.length) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, processingIds: stillProcessing },
+ };
+ }
+
+ // witness the armed create so finalizePost knows the attempt is uniquely
+ // accounted for before it mutates anything
+ if (pendingData.attempting && !pendingData.confirmed) {
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, processingIds: [], confirmed: true },
+ };
+ }
+
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, processingIds: [] },
+ };
+ }
- const media_ids = (uploadAll[firstPost.id] || []).filter((f) => f);
+ override async finalizePost(
+ accessToken: string,
+ pendingData: XPendingData,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Create with an arm -> confirm -> publish handshake: the create only runs
+ // after checkPostStatus witnessed the intent, so a run that dies
+ // mid-create is detectable and the tweet is never published twice.
+ if (!pendingData.attempting || !pendingData.confirmed) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, attempting: true, confirmed: false },
+ };
+ }
+
+ const [accessTokenSplit, accessSecretSplit] = accessToken.split(':');
+ const settings = pendingData.settings || {};
+ const mediaIds = (pendingData.mediaIds || []).filter((f) => f);
const tweetUrl = 'https://api.x.com/2/tweets';
const tweetBody = {
- ...(!firstPost?.settings?.who_can_reply_post ||
- firstPost?.settings?.who_can_reply_post === 'everyone'
+ ...(!settings.who_can_reply_post ||
+ settings.who_can_reply_post === 'everyone'
? {}
: {
- reply_settings: firstPost?.settings?.who_can_reply_post,
+ reply_settings: settings.who_can_reply_post,
}),
- ...(firstPost?.settings?.community
+ ...(settings.community
? {
share_with_followers: true,
- community_id:
- firstPost?.settings?.community?.split('/').pop() || '',
+ community_id: settings.community?.split('/').pop() || '',
}
: {}),
text: this.stripLinks()
- ? removeLinks(firstPost.message)
- : firstPost.message,
- ...(media_ids.length ? { media: { media_ids } } : {}),
- made_with_ai: this.assetBoolean(firstPost?.settings?.made_with_ai),
- paid_partnership: this.assetBoolean(firstPost?.settings?.paid_partnership),
+ ? removeLinks(pendingData.message)
+ : pendingData.message,
+ ...(mediaIds.length ? { media: { media_ids: mediaIds } } : {}),
+ made_with_ai: this.assetBoolean(settings.made_with_ai),
+ paid_partnership: this.assetBoolean(settings.paid_partnership),
};
const tweetResponse = await this.fetch(tweetUrl, {
@@ -606,14 +859,91 @@ export class XProvider extends SocialAbstract implements SocialProvider {
data: { id: string };
};
- return [
- {
- postId: data.id,
- id: firstPost.id,
- releaseURL: `https://twitter.com/${integration.profile}/status/${data.id}`,
- status: 'posted',
- },
- ];
+ return {
+ status: 'completed',
+ postId: data.id,
+ releaseURL: `https://twitter.com/${integration.profile}/status/${data.id}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they wait for the
+ // transcoding and create the tweet inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<{
+ active_thread_finisher: boolean;
+ thread_finisher: string;
+ community?: string;
+ who_can_reply_post:
+ | 'everyone'
+ | 'following'
+ | 'mentionedUsers'
+ | 'subscribers'
+ | 'verified';
+ made_with_ai?: boolean;
+ paid_partnership?: boolean;
+ }>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (the tweet is only created once the
+ // media is ready), timing the activity out is not - a retried activity
+ // would upload and publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ Buffer.from('{}'),
+ 'X took too long to process the media, please try again'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(20000);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [
+ {
+ postId: result.postId,
+ id: response.id,
+ releaseURL: result.releaseURL,
+ status: 'posted',
+ },
+ ];
+ }
+
+ // finalize only armed the handshake (nothing to wait for), loop straight
+ // into the witnessing check
+ pendingData = result.pendingData;
+ }
}
async comment(