Skip to content

πŸ“¨ The ultimate agentic social media scheduling tool πŸ€–

fix: prevent duplicate posts with a pending-post workflow (v1.0.6)

Nevo Davide2cdd5e0df67
diff --git a/apps/orchestrator/src/activities/post.activity.ts b/apps/orchestrator/src/activities/post.activity.ts
index 781d78901..a3abc344e 100644
--- a/apps/orchestrator/src/activities/post.activity.ts
+++ b/apps/orchestrator/src/activities/post.activity.ts
@@ -76,7 +76,7 @@ export class PostActivity {
for (const post of list) {
await this._temporalService.client
.getRawClient()
- .workflow.signalWithStart('postWorkflowV105', {
+ .workflow.signalWithStart('postWorkflowV106', {
workflowId: `post_${post.id}`,
taskQueue: 'main',
signal: 'poke',
@@ -206,6 +206,23 @@ export class PostActivity {
@ActivityMethod()
async postSocial(integration: Integration, posts: Post[]) {
+ return this.postSocialInternal(integration, posts, false);
+ }
+
+ // Used by postWorkflowV106 and up: providers that implement `postPending`
+ // return a `pending` response the workflow resolves via checkPostStatus /
+ // finalizePost. Older workflow versions keep calling `postSocial` and get
+ // the old blocking behavior.
+ @ActivityMethod()
+ async postSocialPending(integration: Integration, posts: Post[]) {
+ return this.postSocialInternal(integration, posts, true);
+ }
+
+ private async postSocialInternal(
+ integration: Integration,
+ posts: Post[],
+ allowPending: boolean
+ ) {
if (process.env.STRIPE_SECRET_KEY) {
const subscription = await this._subscriptionService.getSubscription(
integration.organizationId
@@ -225,47 +242,89 @@ export class PostActivity {
posts
);
- const postNow = await getIntegration.post(
- integration.internalId,
+ const mappedPosts = await Promise.all(
+ (newPosts || []).map(async (p) => ({
+ id: p.id,
+ message: stripHtmlValidation(
+ getIntegration.editor,
+ p.content,
+ true,
+ false,
+ !/<\/?[a-z][\s\S]*>/i.test(p.content),
+ getIntegration.mentionFormat
+ ),
+ settings: JSON.parse(p.settings || '{}'),
+ media: await this._postService.updateMedia(
+ p.id,
+ JSON.parse(p.image || '[]'),
+ getIntegration?.convertToJPEG || false
+ ),
+ }))
+ );
+
+ const postNow =
+ allowPending && getIntegration.postPending
+ ? await getIntegration.postPending(
+ integration.internalId,
+ integration.token,
+ mappedPosts,
+ integration
+ )
+ : await getIntegration.post(
+ integration.internalId,
+ integration.token,
+ mappedPosts,
+ integration
+ );
+
+ // The post is already published at this point: the streak is best-effort,
+ // failing the activity here would retry it and publish again.
+ try {
+ await this._temporalService.client
+ .getRawClient()
+ .workflow.start('streakWorkflow', {
+ args: [{ organizationId: integration.organizationId }],
+ workflowId: `streak_${integration.organizationId}`,
+ taskQueue: 'main',
+ workflowIdConflictPolicy: 'TERMINATE_EXISTING',
+ typedSearchAttributes: new TypedSearchAttributes([
+ {
+ key: organizationId,
+ value: integration.organizationId,
+ },
+ ]),
+ });
+ } catch (err) {
+ /**empty**/
+ }
+
+ return postNow;
+ }
+
+ @ActivityMethod()
+ async checkPostStatus(integration: Integration, pendingData: any) {
+ const getIntegration = this._integrationManager.getSocialIntegration(
+ integration.providerIdentifier
+ );
+
+ return getIntegration.checkPostStatus(
integration.token,
- await Promise.all(
- (newPosts || []).map(async (p) => ({
- id: p.id,
- message: stripHtmlValidation(
- getIntegration.editor,
- p.content,
- true,
- false,
- !/<\/?[a-z][\s\S]*>/i.test(p.content),
- getIntegration.mentionFormat
- ),
- settings: JSON.parse(p.settings || '{}'),
- media: await this._postService.updateMedia(
- p.id,
- JSON.parse(p.image || '[]'),
- getIntegration?.convertToJPEG || false
- ),
- }))
- ),
+ pendingData,
integration
);
+ }
- await this._temporalService.client
- .getRawClient()
- .workflow.start('streakWorkflow', {
- args: [{ organizationId: integration.organizationId }],
- workflowId: `streak_${integration.organizationId}`,
- taskQueue: 'main',
- workflowIdConflictPolicy: 'TERMINATE_EXISTING',
- typedSearchAttributes: new TypedSearchAttributes([
- {
- key: organizationId,
- value: integration.organizationId,
- },
- ]),
- });
+ @ActivityMethod()
+ async finalizePost(integration: Integration, pendingData: any) {
+ const getIntegration = this._integrationManager.getSocialIntegration(
+ integration.providerIdentifier
+ );
- return postNow;
+ return getIntegration.finalizePost(
+ integration.token,
+ pendingData,
+ integration
+ );
}
@ActivityMethod()
diff --git a/apps/orchestrator/src/workflows/index.ts b/apps/orchestrator/src/workflows/index.ts
index ad6e8c45a..8b9edc94f 100644
--- a/apps/orchestrator/src/workflows/index.ts
+++ b/apps/orchestrator/src/workflows/index.ts
@@ -3,6 +3,7 @@ export * from './post-workflows/post.workflow.v1.0.2';
export * from './post-workflows/post.workflow.v1.0.3';
export * from './post-workflows/post.workflow.v1.0.4';
export * from './post-workflows/post.workflow.v1.0.5';
+export * from './post-workflows/post.workflow.v1.0.6';
export * from './autopost.workflow';
export * from './digest.email.workflow';
export * from './missing.post.workflow';
diff --git a/apps/orchestrator/src/workflows/post-workflows/post.workflow.v1.0.6.ts b/apps/orchestrator/src/workflows/post-workflows/post.workflow.v1.0.6.ts
new file mode 100644
index 000000000..6491c5f60
--- /dev/null
+++ b/apps/orchestrator/src/workflows/post-workflows/post.workflow.v1.0.6.ts
@@ -0,0 +1,643 @@
+import { PostActivity } from '@gitroom/orchestrator/activities/post.activity';
+import {
+ ActivityFailure,
+ ApplicationFailure,
+ startChild,
+ proxyActivities,
+ sleep,
+ defineSignal,
+ setHandler,
+} from '@temporalio/workflow';
+import dayjs from 'dayjs';
+import { Integration } from '@prisma/client';
+import { capitalize, sortBy } from 'lodash';
+import { PostResponse } from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
+import { TimeoutFailure, TypedSearchAttributes } from '@temporalio/common';
+import { postId as postIdSearchParam } from '@gitroom/nestjs-libraries/temporal/temporal.search.attribute';
+
+const proxyTaskQueue = (taskQueue: string) => {
+ return proxyActivities<PostActivity>({
+ startToCloseTimeout: '10 minute',
+ taskQueue,
+ retry: {
+ maximumAttempts: 3,
+ backoffCoefficient: 1,
+ initialInterval: '2 minutes',
+ },
+ });
+};
+
+// checkPostStatus is a single read-only status call, so it gets a short timeout
+// and fast retries - retrying it can never duplicate a post.
+const proxyCheckTaskQueue = (taskQueue: string) => {
+ return proxyActivities<PostActivity>({
+ startToCloseTimeout: '2 minute',
+ taskQueue,
+ retry: {
+ maximumAttempts: 3,
+ backoffCoefficient: 1,
+ initialInterval: '10 seconds',
+ },
+ });
+};
+
+// postSocialPending / finalizePost run irreversible publishing mutations, so no
+// automatic retries - a retried activity whose previous (timed-out) attempt
+// still completed in the background would publish twice. The workflow retries
+// deliberately, and treats timeouts as "outcome unknown".
+const proxyMutationTaskQueue = (taskQueue: string) => {
+ return proxyActivities<PostActivity>({
+ startToCloseTimeout: '10 minute',
+ taskQueue,
+ retry: {
+ maximumAttempts: 1,
+ },
+ });
+};
+
+const {
+ getPostsList,
+ getPost,
+ inAppNotification,
+ changeState,
+ updatePost,
+ sendWebhooks,
+ isCommentable,
+} = proxyActivities<PostActivity>({
+ startToCloseTimeout: '10 minute',
+ retry: {
+ maximumAttempts: 3,
+ backoffCoefficient: 1,
+ initialInterval: '2 minutes',
+ },
+});
+
+const poke = defineSignal('poke');
+
+const iterate = Array.from({ length: 5 });
+
+// ~15 minutes at 20s interval (longer than the old in-activity loop, timers are free)
+const maxPendingChecks = 45;
+
+export async function postWorkflowV106({
+ taskQueue,
+ postId,
+ organizationId,
+ postNow = false,
+}: {
+ taskQueue: string;
+ postId: string;
+ organizationId: string;
+ postNow?: boolean;
+}) {
+ // Dynamic task queue, for concurrency
+ const {
+ postComment,
+ getIntegrationById,
+ refreshTokenWithCause,
+ internalPlugs,
+ globalPlugs,
+ processInternalPlug,
+ processPlug,
+ } = proxyTaskQueue(taskQueue);
+
+ const { checkPostStatus } = proxyCheckTaskQueue(taskQueue);
+
+ const { postSocialPending, finalizePost } = proxyMutationTaskQueue(taskQueue);
+
+ let poked = false;
+ setHandler(poke, () => {
+ poked = true;
+ });
+
+ const startTime = new Date();
+ // get all the posts and comments to post
+ const firstPost = await getPost(organizationId, postId);
+
+ // in case doesn't exists for some reason, fail it
+ if (!firstPost) {
+ await changeState(postId, 'ERROR', 'No Post');
+ return;
+ }
+
+ if (!postNow && firstPost.state !== 'QUEUE') {
+ await changeState(firstPost.id, 'ERROR', 'Already posted', [firstPost]);
+ return;
+ }
+
+ // if it's a repeatable post, we should ignore this.
+ if (!postNow) {
+ await sleep(
+ dayjs(firstPost.publishDate).isBefore(dayjs())
+ ? 0
+ : dayjs(firstPost.publishDate).diff(dayjs(), 'millisecond')
+ );
+ }
+
+ const postsListBefore = await getPostsList(organizationId, postId);
+ const [post] = postsListBefore;
+
+ if (!post) {
+ await changeState(postId, 'ERROR', 'No Post');
+ return;
+ }
+
+ // if refresh is needed from last time, let's inform the user
+ if (post.integration?.refreshNeeded) {
+ await inAppNotification(
+ post.organizationId,
+ `We couldn't post to ${post.integration?.providerIdentifier} for ${post?.integration?.name}`,
+ `We couldn't post to ${post.integration?.providerIdentifier} for ${post?.integration?.name} because you need to reconnect it. Please enable it and try again.`,
+ true,
+ false,
+ 'info'
+ );
+
+ await changeState(
+ postsListBefore[0].id,
+ 'ERROR',
+ 'Refresh channel needed',
+ postsListBefore
+ );
+ return;
+ }
+
+ // if it's disabled, inform the user
+ if (post.integration?.disabled) {
+ await inAppNotification(
+ post.organizationId,
+ `We couldn't post to ${post.integration?.providerIdentifier} for ${post?.integration?.name}`,
+ `We couldn't post to ${post.integration?.providerIdentifier} for ${post?.integration?.name} because it's disabled. Please enable it and try again.`,
+ true,
+ false,
+ 'info'
+ );
+
+ await changeState(
+ postsListBefore[0].id,
+ 'ERROR',
+ 'Channel disabled',
+ postsListBefore
+ );
+ return;
+ }
+
+ // Do we need to post comment for this social?
+ const toComment: boolean =
+ postsListBefore.length === 1
+ ? false
+ : await isCommentable(post.integration);
+
+ const postsList = toComment ? postsListBefore : [postsListBefore[0]];
+
+ // list of all the saved results
+ const postsResults: PostResponse[] = [];
+
+ // Every catch block below used to repeat the same failure classification, so
+ // it is centralized here: detect the failure type, refresh the token when
+ // needed, and tell the caller what to do.
+ // 'retry' - the token was refreshed, run the action again
+ // 'stop' - the token could not be refreshed
+ // 'bad-body' - the platform rejected the action
+ // 'timeout' - the activity timed out, its outcome is unknown
+ // 'unknown' - anything else (transient errors)
+ const handleActivityError = async (
+ err: unknown,
+ getIntegration?: () => Promise<any>
+ ): Promise<{
+ type: 'retry' | 'stop' | 'bad-body' | 'timeout' | 'unknown';
+ message: string;
+ }> => {
+ if (
+ err instanceof ActivityFailure &&
+ err.cause instanceof TimeoutFailure
+ ) {
+ return { type: 'timeout', message: '' };
+ }
+
+ const cause =
+ err instanceof ActivityFailure && err.cause instanceof ApplicationFailure
+ ? err.cause
+ : undefined;
+
+ if (cause?.type === 'refresh_token') {
+ const refresh = await refreshTokenWithCause(
+ getIntegration ? await getIntegration() : post.integration,
+ cause.message || ''
+ );
+ if (!refresh || !refresh.accessToken) {
+ return { type: 'stop', message: cause.message || '' };
+ }
+
+ if (!getIntegration) {
+ post.integration.token = refresh.accessToken;
+ }
+
+ return { type: 'retry', message: cause.message || '' };
+ }
+
+ if (cause?.type === 'bad_body') {
+ return { type: 'bad-body', message: cause.message || '' };
+ }
+
+ return { type: 'unknown', message: '' };
+ };
+
+ // The platform may have accepted the post but we can't confirm it was
+ // published - mark the error with a distinct message so the user checks the
+ // account before reposting manually and duplicating it.
+ const markUnconfirmed = async (err: any) => {
+ await changeState(postsList[0].id, 'ERROR', err, postsList);
+ await inAppNotification(
+ post.organizationId,
+ `We couldn't confirm your post on ${capitalize(
+ post.integration?.providerIdentifier
+ )}`,
+ `Your post was sent to ${capitalize(
+ post.integration?.providerIdentifier
+ )}, but we couldn't confirm it was published. Please check your ${
+ post?.integration?.name
+ } account before posting again to avoid duplicates.`,
+ true,
+ false,
+ 'fail'
+ );
+ };
+
+ // The post/comment was already accepted by the platform but returned as
+ // "pending": poll the read-only status check with durable timers until it
+ // completes. Errors are fully handled here (never rethrown), otherwise they
+ // would bubble to the posting retry loop and re-run the publish.
+ const resolvePending = async (
+ pending: PostResponse
+ ): Promise<PostResponse | false> => {
+ let pendingData = pending.pendingData;
+ let errorAttempts = 0;
+
+ for (let check = 0; check < maxPendingChecks; check++) {
+ try {
+ let result = await checkPostStatus(post.integration, pendingData);
+
+ // commit the check's state BEFORE finalizePost runs: if finalize dies
+ // mid-mutation, the next check must see what it had already authorized,
+ // so providers can detect the interrupted attempt instead of running
+ // the mutation again
+ if (result.status !== 'completed') {
+ pendingData = result.pendingData;
+ }
+
+ // polling is done, run the remaining provider mutations
+ if (result.status === 'ready') {
+ result = await finalizePost(post.integration, result.pendingData);
+ }
+
+ if (result.status === 'completed') {
+ return {
+ id: pending.id,
+ postId: result.postId,
+ releaseURL: result.releaseURL,
+ status: 'success',
+ };
+ }
+
+ pendingData = result.pendingData;
+ } catch (err) {
+ const handle = await handleActivityError(err);
+
+ // token refreshed, check again right away
+ if (handle.type === 'retry') {
+ continue;
+ }
+
+ // the token could not be refreshed while checking, but the platform
+ // already accepted the post - warn about a possible live post
+ if (handle.type === 'stop') {
+ await markUnconfirmed(err);
+ return false;
+ }
+
+ // the platform explicitly failed the post, it was not published
+ if (handle.type === 'bad-body') {
+ await changeState(postsList[0].id, 'ERROR', err, postsList);
+ await inAppNotification(
+ post.organizationId,
+ `Error posting on ${post.integration?.providerIdentifier} for ${post?.integration?.name}`,
+ `An error occurred while posting on ${
+ post.integration?.providerIdentifier
+ }${handle.message ? `: ${handle.message}` : ``}`,
+ true,
+ false,
+ 'fail'
+ );
+ return false;
+ }
+
+ // unknown error on a read-only check, retry a few more times
+ errorAttempts++;
+ if (errorAttempts >= iterate.length) {
+ break;
+ }
+ }
+
+ // the platform is still processing, wait before the next check
+ await sleep('20 seconds');
+ }
+
+ // no verdict from the platform after all the checks
+ await markUnconfirmed('Could not confirm the post status');
+ return false;
+ };
+
+ // iterate over the posts
+ for (let i = 0; i < postsList.length; i++) {
+ const before = postsResults.length;
+ // once the platform accepted the post, the catch below must never retry
+ // the publish - retrying after updatePost / notification errors would
+ // duplicate the post
+ let posted = false;
+ let updated = false;
+ // this is a small trick to repeat an action in case of token refresh
+ for (const _ of iterate) {
+ try {
+ // first post the main post
+ if (i === 0) {
+ postsResults.push(
+ ...(await postSocialPending(post.integration as Integration, [
+ postsList[i],
+ ]))
+ );
+
+ // then post the comments if any
+ } else {
+ if (postsList[i].delay) {
+ await sleep(60000 * Math.max(0, Number(postsList[i].delay ?? 0)));
+ }
+
+ postsResults.push(
+ ...(await postComment(
+ postsResults[0].postId,
+ postsResults.length === 1
+ ? undefined
+ : postsResults[i - 1].postId,
+ post.integration,
+ [postsList[i]]
+ ))
+ );
+ }
+
+ posted = true;
+
+ // the platform accepted the post but is still processing it: resolve
+ // it here before marking anything, resolvePending handles its own
+ // errors so a failed status check can never re-run the publish above
+ if (postsResults[i].status === 'pending') {
+ let resolved: PostResponse | false = false;
+ try {
+ resolved = await resolvePending(postsResults[i]);
+ } catch (err) {
+ // never let a pending-resolution error reach the outer catch, it
+ // would retry the post and duplicate it. Best-effort error state,
+ // otherwise the post stays in QUEUE and the missing-posts sweep
+ // would re-publish it.
+ try {
+ await markUnconfirmed(err);
+ } catch (e) {
+ /**empty**/
+ }
+ resolved = false;
+ }
+ if (!resolved) {
+ return false;
+ }
+ postsResults[i] = resolved;
+ }
+
+ // mark post as successful
+ await updatePost(
+ postsList[i].id,
+ postsResults[i].postId,
+ postsResults[i].releaseURL
+ );
+ updated = true;
+
+ if (i === 0) {
+ // send notification on a sucessful post
+ await inAppNotification(
+ post.integration.organizationId,
+ `Your post has been published on ${capitalize(
+ post.integration.providerIdentifier
+ )}`,
+ `Your post has been published on ${capitalize(
+ post.integration.providerIdentifier
+ )} at ${postsResults[0].releaseURL}`,
+ true,
+ true
+ );
+ }
+
+ // break the current while to move to the next post
+ break;
+ } catch (err) {
+ // the post is already live: never re-run the publish
+ if (posted) {
+ if (!updated) {
+ // still marked QUEUE, record the error so the missing-posts sweep
+ // doesn't re-publish it
+ try {
+ await markUnconfirmed(err);
+ } catch (e) {
+ /**empty**/
+ }
+ return false;
+ }
+
+ // already marked published, a failed notification shouldn't abort
+ // the rest of the flow
+ break;
+ }
+
+ const handle = await handleActivityError(err);
+
+ // token refreshed, repeat the action
+ if (handle.type === 'retry') {
+ continue;
+ }
+
+ // the activity timed out: the platform may still complete the publish
+ // in the background, so never retry it
+ if (handle.type === 'timeout') {
+ try {
+ await markUnconfirmed(err);
+ } catch (e) {
+ /**empty**/
+ }
+ return false;
+ }
+
+ // for other errors, change state and inform the user if needed
+ await changeState(postsList[0].id, 'ERROR', err, postsList);
+
+ if (handle.type === 'stop') {
+ return false;
+ }
+
+ // specific case for bad body errors
+ if (handle.type === 'bad-body') {
+ await inAppNotification(
+ post.organizationId,
+ `Error posting${i === 0 ? ' ' : ' comments '}on ${
+ post.integration?.providerIdentifier
+ } for ${post?.integration?.name}`,
+ `An error occurred while posting${i === 0 ? ' ' : ' comments '}on ${
+ post.integration?.providerIdentifier
+ }${handle.message ? `: ${handle.message}` : ``}`,
+ true,
+ false,
+ 'fail'
+ );
+ return false;
+ }
+ }
+ }
+
+ if (postsResults.length === before) {
+ // all retries exhausted without success
+ return false;
+ }
+ }
+
+ // send webhooks for the post
+ await sendWebhooks(
+ postsResults[0].postId,
+ post.organizationId,
+ post.integration.id
+ );
+
+ // load internal plugs like repost by other users
+ const internalPlugsList = await internalPlugs(
+ post.integration,
+ JSON.parse(post.settings)
+ );
+
+ // load global plugs, like repost a post if it gets to a certain number of likes
+ const globalPlugsList = (await globalPlugs(post.integration)).reduce(
+ (all, current) => {
+ for (let i = 1; i <= current.totalRuns; i++) {
+ all.push({
+ ...current,
+ delay: current.delay * i,
+ });
+ }
+
+ return all;
+ },
+ []
+ );
+
+ // Check if the post is repeatable
+ const repeatPost = !post.intervalInDays
+ ? []
+ : [
+ {
+ type: 'repeat-post',
+ delay:
+ post.intervalInDays * 24 * 60 * 60 * 1000 -
+ (new Date().getTime() - startTime.getTime()),
+ },
+ ];
+
+ // Sort all the actions by delay, so we can process them in order
+ const list = sortBy(
+ [...internalPlugsList, ...globalPlugsList, ...repeatPost],
+ 'delay'
+ );
+
+ // process all the plugs in order, we are using while because in some cases we need to remove items from the list
+ while (list.length > 0) {
+ // get the next to process
+ const todo = list.shift();
+
+ // wait for the delay
+ await sleep(Math.max(0, Number(todo.delay ?? 0)));
+
+ // process internal plug
+ if (todo.type === 'internal-plug') {
+ for (const _ of iterate) {
+ try {
+ await processInternalPlug({ ...todo, post: postsResults[0].postId });
+ } catch (err) {
+ const handle = await handleActivityError(err, () =>
+ getIntegrationById(organizationId, todo.integration)
+ );
+
+ if (handle.type === 'stop' || handle.type === 'bad-body') {
+ break;
+ }
+
+ continue;
+ }
+ break;
+ }
+ }
+
+ // process global plug
+ if (todo.type === 'global') {
+ for (const _ of iterate) {
+ try {
+ const process = await processPlug({
+ ...todo,
+ postId: postsResults[0].postId,
+ });
+ if (process) {
+ const toDelete = list
+ .reduce((all, current, index) => {
+ if (current.plugId === todo.plugId) {
+ all.push(index);
+ }
+
+ return all;
+ }, [])
+ .reverse();
+
+ for (const index of toDelete) {
+ list.splice(index, 1);
+ }
+ }
+ } catch (err) {
+ const handle = await handleActivityError(err);
+
+ if (handle.type === 'stop' || handle.type === 'bad-body') {
+ break;
+ }
+
+ continue;
+ }
+
+ break;
+ }
+ }
+
+ // process repeat post in a new workflow, this is important so the other plugs can keep running
+ if (todo.type === 'repeat-post') {
+ await startChild(postWorkflowV106, {
+ parentClosePolicy: 'ABANDON',
+ args: [
+ {
+ taskQueue,
+ postId,
+ organizationId,
+ postNow: true,
+ },
+ ],
+ workflowId: `post_${post.id}_${makeId(10)}`,
+ typedSearchAttributes: new TypedSearchAttributes([
+ {
+ key: postIdSearchParam,
+ value: postId,
+ },
+ ]),
+ });
+ }
+ }
+}
diff --git a/libraries/nestjs-libraries/src/database/prisma/posts/posts.service.ts b/libraries/nestjs-libraries/src/database/prisma/posts/posts.service.ts
index 57f73373e..c5333e0e0 100644
--- a/libraries/nestjs-libraries/src/database/prisma/posts/posts.service.ts
+++ b/libraries/nestjs-libraries/src/database/prisma/posts/posts.service.ts
@@ -726,7 +726,7 @@ export class PostsService {
try {
await this._temporalService.client
.getRawClient()
- ?.workflow.start('postWorkflowV105', {
+ ?.workflow.start('postWorkflowV106', {
workflowId: `post_${postId}`,
taskQueue: 'main',
workflowIdConflictPolicy: 'TERMINATE_EXISTING',
diff --git a/libraries/nestjs-libraries/src/integrations/social.abstract.ts b/libraries/nestjs-libraries/src/integrations/social.abstract.ts
index d0add74ed..42acc7e7e 100644
--- a/libraries/nestjs-libraries/src/integrations/social.abstract.ts
+++ b/libraries/nestjs-libraries/src/integrations/social.abstract.ts
@@ -1,5 +1,6 @@
import { timer } from '@gitroom/helpers/utils/timer';
import { Integration } from '@prisma/client';
+import { PendingCheckResponse } from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
import { ApplicationFailure } from '@temporalio/activity';
import { readOrFetch } from '@gitroom/helpers/utils/read.or.fetch';
import { getSsrfSafeDispatcher } from '@gitroom/nestjs-libraries/dtos/webhooks/ssrf.safe.dispatcher';
@@ -112,6 +113,43 @@ export abstract class SocialAbstract {
return true;
}
+ /**
+ * Providers that return a `pending` PostResponse from `post` / `comment` must
+ * override this with a single, read-only status check (no loops, no timers) -
+ * the polling loop lives in the post workflow, where a retry is harmless.
+ *
+ * The defaults throw so a provider that returns `pending` without overriding
+ * fails loudly on the first test post instead of silently completing with a
+ * bogus releaseURL. They are unreachable for providers that never return
+ * `pending`.
+ */
+ public async checkPostStatus(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'checkPostStatus is not implemented for this provider'
+ );
+ }
+
+ /** Runs the mutations left after `checkPostStatus` returns `ready`. Same contract as `checkPostStatus`. */
+ public async finalizePost(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'finalizePost is not implemented for this provider'
+ );
+ }
+
protected assetBoolean(value: boolean | string) {
if (typeof value === 'string') {
return value.toLowerCase() === 'true';
diff --git a/libraries/nestjs-libraries/src/integrations/social/facebook.provider.ts b/libraries/nestjs-libraries/src/integrations/social/facebook.provider.ts
index 1b2bf9aa2..5ade76092 100644
--- a/libraries/nestjs-libraries/src/integrations/social/facebook.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/facebook.provider.ts
@@ -1,6 +1,7 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -8,6 +9,7 @@ import {
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import dayjs from 'dayjs';
import {
+ BadBody,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
@@ -457,21 +459,49 @@ export class FacebookProvider extends SocialAbstract implements SocialProvider {
throw new Error('Page not found in your accounts');
}
- async post(
+ // Single, read-only status check of a story video - the polling loop that
+ // used to live inside post() is now driven by the post workflow.
+ private async fbVideoStatus(videoId: string, accessToken: string) {
+ const { status } = await (
+ await this.fetch(
+ `https://graph.facebook.com/v20.0/${videoId}?fields=status&access_token=${accessToken}`,
+ undefined,
+ '',
+ 0,
+ true
+ )
+ ).json();
+
+ const videoStatus = status?.video_status || 'in_progress';
+
+ if (videoStatus === 'error') {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify({ status }),
+ '{}',
+ 'Video processing failed'
+ );
+ }
+
+ return videoStatus === 'upload_complete' || videoStatus === 'ready';
+ }
+
+ async postPending(
id: string,
accessToken: string,
- postDetails: PostDetails<FacebookDto>[]
+ postDetails: PostDetails<FacebookDto>[],
+ integration: Integration
): Promise<PostResponse[]> {
const [firstPost] = postDetails;
const isStory = firstPost?.settings?.post_type === 'story';
- let finalId = '';
- let finalUrl = '';
if (isStory) {
- let lastPostId = '';
+ // Only upload the media here - uploads are invisible until the
+ // publish calls, which run one at a time in finalizePost so a failure
+ // can never re-publish the stories that already went out.
+ const items = [];
for (const media of firstPost?.media || []) {
- const isVideoStory = hasExtension(media.path, 'mp4');
- if (isVideoStory) {
+ if (hasExtension(media.path, 'mp4')) {
const { video_id, upload_url } = await (
await this.fetch(
`https://graph.facebook.com/v20.0/${id}/video_stories?upload_phase=start&access_token=${accessToken}`,
@@ -494,43 +524,7 @@ export class FacebookProvider extends SocialAbstract implements SocialProvider {
'upload video story'
);
- let videoStatus = 'in_progress';
- let attempts = 0;
- const maxAttempts = 54; // ~9 minutes at 10s interval
- while (videoStatus !== 'upload_complete' && videoStatus !== 'ready') {
- if (attempts++ >= maxAttempts) {
- throw new Error('Video processing timed out');
- }
-
- const { status } = await (
- await this.fetch(
- `https://graph.facebook.com/v20.0/${video_id}?fields=status&access_token=${accessToken}`,
- undefined,
- '',
- 0,
- true
- )
- ).json();
- videoStatus = status?.video_status || 'in_progress';
- if (videoStatus === 'error') {
- throw new Error('Video processing failed');
- }
- if (videoStatus !== 'upload_complete' && videoStatus !== 'ready') {
- await timer(10000);
- }
- }
-
- const { post_id: storyPostId } = await (
- await this.fetch(
- `https://graph.facebook.com/v20.0/${id}/video_stories?upload_phase=finish&video_id=${video_id}&access_token=${accessToken}`,
- {
- method: 'POST',
- },
- 'finish video story upload'
- )
- ).json();
-
- lastPostId = storyPostId;
+ items.push({ kind: 'video', mediaId: video_id });
} else {
const { id: photoId } = await (
await this.fetch(
@@ -549,23 +543,221 @@ export class FacebookProvider extends SocialAbstract implements SocialProvider {
)
).json();
- const { post_id: storyPostId } = await (
- await this.fetch(
- `https://graph.facebook.com/v20.0/${id}/photo_stories?photo_id=${photoId}&access_token=${accessToken}`,
- {
- method: 'POST',
- },
- 'publish photo story'
- )
- ).json();
-
- lastPostId = storyPostId;
+ items.push({ kind: 'photo', mediaId: photoId });
}
}
- finalId = lastPostId;
- finalUrl = `https://www.facebook.com/stories/${lastPostId}`;
- } else if (hasExtension(firstPost?.media?.[0]?.path, 'mp4')) {
+ return [
+ {
+ id: firstPost.id,
+ postId: '',
+ releaseURL: '',
+ status: 'pending',
+ pendingData: {
+ postType: 'story',
+ items,
+ publishedCount: 0,
+ lastPostId: '',
+ },
+ },
+ ];
+ }
+
+ return this.postNonStory(id, accessToken, postDetails);
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: {
+ postType: 'story';
+ items: { kind: 'video' | 'photo'; mediaId: string }[];
+ publishedCount: number;
+ lastPostId: string;
+ attempting?: number | null;
+ confirmed?: boolean;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // A confirmed publish attempt died without reporting its result: Facebook
+ // has no API to ask whether a story was published, so never publish that
+ // item again - stop with an explicit warning instead.
+ if (pendingData.attempting != null && pendingData.confirmed) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Facebook may have already published part of the story, please check your page before posting again to avoid duplicates'
+ );
+ }
+
+ // wait for every not-yet-published video to finish processing, photos are
+ // ready as soon as they are uploaded
+ for (const item of pendingData.items.slice(pendingData.publishedCount)) {
+ if (item.kind !== 'video') {
+ continue;
+ }
+
+ if (!(await this.fbVideoStatus(item.mediaId, accessToken))) {
+ return { status: 'pending', pendingData };
+ }
+ }
+
+ // witness the armed publish so finalizePost knows the attempt is uniquely
+ // accounted for before it mutates anything
+ if (pendingData.attempting != null && !pendingData.confirmed) {
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, confirmed: true },
+ };
+ }
+
+ return { status: 'ready', pendingData };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: {
+ postType: 'story';
+ items: { kind: 'video' | 'photo'; mediaId: string }[];
+ publishedCount: number;
+ lastPostId: string;
+ attempting?: number | null;
+ confirmed?: boolean;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // Publish exactly one story per call, with an arm -> confirm -> publish
+ // handshake: the publish only runs after checkPostStatus witnessed the
+ // intent, so a run that dies mid-publish is detectable and the item is
+ // never published twice.
+ if (pendingData.attempting == null || !pendingData.confirmed) {
+ return {
+ status: 'pending',
+ pendingData: {
+ ...pendingData,
+ attempting: pendingData.publishedCount,
+ confirmed: false,
+ },
+ };
+ }
+
+ const item = pendingData.items[pendingData.publishedCount];
+
+ const { post_id: storyPostId } = await (
+ await this.fetch(
+ item.kind === 'video'
+ ? `https://graph.facebook.com/v20.0/${integration.internalId}/video_stories?upload_phase=finish&video_id=${item.mediaId}&access_token=${accessToken}`
+ : `https://graph.facebook.com/v20.0/${integration.internalId}/photo_stories?photo_id=${item.mediaId}&access_token=${accessToken}`,
+ {
+ method: 'POST',
+ },
+ item.kind === 'video'
+ ? 'finish video story upload'
+ : 'publish photo story'
+ )
+ ).json();
+
+ const publishedCount = pendingData.publishedCount + 1;
+
+ if (publishedCount < pendingData.items.length) {
+ return {
+ status: 'pending',
+ pendingData: {
+ ...pendingData,
+ publishedCount,
+ lastPostId: storyPostId,
+ attempting: null,
+ confirmed: false,
+ },
+ };
+ }
+
+ return {
+ status: 'completed',
+ postId: storyPostId,
+ releaseURL: `https://www.facebook.com/stories/${storyPostId}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that don't
+ // know how to resolve a `pending` response.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<FacebookDto>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [firstPost] = postDetails;
+ const [response] = await this.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ if (response.status !== 'pending') {
+ return [response];
+ }
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here (non-retryable) is safe, timing the
+ // activity out is not - a retried activity would publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Video processing timed out'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(10000);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [
+ {
+ id: firstPost.id,
+ postId: result.postId,
+ releaseURL: result.releaseURL,
+ status: 'success',
+ },
+ ];
+ }
+
+ pendingData = result.pendingData;
+ }
+ }
+
+ private async postNonStory(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<FacebookDto>[]
+ ): Promise<PostResponse[]> {
+ const [firstPost] = postDetails;
+
+ let finalId = '';
+ let finalUrl = '';
+ if (hasExtension(firstPost?.media?.[0]?.path, 'mp4')) {
const {
id: videoId,
permalink_url,
diff --git a/libraries/nestjs-libraries/src/integrations/social/instagram.provider.ts b/libraries/nestjs-libraries/src/integrations/social/instagram.provider.ts
index d372bd907..caf40db79 100644
--- a/libraries/nestjs-libraries/src/integrations/social/instagram.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/instagram.provider.ts
@@ -1,6 +1,7 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -9,6 +10,7 @@ import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import { timer } from '@gitroom/helpers/utils/timer';
import dayjs from 'dayjs';
import {
+ BadBody,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
@@ -594,14 +596,63 @@ export class InstagramProvider
};
}
- async post(
+ // Single, read-only status check of a media container - the polling loops
+ // that used to live inside post() are now driven by the post workflow.
+ private async igContainerStatus(
+ containerId: string,
+ checkToken: string,
+ type: string
+ ): Promise<string> {
+ const { status_code, status } = await (
+ await this.fetch(
+ `https://${type}/v20.0/${containerId}?access_token=${checkToken}&fields=status_code,status`,
+ undefined,
+ '',
+ 0,
+ true
+ )
+ ).json();
+
+ if (status_code === 'ERROR' || status_code === 'EXPIRED') {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify({ status_code, status }),
+ '{}',
+ status || 'Instagram could not process the media'
+ );
+ }
+
+ return status_code;
+ }
+
+ // The post is live, the permalink is only cosmetic: never fail (and risk
+ // re-publishing) a live post over it.
+ private async igPermalink(
+ mediaId: string,
+ checkToken: string,
+ type: string,
+ integration: Integration
+ ): Promise<string> {
+ try {
+ const { permalink } = await (
+ await this.fetch(
+ `https://${type}/v20.0/${mediaId}?fields=permalink&access_token=${checkToken}`
+ )
+ ).json();
+ return permalink;
+ } catch (err) {
+ return `https://www.instagram.com/${integration.profile}`;
+ }
+ }
+
+ async postPending(
id: string,
token: string,
postDetails: PostDetails<InstagramDto>[],
integration: Integration,
type = 'graph.facebook.com'
): Promise<PostResponse[]> {
- const [accessToken, userToken] = token.split('___');
+ const [accessToken] = token.split('___');
const [firstPost] = postDetails;
console.log('in progress', id);
const isStory = firstPost.settings.post_type === 'story';
@@ -679,102 +730,165 @@ export class InstagramProvider
}
)
).json();
- console.log('in progress2', id);
-
- let status = 'IN_PROGRESS';
- let attempts = 0;
- const maxAttempts = 18; // ~9 minutes at 30s interval
- while (status === 'IN_PROGRESS') {
- if (attempts++ >= maxAttempts) {
- throw new Error('Media processing timed out');
- }
-
- const { status_code } = await (
- await this.fetch(
- `https://${type}/v20.0/${photoId}?access_token=${
- userToken || accessToken
- }&fields=status_code`,
- undefined,
- '',
- 0,
- true
- )
- ).json();
- await timer(30000);
- status = status_code;
- }
- console.log('in progress3', id);
return photoId;
}) || []
);
- if (isStory && medias.length > 1) {
- // Stories don't support carousels - publish each media as a separate story
+ // Containers are invisible until media_publish runs: the processing wait
+ // and the publish itself move to checkPostStatus / finalizePost so a
+ // failure there can never re-create (and re-publish) the whole post.
+ return [
+ {
+ id: firstPost.id,
+ postId: '',
+ releaseURL: '',
+ status: 'pending',
+ pendingData: {
+ type,
+ postType:
+ isStory && medias.length > 1
+ ? 'stories'
+ : medias.length === 1
+ ? 'single'
+ : 'carousel',
+ containers: medias,
+ message: firstPost?.message || '',
+ },
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ token: string,
+ pendingData: {
+ type: string;
+ postType: 'stories' | 'single' | 'carousel';
+ containers: string[];
+ message?: string;
+ carouselId?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ const [accessToken, userToken] = token.split('___');
+ const checkToken = userToken || accessToken;
+
+ // the carousel container was already created: wait for it
+ if (pendingData.carouselId) {
+ const status = await this.igContainerStatus(
+ pendingData.carouselId,
+ checkToken,
+ pendingData.type
+ );
+
+ if (status === 'IN_PROGRESS') {
+ return { status: 'pending', pendingData };
+ }
+
+ // a previous finalizePost published but died before reporting: the post
+ // is live, never publish again
+ if (status === 'PUBLISHED') {
+ return {
+ status: 'completed',
+ postId: pendingData.carouselId,
+ releaseURL: `https://www.instagram.com/${integration.profile}`,
+ };
+ }
+
+ return { status: 'ready', pendingData };
+ }
+
+ for (const containerId of pendingData.containers) {
+ const status = await this.igContainerStatus(
+ containerId,
+ checkToken,
+ pendingData.type
+ );
+
+ if (status === 'IN_PROGRESS') {
+ return { status: 'pending', pendingData };
+ }
+
+ if (status === 'PUBLISHED') {
+ // a previous finalizePost died mid-way: a single post is fully live,
+ // stories are resumed by finalizePost (it skips published containers)
+ if (pendingData.postType === 'single') {
+ return {
+ status: 'completed',
+ postId: containerId,
+ releaseURL: `https://www.instagram.com/${integration.profile}`,
+ };
+ }
+ }
+ }
+
+ return { status: 'ready', pendingData };
+ }
+
+ override async finalizePost(
+ token: string,
+ pendingData: {
+ type: string;
+ postType: 'stories' | 'single' | 'carousel';
+ containers: string[];
+ message?: string;
+ carouselId?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ const [accessToken, userToken] = token.split('___');
+ const checkToken = userToken || accessToken;
+ const igId = integration.internalId;
+
+ if (pendingData.postType === 'stories') {
+ // Stories don't support carousels - publish each media as a separate
+ // story, skipping containers a previous (crashed) run already published
let lastMediaId = '';
- let lastPermalink = '';
- for (const mediaCreationId of medias) {
+ for (const mediaCreationId of pendingData.containers) {
+ const status = await this.igContainerStatus(
+ mediaCreationId,
+ checkToken,
+ pendingData.type
+ );
+ if (status === 'PUBLISHED') {
+ continue;
+ }
+
const { id: mediaId } = await (
await this.fetch(
- `https://${type}/v20.0/${id}/media_publish?creation_id=${mediaCreationId}&access_token=${accessToken}&field=id`,
+ `https://${pendingData.type}/v20.0/${igId}/media_publish?creation_id=${mediaCreationId}&access_token=${accessToken}&field=id`,
{
method: 'POST',
}
)
).json();
lastMediaId = mediaId;
-
- const { permalink } = await (
- await this.fetch(
- `https://${type}/v20.0/${mediaId}?fields=permalink&access_token=${
- userToken || accessToken
- }`
- )
- ).json();
- lastPermalink = permalink;
}
- return [
- {
- id: firstPost.id,
- postId: lastMediaId,
- releaseURL: lastPermalink,
- status: 'success',
- },
- ];
- } else if (medias.length === 1) {
- const { id: mediaId } = await (
- await this.fetch(
- `https://${type}/v20.0/${id}/media_publish?creation_id=${medias[0]}&access_token=${accessToken}&field=id`,
- {
- method: 'POST',
- }
- )
- ).json();
-
- const { permalink } = await (
- await this.fetch(
- `https://${type}/v20.0/${mediaId}?fields=permalink&access_token=${
- userToken || accessToken
- }`
- )
- ).json();
+ return {
+ status: 'completed',
+ postId: lastMediaId || pendingData.containers.at(-1)!,
+ releaseURL: !lastMediaId
+ ? `https://www.instagram.com/${integration.profile}`
+ : await this.igPermalink(
+ lastMediaId,
+ checkToken,
+ pendingData.type,
+ integration
+ ),
+ };
+ }
- return [
- {
- id: firstPost.id,
- postId: mediaId,
- releaseURL: permalink,
- status: 'success',
- },
- ];
- } else {
- const { id: containerId, ...all3 } = await (
+ if (pendingData.postType === 'carousel' && !pendingData.carouselId) {
+ // create the carousel container and hand back to the workflow to wait
+ // for it (an orphan container from a crashed run is invisible, so
+ // re-running this is safe)
+ const { id: containerId } = await (
await this.fetch(
- `https://${type}/v20.0/${id}/media?caption=${encodeURIComponent(
- firstPost?.message
+ `https://${pendingData.type}/v20.0/${igId}/media?caption=${encodeURIComponent(
+ pendingData.message || ''
)}&media_type=CAROUSEL&children=${encodeURIComponent(
- medias.join(',')
+ pendingData.containers.join(',')
)}&access_token=${accessToken}`,
{
method: 'POST',
@@ -782,54 +896,99 @@ export class InstagramProvider
)
).json();
- let status = 'IN_PROGRESS';
- let attempts = 0;
- const maxAttempts = 18; // ~9 minutes at 30s interval
- while (status === 'IN_PROGRESS') {
- if (attempts++ >= maxAttempts) {
- throw new Error('Media processing timed out');
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, carouselId: containerId },
+ };
+ }
+
+ const creationId =
+ pendingData.postType === 'carousel'
+ ? pendingData.carouselId
+ : pendingData.containers[0];
+
+ const { id: mediaId } = await (
+ await this.fetch(
+ `https://${pendingData.type}/v20.0/${igId}/media_publish?creation_id=${creationId}&access_token=${accessToken}&field=id`,
+ {
+ method: 'POST',
}
+ )
+ ).json();
- const { status_code } = await (
- await this.fetch(
- `https://${type}/v20.0/${containerId}?fields=status_code&access_token=${
- userToken || accessToken
- }`,
- undefined,
- '',
- 0,
- true
- )
- ).json();
+ return {
+ status: 'completed',
+ postId: mediaId,
+ releaseURL: await this.igPermalink(
+ mediaId,
+ checkToken,
+ pendingData.type,
+ integration
+ ),
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that don't
+ // know how to resolve a `pending` response.
+ async post(
+ id: string,
+ token: string,
+ postDetails: PostDetails<InstagramDto>[],
+ integration: Integration,
+ type = 'graph.facebook.com'
+ ): Promise<PostResponse[]> {
+ const [firstPost] = postDetails;
+ const [response] = await this.postPending(
+ id,
+ token,
+ postDetails,
+ integration,
+ type
+ );
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here (non-retryable) is safe, timing the
+ // activity out is not - a retried activity would publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Media processing timed out'
+ );
+ }
+
+ const check = await this.checkPostStatus(token, pendingData, integration);
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
await timer(30000);
- status = status_code;
+ continue;
}
- const { id: mediaId, ...all4 } = await (
- await this.fetch(
- `https://${type}/v20.0/${id}/media_publish?creation_id=${containerId}&access_token=${accessToken}&field=id`,
- {
- method: 'POST',
- }
- )
- ).json();
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(token, check.pendingData, integration)
+ : check;
- const { permalink } = await (
- await this.fetch(
- `https://${type}/v20.0/${mediaId}?fields=permalink&access_token=${
- userToken || accessToken
- }`
- )
- ).json();
+ if (result.status === 'completed') {
+ return [
+ {
+ id: firstPost.id,
+ postId: result.postId,
+ releaseURL: result.releaseURL,
+ status: 'success',
+ },
+ ];
+ }
- return [
- {
- id: firstPost.id,
- postId: mediaId,
- releaseURL: permalink,
- status: 'success',
- },
- ];
+ pendingData = result.pendingData;
+ await timer(30000);
}
}
diff --git a/libraries/nestjs-libraries/src/integrations/social/instagram.standalone.provider.ts b/libraries/nestjs-libraries/src/integrations/social/instagram.standalone.provider.ts
index 039690cea..d84c9199a 100644
--- a/libraries/nestjs-libraries/src/integrations/social/instagram.standalone.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/instagram.standalone.provider.ts
@@ -191,6 +191,42 @@ export class InstagramStandaloneProvider
);
}
+ async postPending(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<InstagramDto>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ return instagramProvider.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration,
+ 'graph.instagram.com'
+ );
+ }
+
+ // the graph domain travels inside pendingData, so these are pure delegations
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ) {
+ return instagramProvider.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ) {
+ return instagramProvider.finalizePost(accessToken, pendingData, integration);
+ }
+
async comment(
id: string,
postId: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/social.integrations.interface.ts b/libraries/nestjs-libraries/src/integrations/social/social.integrations.interface.ts
index 1ad3cd7fa..5a4d0b40b 100644
--- a/libraries/nestjs-libraries/src/integrations/social/social.integrations.interface.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/social.integrations.interface.ts
@@ -89,6 +89,13 @@ export interface ISocialMediaIntegration {
integration: Integration
): Promise<PostResponse[]>; // Schedules a new post
+ postPending?(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]>; // Like `post`, but may return a `pending` response the workflow resolves via checkPostStatus / finalizePost
+
comment?(
id: string,
postId: string,
@@ -103,9 +110,26 @@ export type PostResponse = {
id: string; // The db internal id of the post
postId: string; // The ID of the scheduled post returned by the platform
releaseURL: string; // The URL of the post on the platform
- status: string; // Status of the operation or initial post status
+ status: string; // Status of the operation or initial post status, 'pending' means the workflow must poll checkPostStatus
+ pendingData?: any; // Opaque provider state used by checkPostStatus / finalizePost, never inspected by generic code
};
+// Returned by checkPostStatus / finalizePost:
+// 'pending' - the platform is still processing, poll again later
+// 'ready' - processing is done, the workflow must call finalizePost to run the remaining mutations
+// 'completed' - the post is fully published
+//
+// Contract: once finalizePost's mutations have actually gone through on the
+// platform, checkPostStatus must return 'completed' - never 'ready' again -
+// otherwise a finalizePost retry after an unknown-outcome failure would re-run
+// the mutations and duplicate the post. The only exception: when finalizePost's
+// mutation is idempotent (like setting a thumbnail), returning 'ready' again is
+// allowed, since re-running it cannot duplicate anything.
+export type PendingCheckResponse =
+ | { status: 'pending'; pendingData: any }
+ | { status: 'ready'; pendingData: any }
+ | { status: 'completed'; postId: string; releaseURL: string };
+
export type PostDetails<T = any> = {
id: string;
message: string;
@@ -150,6 +174,16 @@ export interface SocialProvider
settings: any,
additionalSettings: any[]
): Promise<string | true>;
+ checkPostStatus(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ): Promise<PendingCheckResponse>;
+ finalizePost(
+ accessToken: string,
+ pendingData: any,
+ integration: Integration
+ ): Promise<PendingCheckResponse>;
isWeb3?: boolean;
isChromeExtension?: boolean;
extensionCookies?: { name: string; domain: string }[];
diff --git a/libraries/nestjs-libraries/src/integrations/social/threads.provider.ts b/libraries/nestjs-libraries/src/integrations/social/threads.provider.ts
index 30092f83f..08f898897 100644
--- a/libraries/nestjs-libraries/src/integrations/social/threads.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/threads.provider.ts
@@ -1,6 +1,7 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -8,7 +9,10 @@ import {
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import { timer } from '@gitroom/helpers/utils/timer';
import dayjs from 'dayjs';
-import { SocialAbstract } from '@gitroom/nestjs-libraries/integrations/social.abstract';
+import {
+ BadBody,
+ SocialAbstract,
+} from '@gitroom/nestjs-libraries/integrations/social.abstract';
import { capitalize, chunk } from 'lodash';
import { Plug } from '@gitroom/helpers/decorators/plug.decorator';
import { Integration } from '@prisma/client';
@@ -165,27 +169,60 @@ export class ThreadsProvider extends SocialAbstract implements SocialProvider {
};
}
- private async checkLoaded(
+ // Single, read-only status check of a media container - no loops and no
+ // timers, so the post workflow can poll it with durable timers.
+ private async checkContainerStatus(
mediaContainerId: string,
accessToken: string
- ): Promise<boolean> {
- const { status, id, error_message } = await (
+ ): Promise<'FINISHED' | 'IN_PROGRESS' | 'PUBLISHED'> {
+ const { status, error_message } = await (
await this.fetch(
`https://graph.threads.net/v1.0/${mediaContainerId}?fields=status,error_message&access_token=${accessToken}`
)
).json();
- if (status === 'ERROR') {
- throw new Error(id);
+ if (status === 'ERROR' || status === 'EXPIRED') {
+ throw new BadBody(
+ this.identifier,
+ JSON.stringify({ status, error_message }),
+ '{}',
+ error_message || 'Threads could not process the media'
+ );
+ }
+
+ if (status === 'FINISHED' || status === 'PUBLISHED') {
+ return status;
}
- if (status === 'FINISHED') {
- await timer(2000);
- return true;
+ return 'IN_PROGRESS';
+ }
+
+ private async checkLoaded(
+ mediaContainerId: string,
+ accessToken: string
+ ): Promise<boolean> {
+ // Bounded (the old version recursed forever and could hang the activity
+ // into a timeout): ~5.5 minutes at 2.2s intervals.
+ for (let i = 0; i < 150; i++) {
+ const status = await this.checkContainerStatus(
+ mediaContainerId,
+ accessToken
+ );
+
+ if (status === 'FINISHED' || status === 'PUBLISHED') {
+ await timer(2000);
+ return true;
+ }
+
+ await timer(2200);
}
- await timer(2200);
- return this.checkLoaded(mediaContainerId, accessToken);
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Threads took too long to process the media, please try again'
+ );
}
private async fetchUserInfo(accessToken: string) {
@@ -376,13 +413,33 @@ export class ThreadsProvider extends SocialAbstract implements SocialProvider {
}
}
- async post(
+ // The thread is live, the permalink is only cosmetic: never fail (and risk
+ // re-publishing) a live post over it.
+ private async threadPermalink(
+ threadId: string,
+ accessToken: string,
+ integration: Integration
+ ): Promise<string> {
+ try {
+ const { permalink } = await (
+ await this.fetch(
+ `https://graph.threads.net/v1.0/${threadId}?fields=id,permalink&access_token=${accessToken}`
+ )
+ ).json();
+ return permalink;
+ } catch (err) {
+ return `https://www.threads.net/@${integration.profile}`;
+ }
+ }
+
+ async postPending(
userId: string,
accessToken: string,
postDetails: PostDetails<{
active_thread_finisher: boolean;
thread_finisher: string;
- }>[]
+ }>[],
+ integration: Integration
): Promise<PostResponse[]> {
if (!postDetails.length) {
return [];
@@ -390,31 +447,231 @@ export class ThreadsProvider extends SocialAbstract implements SocialProvider {
const [firstPost] = postDetails;
- // Create the initial thread
- const initialContentId = await this.createThreadContent(
- userId,
- accessToken,
- firstPost
- );
+ // Carousels: only create the child containers here, the carousel container
+ // itself is created by finalizePost once the children are processed.
+ if ((firstPost.media?.length || 0) > 1) {
+ const childIds = [];
+ for (const mediaItem of firstPost.media!) {
+ childIds.push(
+ await this.createSingleMediaContent(
+ userId,
+ accessToken,
+ mediaItem,
+ firstPost.message,
+ true
+ )
+ );
+ }
- // Publish the thread
- const { threadId, permalink } = await this.publishThread(
- userId,
- accessToken,
- initialContentId
- );
+ return [
+ {
+ id: firstPost.id,
+ postId: '',
+ releaseURL: '',
+ status: 'pending',
+ pendingData: {
+ step: 'children',
+ childIds,
+ message: firstPost.message,
+ },
+ },
+ ];
+ }
+
+ // Text / single media: one container, nothing is visible until
+ // threads_publish runs in finalizePost.
+ const containerId =
+ !firstPost.media || firstPost.media.length === 0
+ ? await this.createTextContent(userId, accessToken, firstPost.message)
+ : await this.createSingleMediaContent(
+ userId,
+ accessToken,
+ firstPost.media[0],
+ firstPost.message,
+ false
+ );
- // Return the main post response
return [
{
id: firstPost.id,
- postId: threadId,
- status: 'success',
- releaseURL: permalink,
+ postId: '',
+ releaseURL: '',
+ status: 'pending',
+ pendingData: { step: 'container', containerId },
},
];
}
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: {
+ step: 'children' | 'container';
+ childIds?: string[];
+ containerId?: string;
+ message?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // waiting for the carousel children to be processed
+ if (pendingData.step === 'children') {
+ for (const childId of pendingData.childIds || []) {
+ const status = await this.checkContainerStatus(childId, accessToken);
+ if (status === 'IN_PROGRESS') {
+ return { status: 'pending', pendingData };
+ }
+ }
+
+ // all children processed, finalizePost creates the carousel container
+ return { status: 'ready', pendingData };
+ }
+
+ const status = await this.checkContainerStatus(
+ pendingData.containerId!,
+ accessToken
+ );
+
+ if (status === 'IN_PROGRESS') {
+ return { status: 'pending', pendingData };
+ }
+
+ // a previous finalizePost published but died before reporting: the post is
+ // live, never publish again
+ if (status === 'PUBLISHED') {
+ return {
+ status: 'completed',
+ postId: pendingData.containerId!,
+ releaseURL: `https://www.threads.net/@${integration.profile}`,
+ };
+ }
+
+ return { status: 'ready', pendingData };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: {
+ step: 'children' | 'container';
+ childIds?: string[];
+ containerId?: string;
+ message?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ // the carousel children are processed: create the carousel container and
+ // hand back to the workflow to wait for it (an orphan container from a
+ // crashed run is invisible, so re-running this is safe)
+ if (pendingData.step === 'children') {
+ const params = new URLSearchParams({
+ text: pendingData.message || '',
+ media_type: 'CAROUSEL',
+ children: (pendingData.childIds || []).join(','),
+ access_token: accessToken,
+ });
+
+ const { id: containerId } = await (
+ await this.fetch(
+ `https://graph.threads.net/v1.0/${integration.internalId}/threads?${params.toString()}`,
+ {
+ method: 'POST',
+ }
+ )
+ ).json();
+
+ return {
+ status: 'pending',
+ pendingData: { step: 'container', containerId },
+ };
+ }
+
+ const { id: threadId } = await (
+ await this.fetch(
+ `https://graph.threads.net/v1.0/${integration.internalId}/threads_publish?creation_id=${pendingData.containerId}&access_token=${accessToken}`,
+ {
+ method: 'POST',
+ }
+ )
+ ).json();
+
+ return {
+ status: 'completed',
+ postId: threadId,
+ releaseURL: await this.threadPermalink(threadId, accessToken, integration),
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that don't
+ // know how to resolve a `pending` response.
+ async post(
+ userId: string,
+ accessToken: string,
+ postDetails: PostDetails<{
+ active_thread_finisher: boolean;
+ thread_finisher: string;
+ }>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ if (!postDetails.length) {
+ return [];
+ }
+
+ const [firstPost] = postDetails;
+ const [response] = await this.postPending(
+ userId,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here (non-retryable) is safe, timing the
+ // activity out is not - a retried activity would publish again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Threads took too long to process the media, please try again'
+ );
+ }
+
+ const check = await this.checkPostStatus(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (check.status === 'pending') {
+ pendingData = check.pendingData;
+ await timer(2200);
+ continue;
+ }
+
+ const result =
+ check.status === 'ready'
+ ? await this.finalizePost(accessToken, check.pendingData, integration)
+ : check;
+
+ if (result.status === 'completed') {
+ return [
+ {
+ id: firstPost.id,
+ postId: result.postId,
+ status: 'success',
+ releaseURL: result.releaseURL,
+ },
+ ];
+ }
+
+ pendingData = result.pendingData;
+ await timer(2200);
+ }
+ }
+
async comment(
userId: string,
postId: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/tiktok.provider.ts b/libraries/nestjs-libraries/src/integrations/social/tiktok.provider.ts
index fff84f7ca..1f70bcec1 100644
--- a/libraries/nestjs-libraries/src/integrations/social/tiktok.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/tiktok.provider.ts
@@ -1,6 +1,7 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
@@ -8,6 +9,7 @@ import {
import dayjs from 'dayjs';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
@@ -412,15 +414,18 @@ export class TiktokProvider extends SocialAbstract implements SocialProvider {
};
}
- private async uploadedVideoSuccess(
- id: string,
- publishId: string,
- accessToken: string
- ): Promise<{ url: string; id: string }> {
- // eslint-disable-next-line no-constant-condition
- for (const i of Array(27).keys()) {
- // ~9 minutes at 20s interval
- const post = await (
+ // Single status check for a publish_id, no loops and no timers: `post` returns
+ // a `pending` PostResponse right after the upload, and the post workflow polls
+ // this method with durable timers, so a stuck/retried check can never re-run
+ // the publish and duplicate the post.
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: { publishId: string },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ let post: any;
+ try {
+ post = await (
await this.fetch(
'https://open.tiktokapis.com/v2/post/publish/status/fetch/',
{
@@ -430,7 +435,7 @@ export class TiktokProvider extends SocialAbstract implements SocialProvider {
Authorization: `Bearer ${accessToken}`,
},
body: JSON.stringify({
- publish_id: publishId,
+ publish_id: pendingData.publishId,
}),
},
'',
@@ -438,47 +443,52 @@ export class TiktokProvider extends SocialAbstract implements SocialProvider {
true
)
).json();
+ } catch (err) {
+ if (err instanceof RefreshToken) {
+ throw err;
+ }
- const { status, publicaly_available_post_id } = post.data;
+ // Transient API error while checking the status: the post may already
+ // be live, so keep polling instead of failing it - if the API stays
+ // broken the caller exhausts its checks and warns the user properly.
+ return { status: 'pending', pendingData };
+ }
- if (status === 'SEND_TO_USER_INBOX') {
- return {
- url: 'https://www.tiktok.com/messages?lang=en',
- id: 'missing',
- };
- }
+ const { status, publicaly_available_post_id } = post?.data || {};
- if (status === 'PUBLISH_COMPLETE') {
- return {
- url: !publicaly_available_post_id
- ? `https://www.tiktok.com/@${id}`
- : `https://www.tiktok.com/@${id}/video/` +
- publicaly_available_post_id,
- id: !publicaly_available_post_id
- ? publishId
- : publicaly_available_post_id?.[0],
- };
- }
+ if (status === 'SEND_TO_USER_INBOX') {
+ return {
+ status: 'completed',
+ releaseURL: 'https://www.tiktok.com/messages?lang=en',
+ postId: 'missing',
+ };
+ }
- if (status === 'FAILED') {
- const handleError = this.handleErrors(JSON.stringify(post));
- throw new BadBody(
- 'titok-error-upload',
- JSON.stringify(post),
- Buffer.from(JSON.stringify(post)),
- handleError?.value || ''
- );
- }
+ if (status === 'PUBLISH_COMPLETE') {
+ return {
+ status: 'completed',
+ releaseURL: !publicaly_available_post_id
+ ? `https://www.tiktok.com/@${integration.profile}`
+ : `https://www.tiktok.com/@${integration.profile}/video/` +
+ publicaly_available_post_id,
+ // TikTok returns the id as a number, releaseId in the db is a string
+ postId: !publicaly_available_post_id
+ ? pendingData.publishId
+ : String(publicaly_available_post_id?.[0]),
+ };
+ }
- await timer(20000);
+ if (status === 'FAILED') {
+ const handleError = this.handleErrors(JSON.stringify(post));
+ throw new BadBody(
+ 'titok-error-upload',
+ JSON.stringify(post),
+ Buffer.from(JSON.stringify(post)),
+ handleError?.value || ''
+ );
}
- throw new BadBody(
- 'titok-error-upload',
- JSON.stringify({}),
- Buffer.from(JSON.stringify({})),
- 'TikTok refused to publish your post'
- );
+ return { status: 'pending', pendingData };
}
// UPLOAD does not publish - it only drops the media into the user's TikTok
@@ -789,7 +799,7 @@ export class TiktokProvider extends SocialAbstract implements SocialProvider {
};
}
- async post(
+ async postPending(
id: string,
accessToken: string,
postDetails: PostDetails<TikTokDto>[],
@@ -830,30 +840,96 @@ export class TiktokProvider extends SocialAbstract implements SocialProvider {
// Videos: stream the bytes to the upload_url returned by the init call.
if (!isPhoto && upload_url && videoSize) {
- await this.uploadTikTokVideoBytes(
- upload_url,
- videoPath,
- videoSize,
- 'video/mp4'
- );
- }
+ try {
+ await this.uploadTikTokVideoBytes(
+ upload_url,
+ videoPath,
+ videoSize,
+ 'video/mp4'
+ );
+ } catch (err) {
+ // An explicit rejection from TikTok: the post won't publish, fail it
+ if (err instanceof BadBody) {
+ throw err;
+ }
- const { url, id: videoId } = await this.uploadedVideoSuccess(
- integration.profile!,
- publish_id,
- accessToken
- );
+ // A network error here is ambiguous - TikTok may have received all
+ // the bytes and still publish. Return pending and let checkPostStatus
+ // deliver TikTok's own verdict instead of letting the caller retry
+ // the publish.
+ }
+ }
+ // The publish is now irreversible on TikTok's side: return `pending` so the
+ // workflow polls checkPostStatus instead of blocking (and possibly timing
+ // out and re-posting) inside this activity.
return [
{
id: firstPost.id,
- releaseURL: url,
- postId: String(videoId),
- status: 'success',
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: { publishId: publish_id },
},
];
}
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run (scheduled posts sleep inside them until publish time) and don't know
+ // how to resolve a `pending` response - they keep polling inside the
+ // activity exactly like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails<TikTokDto>[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ const started = Date.now();
+
+ for (const i of Array(27).keys()) {
+ // ~9 minutes at 20s interval
+ const check = await this.checkPostStatus(
+ accessToken,
+ response.pendingData,
+ integration
+ );
+
+ if (check.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ releaseURL: check.releaseURL,
+ postId: String(check.postId),
+ status: 'success',
+ },
+ ];
+ }
+
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here (non-retryable) is safe, timing the activity
+ // out is not - a retried activity would publish the post again.
+ if (Date.now() - started > 8 * 60 * 1000) {
+ break;
+ }
+
+ await timer(20000);
+ }
+
+ throw new BadBody(
+ 'titok-error-upload',
+ JSON.stringify({}),
+ Buffer.from(JSON.stringify({})),
+ 'TikTok refused to publish your post'
+ );
+ }
+
async analytics(
id: string,
accessToken: string,
diff --git a/libraries/nestjs-libraries/src/integrations/social/youtube.provider.ts b/libraries/nestjs-libraries/src/integrations/social/youtube.provider.ts
index d73fb2283..eeaf17b87 100644
--- a/libraries/nestjs-libraries/src/integrations/social/youtube.provider.ts
+++ b/libraries/nestjs-libraries/src/integrations/social/youtube.provider.ts
@@ -1,24 +1,26 @@
import {
AnalyticsData,
AuthTokenDetails,
+ PendingCheckResponse,
PostDetails,
PostResponse,
SocialProvider,
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
+import { Integration } from '@prisma/client';
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
-import { google, youtube_v3 } from 'googleapis';
+import { google } from 'googleapis';
import { OAuth2Client } from 'google-auth-library/build/src/auth/oauth2client';
import axios from 'axios';
import { YoutubeSettingsDto } from '@gitroom/nestjs-libraries/dtos/posts/providers-settings/youtube.settings.dto';
import {
BadBody,
+ RefreshToken,
SocialAbstract,
ValidityMedia,
} from '@gitroom/nestjs-libraries/integrations/social.abstract';
import * as process from 'node:process';
import dayjs from 'dayjs';
-import { GaxiosResponse } from 'gaxios/build/src/common';
-import Schema$Video = youtube_v3.Schema$Video;
+import { createReadStream, statSync } from 'fs';
import { Rules } from '@gitroom/nestjs-libraries/chat/rules.description.decorator';
const clientAndYoutube = () => {
@@ -411,59 +413,361 @@ export class YoutubeProvider extends SocialAbstract implements SocialProvider {
};
}
- async post(
+ // ---------------------------------------------------------------------------
+ // RESUMABLE UPLOAD (no more videos.insert): YouTube only creates the video
+ // resource when the final byte of the session is received, so a failed or
+ // retried upload can never leave a video behind - and the session URI can
+ // always be asked whether the video was created (the "unknown outcome" cure).
+ // ---------------------------------------------------------------------------
+ // YouTube requires chunk sizes to be a multiple of 256KB (except the last).
+ private static readonly YOUTUBE_CHUNK_SIZE = 8 * 1024 * 1024;
+ // Keep each upload batch well below the 10-minute activity timeout, the
+ // workflow calls finalizePost again for the remaining bytes.
+ private static readonly YOUTUBE_UPLOAD_BATCH_MS = 4 * 60 * 1000;
+
+ // Resolves the total byte size of the media without loading it into memory:
+ // a HEAD request for remote URLs, statSync for local files.
+ private async youtubeMediaSize(path: string): Promise<number> {
+ if (path.indexOf('http') === 0) {
+ const head = await fetch(path, { method: 'HEAD' });
+ const length = head.headers.get('content-length');
+ if (!length) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Could not determine the video size for the YouTube upload'
+ );
+ }
+ return Number(length);
+ }
+
+ return statSync(path).size;
+ }
+
+ // Returns a streaming body for the [start, end] byte range of the media so we
+ // never hold the whole file in memory: a ranged GET for remote URLs, a ranged
+ // read stream for local files.
+ private async youtubeChunkStream(path: string, start: number, end: number) {
+ if (path.indexOf('http') === 0) {
+ const response = await fetch(path, {
+ headers: { Range: `bytes=${start}-${end}` },
+ });
+ return response.body;
+ }
+
+ return createReadStream(path, { start, end });
+ }
+
+ // Asks the upload session for the truth: the created video when the upload
+ // completed, or the exact byte offset to resume from. We use the global fetch
+ // because the probe answers with 308 (Resume Incomplete), which this.fetch
+ // would treat as an error.
+ private async probeUploadSession(
+ accessToken: string,
+ uploadUri: string,
+ videoSize: number
+ ): Promise<{ videoId: string } | { uploadedBytes: number }> {
+ const probe = await fetch(uploadUri, {
+ method: 'PUT',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Range': `bytes */${videoSize}`,
+ },
+ });
+
+ if (probe.status === 200 || probe.status === 201) {
+ const { id } = await probe.json();
+ return { videoId: String(id) };
+ }
+
+ if (probe.status === 308) {
+ const range = probe.headers.get('range');
+ return {
+ uploadedBytes: range ? Number(range.split('-')[1]) + 1 : 0,
+ };
+ }
+
+ const text = await probe.text().catch(() => '{}');
+
+ // Transient Google-side errors: the session stays resumable, so throw a
+ // plain error - the caller treats it as "probe again later" instead of
+ // permanently failing a resumable upload.
+ if (probe.status === 429 || probe.status >= 500) {
+ throw new Error(
+ `YouTube upload status check failed with ${probe.status}`
+ );
+ }
+
+ const handleError = this.handleErrors(text);
+
+ // this.fetch has the same 401 fallback: Google 401 bodies don't always
+ // match the handleErrors strings
+ if (probe.status === 401 || handleError?.type === 'refresh-token') {
+ throw new RefreshToken(this.identifier, text, '{}');
+ }
+
+ throw new BadBody(
+ this.identifier,
+ text,
+ '{}',
+ probe.status === 404 || probe.status === 410
+ ? 'The upload session expired before the video was uploaded, please post again'
+ : handleError?.value || 'Failed to check the video upload status'
+ );
+ }
+
+ async postPending(
id: string,
accessToken: string,
- postDetails: PostDetails[]
+ postDetails: PostDetails[],
+ integration: Integration
): Promise<PostResponse[]> {
const [firstPost, ...comments] = postDetails;
- const { client, youtube } = clientAndYoutube();
- client.setCredentials({ access_token: accessToken });
- const youtubeClient = youtube(client);
-
const { settings }: { settings: YoutubeSettingsDto } = firstPost;
-
- const response = await axios({
- url: firstPost?.media?.[0]?.path,
- method: 'GET',
- responseType: 'stream',
- });
-
- const all: GaxiosResponse<Schema$Video> = await this.runInConcurrent(
- async () =>
- youtubeClient.videos.insert({
- part: ['id', 'snippet', 'status'],
- notifySubscribers: true,
- requestBody: {
- snippet: {
- title: settings.title,
- description: firstPost?.message,
- ...(settings?.tags?.length
- ? { tags: settings.tags.map((p) => p.label) }
- : {}),
- },
- status: {
- privacyStatus: settings.type,
- selfDeclaredMadeForKids:
- settings.selfDeclaredMadeForKids === 'yes',
- },
+ const path = firstPost?.media?.[0]?.path!;
+ const videoSize = await this.youtubeMediaSize(path);
+
+ // Start a resumable upload session: nothing exists on the channel until
+ // the final byte is received, so nothing here is irreversible yet - the
+ // bytes themselves are streamed by finalizePost, outside this activity.
+ const session = await this.fetch(
+ 'https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=id,snippet,status&notifySubscribers=true',
+ {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Type': 'application/json; charset=UTF-8',
+ 'X-Upload-Content-Type': 'video/mp4',
+ 'X-Upload-Content-Length': String(videoSize),
+ },
+ body: JSON.stringify({
+ snippet: {
+ title: settings.title,
+ description: firstPost?.message,
+ ...(settings?.tags?.length
+ ? { tags: settings.tags.map((p) => p.label) }
+ : {}),
},
- media: {
- body: response.data,
+ status: {
+ privacyStatus: settings.type,
+ selfDeclaredMadeForKids: settings.selfDeclaredMadeForKids === 'yes',
},
}),
- true
+ }
);
- if (settings?.thumbnail?.path) {
+ const uploadUri = session.headers.get('location');
+ if (!uploadUri) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'Could not start the video upload, please try again'
+ );
+ }
+
+ return [
+ {
+ id: firstPost.id,
+ releaseURL: '',
+ postId: '',
+ status: 'pending',
+ pendingData: {
+ uploadUri,
+ videoSize,
+ path,
+ uploadedBytes: 0,
+ thumbnail: settings?.thumbnail?.path || '',
+ },
+ },
+ ];
+ }
+
+ override async checkPostStatus(
+ accessToken: string,
+ pendingData: {
+ uploadUri: string;
+ videoSize: number;
+ path: string;
+ uploadedBytes: number;
+ thumbnail: string;
+ videoId?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ let probe: { videoId: string } | { uploadedBytes: number };
+ try {
+ probe = await this.probeUploadSession(
+ accessToken,
+ pendingData.uploadUri,
+ pendingData.videoSize
+ );
+ } catch (err) {
+ if (err instanceof RefreshToken || err instanceof BadBody) {
+ throw err;
+ }
+
+ // transient network error, the workflow will check again
+ return { status: 'pending', pendingData };
+ }
+
+ // the upload completed: the video exists, only the thumbnail may be left
+ if ('videoId' in probe) {
+ if (pendingData.thumbnail) {
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, videoId: probe.videoId },
+ };
+ }
+
+ return {
+ status: 'completed',
+ postId: probe.videoId,
+ releaseURL: `https://www.youtube.com/watch?v=${probe.videoId}`,
+ };
+ }
+
+ // bytes are still missing, finalizePost resumes from the exact offset
+ return {
+ status: 'ready',
+ pendingData: { ...pendingData, uploadedBytes: probe.uploadedBytes },
+ };
+ }
+
+ override async finalizePost(
+ accessToken: string,
+ pendingData: {
+ uploadUri: string;
+ videoSize: number;
+ path: string;
+ uploadedBytes: number;
+ thumbnail: string;
+ videoId?: string;
+ },
+ integration: Integration
+ ): Promise<PendingCheckResponse> {
+ let videoId = pendingData.videoId;
+
+ if (!videoId) {
+ // ask the session for the truth before sending anything - a previous
+ // run may have died mid-chunk, or even completed without us knowing
+ const probe = await this.probeUploadSession(
+ accessToken,
+ pendingData.uploadUri,
+ pendingData.videoSize
+ );
+
+ if ('uploadedBytes' in probe) {
+ const started = Date.now();
+ let uploaded = probe.uploadedBytes;
+
+ while (uploaded < pendingData.videoSize) {
+ // batch budget exhausted: hand back to the workflow, the next
+ // finalizePost resumes from the offset the session reports
+ if (Date.now() - started > YoutubeProvider.YOUTUBE_UPLOAD_BATCH_MS) {
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, uploadedBytes: uploaded },
+ };
+ }
+
+ const end =
+ Math.min(
+ uploaded + YoutubeProvider.YOUTUBE_CHUNK_SIZE,
+ pendingData.videoSize
+ ) - 1;
+ const body = await this.youtubeChunkStream(
+ pendingData.path,
+ uploaded,
+ end
+ );
+
+ const upload = await fetch(pendingData.uploadUri, {
+ method: 'PUT',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Type': 'video/mp4',
+ 'Content-Length': String(end - uploaded + 1),
+ 'Content-Range': `bytes ${uploaded}-${end}/${pendingData.videoSize}`,
+ },
+ body,
+ // Required by undici when streaming a request body.
+ duplex: 'half',
+ } as any);
+
+ // more bytes expected, resume from the offset the session reports
+ if (upload.status === 308) {
+ const range = upload.headers.get('range');
+ if (!range) {
+ // no committed range reported: don't guess the offset, let the
+ // next probe find the truth
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, uploadedBytes: uploaded },
+ };
+ }
+ uploaded = Number(range.split('-')[1]) + 1;
+ continue;
+ }
+
+ // final chunk received: the video now exists on the channel
+ if (upload.status === 200 || upload.status === 201) {
+ const { id } = await upload.json();
+ videoId = String(id);
+ break;
+ }
+
+ const text = await upload.text().catch(() => '{}');
+
+ // Transient Google-side errors: the session stays resumable, a plain
+ // error makes the workflow probe and resume instead of failing the
+ // whole upload.
+ if (upload.status === 429 || upload.status >= 500) {
+ throw new Error(
+ `YouTube chunk upload failed with ${upload.status}`
+ );
+ }
+
+ const handleError = this.handleErrors(text);
+
+ if (upload.status === 401 || handleError?.type === 'refresh-token') {
+ throw new RefreshToken(this.identifier, text, '{}');
+ }
+
+ throw new BadBody(
+ this.identifier,
+ text,
+ '{}',
+ handleError?.value || 'Failed to upload the video to YouTube'
+ );
+ }
+
+ if (!videoId) {
+ // all bytes sent without a confirmation, the next check asks the session
+ return {
+ status: 'pending',
+ pendingData: { ...pendingData, uploadedBytes: uploaded },
+ };
+ }
+ } else {
+ videoId = probe.videoId;
+ }
+ }
+
+ if (pendingData.thumbnail) {
+ const { client, youtube } = clientAndYoutube();
+ client.setCredentials({ access_token: accessToken });
+ const youtubeClient = youtube(client);
+
await this.runInConcurrent(async () =>
youtubeClient.thumbnails.set({
- videoId: all?.data?.id!,
+ videoId,
media: {
body: (
await axios({
- url: settings?.thumbnail?.path,
+ url: pendingData.thumbnail,
method: 'GET',
responseType: 'stream',
})
@@ -473,14 +777,70 @@ export class YoutubeProvider extends SocialAbstract implements SocialProvider {
);
}
- return [
- {
- id: firstPost.id,
- releaseURL: `https://www.youtube.com/watch?v=${all?.data?.id}`,
- postId: all?.data?.id!,
- status: 'success',
- },
- ];
+ return {
+ status: 'completed',
+ postId: videoId,
+ releaseURL: `https://www.youtube.com/watch?v=${videoId}`,
+ };
+ }
+
+ // Old blocking behavior, kept for workflow versions before v1.0.6 that still
+ // run and don't know how to resolve a `pending` response - they upload and
+ // set the thumbnail inside the activity like before.
+ async post(
+ id: string,
+ accessToken: string,
+ postDetails: PostDetails[],
+ integration: Integration
+ ): Promise<PostResponse[]> {
+ const [response] = await this.postPending(
+ id,
+ accessToken,
+ postDetails,
+ integration
+ );
+
+ if (response.status !== 'pending') {
+ return [response];
+ }
+
+ let pendingData = response.pendingData;
+ const started = Date.now();
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ // Cap below the 10-minute activity timeout of the old workflows using
+ // this method: failing here is safe (no video exists until the upload
+ // completes and the abandoned session just expires), timing the
+ // activity out is not.
+ if (Date.now() - started > 4.5 * 60 * 1000) {
+ throw new BadBody(
+ this.identifier,
+ '{}',
+ '{}',
+ 'The video upload took too long, please try a smaller video'
+ );
+ }
+
+ const finalize = await this.finalizePost(
+ accessToken,
+ pendingData,
+ integration
+ );
+
+ if (finalize.status === 'completed') {
+ return [
+ {
+ id: response.id,
+ releaseURL: finalize.releaseURL,
+ postId: finalize.postId,
+ status: 'success',
+ },
+ ];
+ }
+
+ pendingData = finalize.pendingData;
+ }
}
async analytics(