Skip to content

πŸ“¨ The ultimate agentic social media scheduling tool πŸ€–

video status tool

Nevo David883327663d36
diff --git a/apps/orchestrator/src/activities/video.activity.ts b/apps/orchestrator/src/activities/video.activity.ts
new file mode 100644
index 000000000..8964f8ce9
--- /dev/null
+++ b/apps/orchestrator/src/activities/video.activity.ts
@@ -0,0 +1,24 @@
+import { Injectable } from '@nestjs/common';
+import { Activity, ActivityMethod } from 'nestjs-temporal-core';
+import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
+import { OrganizationService } from '@gitroom/nestjs-libraries/database/prisma/organizations/organization.service';
+import { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';
+
+@Injectable()
+@Activity()
+export class VideoActivity {
+ constructor(
+ private _mediaService: MediaService,
+ private _organizationService: OrganizationService
+ ) {}
+
+ @ActivityMethod()
+ async generateVideo(organizationId: string, body: VideoDto) {
+ const org = await this._organizationService.getOrgById(organizationId);
+ if (!org) {
+ throw new Error('Organization not found');
+ }
+
+ return this._mediaService.generateVideo(org, body);
+ }
+}
diff --git a/apps/orchestrator/src/app.module.ts b/apps/orchestrator/src/app.module.ts
index d7b7f8f1e..e2707b93c 100644
--- a/apps/orchestrator/src/app.module.ts
+++ b/apps/orchestrator/src/app.module.ts
@@ -5,6 +5,8 @@ import { DatabaseModule } from '@gitroom/nestjs-libraries/database/prisma/databa
import { AutopostService } from '@gitroom/nestjs-libraries/database/prisma/autopost/autopost.service';
import { EmailActivity } from '@gitroom/orchestrator/activities/email.activity';
import { IntegrationsActivity } from '@gitroom/orchestrator/activities/integrations.activity';
+import { VideoActivity } from '@gitroom/orchestrator/activities/video.activity';
+import { VideoModule } from '@gitroom/nestjs-libraries/videos/video.module';
import { HealthController } from '@gitroom/orchestrator/health.controller';
const activities = [
@@ -12,10 +14,12 @@ const activities = [
AutopostService,
EmailActivity,
IntegrationsActivity,
+ VideoActivity,
];
@Module({
imports: [
DatabaseModule,
+ VideoModule,
getTemporalModule(true, require.resolve('./workflows'), activities),
],
controllers: [HealthController],
diff --git a/apps/orchestrator/src/workflows/generate.video.workflow.ts b/apps/orchestrator/src/workflows/generate.video.workflow.ts
new file mode 100644
index 000000000..51cb9a573
--- /dev/null
+++ b/apps/orchestrator/src/workflows/generate.video.workflow.ts
@@ -0,0 +1,22 @@
+import { proxyActivities } from '@temporalio/workflow';
+import { VideoActivity } from '@gitroom/orchestrator/activities/video.activity';
+import type { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';
+
+// Generating a video consumes a credit, so a failed attempt is never retried
+const { generateVideo } = proxyActivities<VideoActivity>({
+ startToCloseTimeout: '40 minute',
+ taskQueue: 'main',
+ retry: {
+ maximumAttempts: 1,
+ },
+});
+
+export async function generateVideoWorkflow({
+ organizationId,
+ body,
+}: {
+ organizationId: string;
+ body: VideoDto;
+}) {
+ return generateVideo(organizationId, body);
+}
diff --git a/apps/orchestrator/src/workflows/index.ts b/apps/orchestrator/src/workflows/index.ts
index 7fa596225..b94459d15 100644
--- a/apps/orchestrator/src/workflows/index.ts
+++ b/apps/orchestrator/src/workflows/index.ts
@@ -14,3 +14,4 @@ export * from './missing.post.workflow';
export * from './send.email.workflow';
export * from './refresh.token.workflow';
export * from './streak.workflow';
+export * from './generate.video.workflow';
diff --git a/chatgpt-app-submission.json b/chatgpt-app-submission.json
index d85c0a108..82d417524 100644
--- a/chatgpt-app-submission.json
+++ b/chatgpt-app-submission.json
@@ -123,11 +123,23 @@
"destructiveHint": false
},
"justifications": {
- "read_only_justification": "Generates a video, consumes the user's video generation credit, uploads the result, and saves it to the media library.",
+ "read_only_justification": "Starts a background video generation job that consumes the user's video generation credit, uploads the result, and saves it to the media library.",
"open_world_justification": "Calls external generation and storage services to create media for the user's workspace.",
"destructive_justification": "Does not delete or overwrite existing media, revoke access, or publish content."
}
},
+ "videoStatusTool": {
+ "annotations": {
+ "readOnlyHint": true,
+ "openWorldHint": false,
+ "destructiveHint": false
+ },
+ "justifications": {
+ "read_only_justification": "Only reads the status of a video generation job the user started and returns the hosted video URL once it is done.",
+ "open_world_justification": "Does not write to public internet state or third-party systems.",
+ "destructive_justification": "Does not delete, overwrite, revoke access, or send content."
+ }
+ },
"generateImageTool": {
"annotations": {
"readOnlyHint": false,
diff --git a/libraries/nestjs-libraries/src/chat/start.mcp.ts b/libraries/nestjs-libraries/src/chat/start.mcp.ts
index 70881b7d5..889079921 100644
--- a/libraries/nestjs-libraries/src/chat/start.mcp.ts
+++ b/libraries/nestjs-libraries/src/chat/start.mcp.ts
@@ -49,6 +49,7 @@ export const startMcp = async (app: INestApplication) => {
const claudeHiddenTools = [
'generateImageTool',
'generateVideoTool',
+ 'videoStatusTool',
'generateVideoOptions',
'videoFunctionTool',
];
diff --git a/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts b/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts
index e8b1e969f..eab6f1904 100644
--- a/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts
+++ b/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts
@@ -38,6 +38,8 @@ export class GenerateVideoTool implements AgentToolInterface {
in case the user specified a platform that requires attachment and attachment was not provided,
ask if they want to generate a picture of a video.
In many cases 'videoFunctionTool' will need to be called first, to get things like voice id
+ Generating takes a few minutes, so this only starts the generation and returns a jobId:
+ poll 'videoStatusTool' with it until the status is "completed" to get the video url.
Here are the type of video that can be generated:
${this._videoManager
.getAllVideos()
@@ -55,14 +57,14 @@ export class GenerateVideoTool implements AgentToolInterface {
),
}),
outputSchema: z.object({
- url: z.string().optional(),
+ jobId: z.string().optional(),
error: z.string().optional(),
}),
execute: async (inputData, context) => {
checkAuth(inputData, context);
const org = JSON.parse((context?.requestContext as any)?.get('organization') as string);
try {
- const value = await this._mediaService.generateVideo(org, {
+ const value = await this._mediaService.startGenerateVideo(org, {
type: inputData.identifier,
output: inputData.output,
customParams: inputData.customParams.reduce(
@@ -75,7 +77,7 @@ export class GenerateVideoTool implements AgentToolInterface {
});
return {
- url: value.path,
+ jobId: value.jobId,
};
} catch (err) {
// SubscriptionException (402) carries { section, action } and its
diff --git a/libraries/nestjs-libraries/src/chat/tools/tool.list.ts b/libraries/nestjs-libraries/src/chat/tools/tool.list.ts
index ef386474f..dc5fc9893 100644
--- a/libraries/nestjs-libraries/src/chat/tools/tool.list.ts
+++ b/libraries/nestjs-libraries/src/chat/tools/tool.list.ts
@@ -4,6 +4,7 @@ import { IntegrationSchedulePostTool } from './integration.schedule.post';
import { GenerateVideoOptionsTool } from '@gitroom/nestjs-libraries/chat/tools/generate.video.options.tool';
import { VideoFunctionTool } from '@gitroom/nestjs-libraries/chat/tools/video.function.tool';
import { GenerateVideoTool } from '@gitroom/nestjs-libraries/chat/tools/generate.video.tool';
+import { VideoStatusTool } from '@gitroom/nestjs-libraries/chat/tools/video.status.tool';
import { GenerateImageTool } from '@gitroom/nestjs-libraries/chat/tools/generate.image.tool';
import { IntegrationListTool } from '@gitroom/nestjs-libraries/chat/tools/integration.list.tool';
import { GroupListTool } from '@gitroom/nestjs-libraries/chat/tools/group.list.tool';
@@ -22,6 +23,7 @@ export const toolList = [
GenerateVideoOptionsTool,
VideoFunctionTool,
GenerateVideoTool,
+ VideoStatusTool,
GenerateImageTool,
UploadFromUrlTool,
];
diff --git a/libraries/nestjs-libraries/src/chat/tools/video.status.tool.ts b/libraries/nestjs-libraries/src/chat/tools/video.status.tool.ts
new file mode 100644
index 000000000..225103ce5
--- /dev/null
+++ b/libraries/nestjs-libraries/src/chat/tools/video.status.tool.ts
@@ -0,0 +1,65 @@
+import { AgentToolInterface } from '@gitroom/nestjs-libraries/chat/agent.tool.interface';
+import { createTool } from '@mastra/core/tools';
+import { Injectable } from '@nestjs/common';
+import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
+import { checkAuth } from '@gitroom/nestjs-libraries/chat/auth.context';
+import { z } from 'zod';
+
+@Injectable()
+export class VideoStatusTool implements AgentToolInterface {
+ constructor(private _mediaService: MediaService) {}
+ name = 'videoStatusTool';
+
+ run() {
+ return createTool({
+ id: 'videoStatusTool',
+ mcp: {
+ annotations: {
+ title: 'Video Generation Status',
+ readOnlyHint: true,
+ destructiveHint: false,
+ idempotentHint: true,
+ openWorldHint: false,
+ },
+ },
+ description: `Check the status of a video generation started with 'generateVideoTool', using the jobId it returned.
+ Generating a video takes a few minutes: while the status is "pending", wait about 30 seconds and call again.
+ When the status is "completed" the result contains the hosted video url, which can be used as a post attachment.
+ When the status is "failed" the result contains the error message.
+ `,
+ inputSchema: z.object({
+ jobId: z.string().describe('The jobId returned by generateVideoTool'),
+ }),
+ outputSchema: z.object({
+ status: z.enum(['pending', 'completed', 'failed']).optional(),
+ id: z.string().optional(),
+ url: z.string().optional(),
+ error: z.string().optional(),
+ }),
+ execute: async (inputData, context) => {
+ checkAuth(inputData, context);
+ const org = JSON.parse(
+ (context?.requestContext as any)?.get('organization') as string
+ );
+ try {
+ const job = await this._mediaService.getGenerateVideoStatus(
+ org,
+ inputData.jobId
+ );
+
+ return {
+ status: job.status,
+ id: job.id,
+ url: job.path,
+ error: job.error,
+ };
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ return {
+ error: `Video job lookup failed: ${message}`,
+ };
+ }
+ },
+ });
+ }
+}
diff --git a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts b/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts
index 1d1164a91..c56534b85 100644
--- a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts
+++ b/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts
@@ -13,6 +13,10 @@ import {
Sections,
SubscriptionException,
} from '@gitroom/backend/services/auth/permissions/permission.exception.class';
+import { TemporalService } from 'nestjs-temporal-core';
+import { TypedSearchAttributes } from '@temporalio/common';
+import { organizationId } from '@gitroom/nestjs-libraries/temporal/temporal.search.attribute';
+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
@Injectable()
export class MediaService {
@@ -22,7 +26,8 @@ export class MediaService {
private _mediaRepository: MediaRepository,
private _openAi: OpenaiService,
private _subscriptionService: SubscriptionService,
- private _videoManager: VideoManager
+ private _videoManager: VideoManager,
+ private _temporalService: TemporalService
) {}
async deleteMedia(org: string, id: string) {
@@ -86,35 +91,35 @@ export class MediaService {
return true;
}
- async generateVideo(org: Organization, body: VideoDto) {
- try {
- const totalCredits = await this._subscriptionService.checkCredits(
- org,
- 'ai_videos'
- );
+ private async validateVideoRequest(org: Organization, body: VideoDto) {
+ const totalCredits = await this._subscriptionService.checkCredits(
+ org,
+ 'ai_videos'
+ );
+
+ if (totalCredits.credits <= 0) {
+ throw new SubscriptionException({
+ action: AuthorizationActions.Create,
+ section: Sections.VIDEOS_PER_MONTH,
+ });
+ }
- if (totalCredits.credits <= 0) {
- throw new SubscriptionException({
- action: AuthorizationActions.Create,
- section: Sections.VIDEOS_PER_MONTH,
- });
- }
+ const video = this._videoManager.getVideoByName(body.type);
+ if (!video) {
+ throw new Error(`Video type ${body.type} not found`);
+ }
- const video = this._videoManager.getVideoByName(body.type);
- if (!video) {
- throw new Error(`Video type ${body.type} not found`);
- }
+ if (!video.trial && org.isTrailing) {
+ throw new HttpException('This video is not available in trial mode', 406);
+ }
- if (!video.trial && org.isTrailing) {
- throw new HttpException(
- 'This video is not available in trial mode',
- 406
- );
- }
+ await video.instance.processAndValidate(body.customParams);
+ return video;
+ }
- console.log(body.customParams);
- await video.instance.processAndValidate(body.customParams);
- console.log('no err');
+ async generateVideo(org: Organization, body: VideoDto) {
+ try {
+ const video = await this.validateVideoRequest(org, body);
return await this._subscriptionService.useCredit(
org,
@@ -134,6 +139,86 @@ export class MediaService {
}
}
+ // Generating a video takes minutes, longer than an MCP request can stay open,
+ // so the generation runs in a workflow and the caller polls its status by job id
+ async startGenerateVideo(org: Organization, body: VideoDto) {
+ // validated here as well as in the workflow so bad input fails before a job exists
+ try {
+ await this.validateVideoRequest(org, body);
+ } catch (err) {
+ throw generationError(err);
+ }
+
+ const client = this._temporalService.client.getRawClient();
+ if (!client) {
+ throw new HttpException('Video generation is not available', 503);
+ }
+
+ const jobId = `video_${org.id}_${makeId(10)}`;
+ await client.workflow.start('generateVideoWorkflow', {
+ workflowId: jobId,
+ taskQueue: 'main',
+ args: [
+ {
+ organizationId: org.id,
+ body,
+ },
+ ],
+ typedSearchAttributes: new TypedSearchAttributes([
+ {
+ key: organizationId,
+ value: org.id,
+ },
+ ]),
+ });
+
+ return { jobId };
+ }
+
+ async getGenerateVideoStatus(
+ org: Organization,
+ jobId: string
+ ): Promise<{
+ status: 'pending' | 'completed' | 'failed';
+ id?: string;
+ path?: string;
+ error?: string;
+ }> {
+ // the job id carries the organization, so one org can't poll another's job
+ if (!jobId.startsWith(`video_${org.id}_`)) {
+ throw new HttpException('Video job not found', 404);
+ }
+
+ const handle = await this._temporalService.client.getWorkflowHandle(jobId);
+ let status: string;
+ try {
+ status = (await handle.describe()).status.name;
+ } catch (err) {
+ throw new HttpException('Video job not found', 404);
+ }
+
+ if (status === 'RUNNING') {
+ return { status: 'pending' };
+ }
+
+ try {
+ const media = (await handle.result()) as Awaited<
+ ReturnType<MediaService['saveFile']>
+ >;
+ return { status: 'completed', id: media.id, path: media.path };
+ } catch (err) {
+ // the workflow failure wraps the activity failure which wraps the actual error
+ let cause: any = err;
+ while (cause?.cause && cause.cause !== cause) {
+ cause = cause.cause;
+ }
+ return {
+ status: 'failed',
+ error: cause?.message || String(err),
+ };
+ }
+ }
+
async videoFunction(identifier: string, functionName: string, body: any) {
const video = this._videoManager.getVideoByName(identifier);
if (!video) {