π¨ The ultimate agentic social media scheduling tool π€
TypeScript nextjsopen-sourceopen-source-social-media-scheduling-toolosspostiz-cloudpostiz-hostedredisscheduling-toolsocial-media-scheduling-tooltypescript
video status tool
Nevo David883327663d36
diff --git a/apps/orchestrator/src/activities/video.activity.ts b/apps/orchestrator/src/activities/video.activity.tsnew file mode 100644index 000000000..8964f8ce9--- /dev/null+++ b/apps/orchestrator/src/activities/video.activity.ts@@ -0,0 +1,24 @@+import { Injectable } from '@nestjs/common';+import { Activity, ActivityMethod } from 'nestjs-temporal-core';+import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';+import { OrganizationService } from '@gitroom/nestjs-libraries/database/prisma/organizations/organization.service';+import { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';++@Injectable()+@Activity()+export class VideoActivity {+ constructor(+ private _mediaService: MediaService,+ private _organizationService: OrganizationService+ ) {}++ @ActivityMethod()+ async generateVideo(organizationId: string, body: VideoDto) {+ const org = await this._organizationService.getOrgById(organizationId);+ if (!org) {+ throw new Error('Organization not found');+ }++ return this._mediaService.generateVideo(org, body);+ }+}diff --git a/apps/orchestrator/src/app.module.ts b/apps/orchestrator/src/app.module.tsindex d7b7f8f1e..e2707b93c 100644--- a/apps/orchestrator/src/app.module.ts+++ b/apps/orchestrator/src/app.module.ts@@ -5,6 +5,8 @@ import { DatabaseModule } from '@gitroom/nestjs-libraries/database/prisma/databaimport { AutopostService } from '@gitroom/nestjs-libraries/database/prisma/autopost/autopost.service';import { EmailActivity } from '@gitroom/orchestrator/activities/email.activity';import { IntegrationsActivity } from '@gitroom/orchestrator/activities/integrations.activity';+import { VideoActivity } from '@gitroom/orchestrator/activities/video.activity';+import { VideoModule } from '@gitroom/nestjs-libraries/videos/video.module';import { HealthController } from '@gitroom/orchestrator/health.controller';const activities = [@@ -12,10 +14,12 @@ const activities = [AutopostService,EmailActivity,IntegrationsActivity,+ VideoActivity,];@Module({imports: [DatabaseModule,+ VideoModule,getTemporalModule(true, require.resolve('./workflows'), activities),],controllers: [HealthController],diff --git a/apps/orchestrator/src/workflows/generate.video.workflow.ts b/apps/orchestrator/src/workflows/generate.video.workflow.tsnew file mode 100644index 000000000..51cb9a573--- /dev/null+++ b/apps/orchestrator/src/workflows/generate.video.workflow.ts@@ -0,0 +1,22 @@+import { proxyActivities } from '@temporalio/workflow';+import { VideoActivity } from '@gitroom/orchestrator/activities/video.activity';+import type { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';++// Generating a video consumes a credit, so a failed attempt is never retried+const { generateVideo } = proxyActivities<VideoActivity>({+ startToCloseTimeout: '40 minute',+ taskQueue: 'main',+ retry: {+ maximumAttempts: 1,+ },+});++export async function generateVideoWorkflow({+ organizationId,+ body,+}: {+ organizationId: string;+ body: VideoDto;+}) {+ return generateVideo(organizationId, body);+}diff --git a/apps/orchestrator/src/workflows/index.ts b/apps/orchestrator/src/workflows/index.tsindex 7fa596225..b94459d15 100644--- a/apps/orchestrator/src/workflows/index.ts+++ b/apps/orchestrator/src/workflows/index.ts@@ -14,3 +14,4 @@ export * from './missing.post.workflow';export * from './send.email.workflow';export * from './refresh.token.workflow';export * from './streak.workflow';+export * from './generate.video.workflow';diff --git a/chatgpt-app-submission.json b/chatgpt-app-submission.jsonindex d85c0a108..82d417524 100644--- a/chatgpt-app-submission.json+++ b/chatgpt-app-submission.json@@ -123,11 +123,23 @@"destructiveHint": false},"justifications": {- "read_only_justification": "Generates a video, consumes the user's video generation credit, uploads the result, and saves it to the media library.",+ "read_only_justification": "Starts a background video generation job that consumes the user's video generation credit, uploads the result, and saves it to the media library.","open_world_justification": "Calls external generation and storage services to create media for the user's workspace.","destructive_justification": "Does not delete or overwrite existing media, revoke access, or publish content."}},+ "videoStatusTool": {+ "annotations": {+ "readOnlyHint": true,+ "openWorldHint": false,+ "destructiveHint": false+ },+ "justifications": {+ "read_only_justification": "Only reads the status of a video generation job the user started and returns the hosted video URL once it is done.",+ "open_world_justification": "Does not write to public internet state or third-party systems.",+ "destructive_justification": "Does not delete, overwrite, revoke access, or send content."+ }+ },"generateImageTool": {"annotations": {"readOnlyHint": false,diff --git a/libraries/nestjs-libraries/src/chat/start.mcp.ts b/libraries/nestjs-libraries/src/chat/start.mcp.tsindex 70881b7d5..889079921 100644--- a/libraries/nestjs-libraries/src/chat/start.mcp.ts+++ b/libraries/nestjs-libraries/src/chat/start.mcp.ts@@ -49,6 +49,7 @@ export const startMcp = async (app: INestApplication) => {const claudeHiddenTools = ['generateImageTool','generateVideoTool',+ 'videoStatusTool','generateVideoOptions','videoFunctionTool',];diff --git a/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts b/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.tsindex e8b1e969f..eab6f1904 100644--- a/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts+++ b/libraries/nestjs-libraries/src/chat/tools/generate.video.tool.ts@@ -38,6 +38,8 @@ export class GenerateVideoTool implements AgentToolInterface {in case the user specified a platform that requires attachment and attachment was not provided,ask if they want to generate a picture of a video.In many cases 'videoFunctionTool' will need to be called first, to get things like voice id+ Generating takes a few minutes, so this only starts the generation and returns a jobId:+ poll 'videoStatusTool' with it until the status is "completed" to get the video url.Here are the type of video that can be generated:${this._videoManager.getAllVideos()@@ -55,14 +57,14 @@ export class GenerateVideoTool implements AgentToolInterface {),}),outputSchema: z.object({- url: z.string().optional(),+ jobId: z.string().optional(),error: z.string().optional(),}),execute: async (inputData, context) => {checkAuth(inputData, context);const org = JSON.parse((context?.requestContext as any)?.get('organization') as string);try {- const value = await this._mediaService.generateVideo(org, {+ const value = await this._mediaService.startGenerateVideo(org, {type: inputData.identifier,output: inputData.output,customParams: inputData.customParams.reduce(@@ -75,7 +77,7 @@ export class GenerateVideoTool implements AgentToolInterface {});return {- url: value.path,+ jobId: value.jobId,};} catch (err) {// SubscriptionException (402) carries { section, action } and itsdiff --git a/libraries/nestjs-libraries/src/chat/tools/tool.list.ts b/libraries/nestjs-libraries/src/chat/tools/tool.list.tsindex ef386474f..dc5fc9893 100644--- a/libraries/nestjs-libraries/src/chat/tools/tool.list.ts+++ b/libraries/nestjs-libraries/src/chat/tools/tool.list.ts@@ -4,6 +4,7 @@ import { IntegrationSchedulePostTool } from './integration.schedule.post';import { GenerateVideoOptionsTool } from '@gitroom/nestjs-libraries/chat/tools/generate.video.options.tool';import { VideoFunctionTool } from '@gitroom/nestjs-libraries/chat/tools/video.function.tool';import { GenerateVideoTool } from '@gitroom/nestjs-libraries/chat/tools/generate.video.tool';+import { VideoStatusTool } from '@gitroom/nestjs-libraries/chat/tools/video.status.tool';import { GenerateImageTool } from '@gitroom/nestjs-libraries/chat/tools/generate.image.tool';import { IntegrationListTool } from '@gitroom/nestjs-libraries/chat/tools/integration.list.tool';import { GroupListTool } from '@gitroom/nestjs-libraries/chat/tools/group.list.tool';@@ -22,6 +23,7 @@ export const toolList = [GenerateVideoOptionsTool,VideoFunctionTool,GenerateVideoTool,+ VideoStatusTool,GenerateImageTool,UploadFromUrlTool,];diff --git a/libraries/nestjs-libraries/src/chat/tools/video.status.tool.ts b/libraries/nestjs-libraries/src/chat/tools/video.status.tool.tsnew file mode 100644index 000000000..225103ce5--- /dev/null+++ b/libraries/nestjs-libraries/src/chat/tools/video.status.tool.ts@@ -0,0 +1,65 @@+import { AgentToolInterface } from '@gitroom/nestjs-libraries/chat/agent.tool.interface';+import { createTool } from '@mastra/core/tools';+import { Injectable } from '@nestjs/common';+import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';+import { checkAuth } from '@gitroom/nestjs-libraries/chat/auth.context';+import { z } from 'zod';++@Injectable()+export class VideoStatusTool implements AgentToolInterface {+ constructor(private _mediaService: MediaService) {}+ name = 'videoStatusTool';++ run() {+ return createTool({+ id: 'videoStatusTool',+ mcp: {+ annotations: {+ title: 'Video Generation Status',+ readOnlyHint: true,+ destructiveHint: false,+ idempotentHint: true,+ openWorldHint: false,+ },+ },+ description: `Check the status of a video generation started with 'generateVideoTool', using the jobId it returned.+ Generating a video takes a few minutes: while the status is "pending", wait about 30 seconds and call again.+ When the status is "completed" the result contains the hosted video url, which can be used as a post attachment.+ When the status is "failed" the result contains the error message.+ `,+ inputSchema: z.object({+ jobId: z.string().describe('The jobId returned by generateVideoTool'),+ }),+ outputSchema: z.object({+ status: z.enum(['pending', 'completed', 'failed']).optional(),+ id: z.string().optional(),+ url: z.string().optional(),+ error: z.string().optional(),+ }),+ execute: async (inputData, context) => {+ checkAuth(inputData, context);+ const org = JSON.parse(+ (context?.requestContext as any)?.get('organization') as string+ );+ try {+ const job = await this._mediaService.getGenerateVideoStatus(+ org,+ inputData.jobId+ );++ return {+ status: job.status,+ id: job.id,+ url: job.path,+ error: job.error,+ };+ } catch (err) {+ const message = err instanceof Error ? err.message : String(err);+ return {+ error: `Video job lookup failed: ${message}`,+ };+ }+ },+ });+ }+}diff --git a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts b/libraries/nestjs-libraries/src/database/prisma/media/media.service.tsindex 1d1164a91..c56534b85 100644--- a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts+++ b/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts@@ -13,6 +13,10 @@ import {Sections,SubscriptionException,} from '@gitroom/backend/services/auth/permissions/permission.exception.class';+import { TemporalService } from 'nestjs-temporal-core';+import { TypedSearchAttributes } from '@temporalio/common';+import { organizationId } from '@gitroom/nestjs-libraries/temporal/temporal.search.attribute';+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';@Injectable()export class MediaService {@@ -22,7 +26,8 @@ export class MediaService {private _mediaRepository: MediaRepository,private _openAi: OpenaiService,private _subscriptionService: SubscriptionService,- private _videoManager: VideoManager+ private _videoManager: VideoManager,+ private _temporalService: TemporalService) {}async deleteMedia(org: string, id: string) {@@ -86,35 +91,35 @@ export class MediaService {return true;}- async generateVideo(org: Organization, body: VideoDto) {- try {- const totalCredits = await this._subscriptionService.checkCredits(- org,- 'ai_videos'- );+ private async validateVideoRequest(org: Organization, body: VideoDto) {+ const totalCredits = await this._subscriptionService.checkCredits(+ org,+ 'ai_videos'+ );++ if (totalCredits.credits <= 0) {+ throw new SubscriptionException({+ action: AuthorizationActions.Create,+ section: Sections.VIDEOS_PER_MONTH,+ });+ }- if (totalCredits.credits <= 0) {- throw new SubscriptionException({- action: AuthorizationActions.Create,- section: Sections.VIDEOS_PER_MONTH,- });- }+ const video = this._videoManager.getVideoByName(body.type);+ if (!video) {+ throw new Error(`Video type ${body.type} not found`);+ }- const video = this._videoManager.getVideoByName(body.type);- if (!video) {- throw new Error(`Video type ${body.type} not found`);- }+ if (!video.trial && org.isTrailing) {+ throw new HttpException('This video is not available in trial mode', 406);+ }- if (!video.trial && org.isTrailing) {- throw new HttpException(- 'This video is not available in trial mode',- 406- );- }+ await video.instance.processAndValidate(body.customParams);+ return video;+ }- console.log(body.customParams);- await video.instance.processAndValidate(body.customParams);- console.log('no err');+ async generateVideo(org: Organization, body: VideoDto) {+ try {+ const video = await this.validateVideoRequest(org, body);return await this._subscriptionService.useCredit(org,@@ -134,6 +139,86 @@ export class MediaService {}}+ // Generating a video takes minutes, longer than an MCP request can stay open,+ // so the generation runs in a workflow and the caller polls its status by job id+ async startGenerateVideo(org: Organization, body: VideoDto) {+ // validated here as well as in the workflow so bad input fails before a job exists+ try {+ await this.validateVideoRequest(org, body);+ } catch (err) {+ throw generationError(err);+ }++ const client = this._temporalService.client.getRawClient();+ if (!client) {+ throw new HttpException('Video generation is not available', 503);+ }++ const jobId = `video_${org.id}_${makeId(10)}`;+ await client.workflow.start('generateVideoWorkflow', {+ workflowId: jobId,+ taskQueue: 'main',+ args: [+ {+ organizationId: org.id,+ body,+ },+ ],+ typedSearchAttributes: new TypedSearchAttributes([+ {+ key: organizationId,+ value: org.id,+ },+ ]),+ });++ return { jobId };+ }++ async getGenerateVideoStatus(+ org: Organization,+ jobId: string+ ): Promise<{+ status: 'pending' | 'completed' | 'failed';+ id?: string;+ path?: string;+ error?: string;+ }> {+ // the job id carries the organization, so one org can't poll another's job+ if (!jobId.startsWith(`video_${org.id}_`)) {+ throw new HttpException('Video job not found', 404);+ }++ const handle = await this._temporalService.client.getWorkflowHandle(jobId);+ let status: string;+ try {+ status = (await handle.describe()).status.name;+ } catch (err) {+ throw new HttpException('Video job not found', 404);+ }++ if (status === 'RUNNING') {+ return { status: 'pending' };+ }++ try {+ const media = (await handle.result()) as Awaited<+ ReturnType<MediaService['saveFile']>+ >;+ return { status: 'completed', id: media.id, path: media.path };+ } catch (err) {+ // the workflow failure wraps the activity failure which wraps the actual error+ let cause: any = err;+ while (cause?.cause && cause.cause !== cause) {+ cause = cause.cause;+ }+ return {+ status: 'failed',+ error: cause?.message || String(err),+ };+ }+ }+async videoFunction(identifier: string, functionName: string, body: any) {const video = this._videoManager.getVideoByName(identifier);if (!video) {