Video infrastructure for AI workflows

Mux works seamlessly as the video foundation for your AI tools through to final end user video delivery.

Or your favorite multi-modal LLM

Turn your video into AI-ready primitives to build workflows at scale. Mux bridges raw video and AI workflows with developer-first infrastructure that handles video complexity while exposing AI-ready video primitives.

Or your favorite multi-modal LLM

import Mux from '@mux/mux-node';
const mux = new Mux();

export async function translateAndDub(assetId: string, playbackId: string, languageName: string, languageCode: string) {
  const dubbingResponse = await fetch('https://mango.sievedata.com/v2/push', {
    method: 'POST',
    headers: {
      'X-API-Key': process.env.SIEVE_API_KEY,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      function: 'sieve/dubbing',
      inputs: {
        source_file: {
          url: `https://stream.mux.com/${playbackId}/audio.m4a`,
        },
        target_language: languageName,
      },
    }),
  });

  const dubbingJob = await dubbingResponse.json();
  const result = await pollDubbingJob(dubbingJob.id);

  await mux.video.assets.createTrack(assetId, {
    type: 'audio',
    language_code: languageCode,
    url: result.outputs[0].data.url,
  });

  return mux.video.assets.retrieve(assetId);
}

async function pollDubbingJob(jobId: string): Promise<any> {
  const response = await fetch(`https://mango.sievedata.com/v2/jobs/${jobId}`, {
    headers: {
      'X-API-Key': process.env.SIEVE_API_KEY!,
    },
  });

  const job = await response.json();
  if (job.status === 'finished') return job;

  await new Promise((resolve) => setTimeout(resolve, 1000));
  return pollDubbingJob(jobId);
}

import OpenAI from 'openai';
import { zodTextFormat } from 'openai/helpers/zod';
import { z } from 'zod';

const openaiClient = new OpenAI();

const schema = z.object({
  chapters: z.array(
    z.object({
      startTime: z.number().describe('Timestamp in seconds from the start of the video'),
      value: z.string().describe('Chapter title'),
    })
  ),
});

export async function generateChapters(playbackId: string, transcriptTrackId: string) {
  const transcriptResponse = await fetch(`https://stream.mux.com/${playbackId}/text/${transcriptTrackId}.vtt`);
  const transcript = await transcriptResponse.text();

  const response = await openaiClient.responses.parse({
    model: 'gpt-4.1-mini',
    input: [
      { role: 'system', content: 'You are a video analysis tool. Segment video transcripts into chapters with timestamps and titles.' },
      { role: 'user', content: `Segment this transcript into chapters with concise titles. Timestamps should be numbers representing seconds.\n\nTranscript:\n${transcript}` },
    ],
    text: { format: zodTextFormat(schema, 'chapters') },
  });

  return response.output_parsed;
}

import OpenAI from 'openai';
import { zodTextFormat } from 'openai/helpers/zod';
import { z } from 'zod';

const openaiClient = new OpenAI();

const schema = z.object({
  summary: z.string().describe('A summary of the video content in 500 characters or less'),
  tags: z.array(z.string()).describe('An array of up to 5 relevant tags that describe the video content'),
});

export async function summarizeVideo(playbackId: string, transcriptTrackId: string) {
  const transcriptResponse = await fetch(`https://stream.mux.com/${playbackId}/text/${transcriptTrackId}.vtt`);
  const transcript = await transcriptResponse.text();

  const response = await openaiClient.responses.parse({
    model: 'gpt-4.1-mini',
    input: [
      { role: 'system', content: 'You are a video analysis tool. Analyze video content and return structured summaries and tags.' },
      {
        role: 'user',
        content: [
          { type: 'input_text', text: `Summarize this video in 500 characters or less and provide up to 5 relevant tags.\n\nTranscript:\n${transcript}` },
          { type: 'input_image', image_url: `https://image.mux.com/${playbackId}/storyboard.png`, detail: 'high' },
        ],
      },
    ],
    text: { format: zodTextFormat(schema, 'tags') },
  });

  return response.output_parsed;
}

const playbackId = "dE02GfTAlJD4RcqNAlgiS2m00LqbdFqlBm"

// Generate a list of thumbnail URLs at regular intervals
const timestamps = [];

const interval = duration / 6;
for (let i = 1; i <= 5; i++) {
  timestamps.push(Math.round(i * interval));
}

const imageUrls = timestamps.map(
  (time) => `https://image.mux.com/${playbackId}/thumbnail.png?time=${time}&width=640`
);

// Send all the thumbnail URLs to OpenAI's moderation API concurrently, and return the scores for each thumbnail
const moderationPromises = imageUrls.map(async (url) => {
  const moderation = await openaiClient.moderations.create({
    model: "omni-moderation-latest",
    input: [
      {
        type: "image_url",
        image_url: {
          url: url,
        },
      },
    ],
  });

  const categoryScores = moderation.results[0].category_scores;

  return {
    nsfw: categoryScores.sexual,
    violence: categoryScores.violence,
  };
});

// Find highest scores across all thumbnails
const scores = await Promise.all(moderationPromises);

const maxNsfw = Math.max(...scores.map(s => s.nsfw));
const maxViolence = Math.max(...scores.map(s => s.violence));

import OpenAI from 'openai';

const openaiClient = new OpenAI();

// Get recommendations for a video
export async function getRecommendations(playbackId: string, transcriptTrackId: string) {
  const embedding = await generateEmbedding(playbackId, transcriptTrackId);
  const recommendations = await findSimilarVideos(embedding, 5);
  
  return recommendations;
}

// Index new video for recommendations
export async function indexVideo(videoId: string, playbackId: string, transcriptTrackId: string, title: string) {
  const embedding = await generateEmbedding(playbackId, transcriptTrackId);
  await storeEmbedding(videoId, embedding, { title, playbackId });
}

// Generate embedding from video transcript
export async function generateEmbedding(playbackId: string, transcriptTrackId: string) {
  const transcriptResponse = await fetch(`https://stream.mux.com/${playbackId}/text/${transcriptTrackId}.vtt`);
  const transcript = await transcriptResponse.text();

  const embedding = await openaiClient.embeddings.create({
    model: 'text-embedding-3-small',
    input: transcript,
  });

  return embedding.data[0].embedding;
}

// Store embedding in vector database
async function storeEmbedding(videoId: string, embedding: number[], metadata: { title: string; playbackId: string }) {
  // Insert into PostgreSQL with pgvector or Pinecone
  await db.query(
    'INSERT INTO video_embeddings (video_id, embedding, metadata) VALUES ($1, $2, $3)',
    [videoId, embedding, metadata]
  );
}

// Find similar videos using vector similarity search
async function findSimilarVideos(embedding: number[], limit: number = 5) {
  // Perform cosine similarity search in vector database
  const results = await db.query(
    'SELECT video_id, title, 1 - (embedding <=> $1) AS similarity FROM video_embeddings ORDER BY embedding <=> $1 LIMIT $2',
    [embedding, limit]
  );
  
  return results.rows;
}

Translation & dubbing

Generate new language tracks of your video’s audio using AI. Then, create translated versions of your video with multi-track playback that automatically switches languages based on viewer preferences.

View the guide

Chaptering

Turn your long-form videos into navigable content using AI. Automatically analyze your video content to identify natural breakpoints, create descriptive chapter titles, and add timestamps to help your viewers find what they need.

View the guide

Summarization & tagging

Intelligently analyze and summarize the content of your video through AI processing of your storyboards and transcripts. Automatically generate summaries, contextual tags, and metadata to ensure content organization and discoverability.

View the guide

Content moderation

Safeguard your platform using AI to detect policy violations. Video frames pass through your moderation flow to be scanned for violence, nudity, or prohibited content. Flagged videos are held, edited, or rejected before reaching your audience.

View the guide

Recommendation engine

Build content-based recommendations by converting video transcripts into AI embeddings. Perform a nearest neighbor search to surface relevant content. Enhance discovery by matching videos based on content similarity, not just tags or categories.

View the guide

Building beyond: your AI workflows, your way

Access your video's AI-ready primitives to build your next workflow. Mux’s video infrastructure powering today’s AI workflows of translation, moderation, and clipping, is the same foundation developers will use to create tomorrow’s innovations.

Start using your video’s AI-ready primitives.  Build something new.

Create a Mux account

Developer friendly video API

Upload and retrieve videos, create clips or monitor video delivery all from an API call. DevEx is core to our product philosophy. We take pride in great docs, tooling, and best-in-class developer support.

Video experts

We serve billions of streams for thousands of customers. Mux is battle-hardened having handled everything from volume spikes to Internet outages, so your mission-critical streams are safe with us.

Affordable, flexible pricing

Start for free with no credit card required. Pay only for what you use with transparent usage-based pricing. Receive automatic discounts as usage grows and see your costs decrease as you scale.

Trusted by top brands

No credit card required to start using Mux.

Start building for free