Using the API

Before starting, make sure you have followed the instructions in the setup page.

Talking to the API

• Generating Video
• Generating Images

In this example, we’ll use the gen4_turbo model to generate a video from an image using the text prompt “Generate a video”. You’ll want to replace the promptImage with a URL of an image and a promptText with your own text prompt.

Node

First, you’ll want to install the Runway SDK. You can do this with npm:

npm install --save @runwayml/sdk

In your code, you can now import the SDK and start making requests:

import RunwayML, { TaskFailedError } from '@runwayml/sdk';

const client = new RunwayML();

// Create a new image-to-video task using the "gen4_turbo" model
try {
  const task = await client.imageToVideo
    .create({
      model: 'gen4_turbo',
      // Point this at your own image file
      promptImage: 'https://example.com/image.jpg',
      promptText: 'Generate a video',
      ratio: '1280:720',
      duration: 5,
    })
    .waitForTaskOutput();

  console.log('Task complete:', task);
} catch (error) {
  if (error instanceof TaskFailedError) {
    console.error('The video failed to generate.');
    console.error(error.taskDetails);
  } else {
    console.error(error);
  }
}

Python

First, you’ll want to install the Runway SDK. You can do this with pip:

pip install runwayml

In your code, you can now import the SDK and start making requests:

from runwayml import RunwayML, TaskFailedError

client = RunwayML()

# Create a new image-to-video task using the "gen4_turbo" model
try:
  task = client.image_to_video.create(
    model='gen4_turbo',
    # Point this at your own image file
    prompt_image='https://example.com/image.jpg',
    prompt_text='Generate a video',
    ratio='1280:720',
    duration=5,
  ).wait_for_task_output()

  print('Task complete:', task)
except TaskFailedError as e:
  print('The video failed to generate.')
  print(e.task_details)

Just testing

If you’re not ready to start writing code, you can test the API with cURL.

# Replace the example URL below with your own image URL
curl -X POST https://api.dev.runwayml.com/v1/image_to_video \
  -d '{
    "promptImage": "https://example.com/",
    "promptText": "string",
    "model": "gen4_turbo",
    "ratio": "1280:720",
    "duration": 5
  }' \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $RUNWAYML_API_SECRET" \
  -H "X-Runway-Version: 2024-11-06"

This command will start an image-to-video task using the “gen4_turbo” model. You’ll see a JSON response with the task ID printed in your terminal, which you can use to fetch the task status.

Uploading base64 encoded images as data URIs

You can also upload base64 encoded images (as a data URI) instead of pointing to an external URL. This can be useful if you’re working with a local image file and want to avoid an extra network round trip to upload the image.

To do this, simply pass the base64 encoded image string as a data URI in the promptImage field instead of a URL. For more information about file types and size limits, see the Inputs page.

Node

import fs from 'node:fs';
import RunwayML, { TaskFailedError } from '@runwayml/sdk';

const client = new RunwayML();

// Read the image file into a Buffer. Replace `example.png` with your own image path.
const imageBuffer = fs.readFileSync('example.png');

// Convert to a data URI. We're using `image/png` here because the input is a PNG.
const dataUri = `data:image/png;base64,${imageBuffer.toString('base64')}`;

// Create a new image-to-video task using the "gen4_turbo" model
try {
  const imageToVideo = await client.imageToVideo
    .create({
      model: 'gen4_turbo',
      // Point this at your own image file
      promptImage: dataUri,
      promptText: 'Generate a video',
      ratio: '1280:720',
      duration: 5,
    })
    .waitForTaskOutput();

  console.log('Task complete:', task);
} catch (error) {
  if (error instanceof TaskFailedError) {
    console.error('The video failed to generate.');
    console.error(error.taskDetails);
  } else {
    console.error(error);
  }
}

Python

import base64
from runwayml import RunwayML

client = RunwayML()

image = './example.png'

# Encode image to a data URI
with open(image, "rb") as f:
    # Read the file as a base64 encoded string
    base64_image = base64.b64encode(f.read()).decode("utf-8")
    # Format it as a data URI
    # We're using `image/png` here because the input is a PNG.
    data_uri = f"data:image/png;base64,{base64_image}"

# Create a new image-to-video task using the "gen4_turbo" model
try:
  task = client.image_to_video.create(
    model='gen4_turbo',
    # Point this at your own image file
    prompt_image=data_uri,
    prompt_text='Generate a video',
    ratio='1280:720',
    duration=5,
  ).wait_for_task_output()

  print('Task complete:', task)
except runwayml.TaskFailedError as e:
  print('The video failed to generate.')
  print(e.task_details)

In this example, we’ll use the gen4_image model to generate an image of the Eiffel Tower rendered in the style of the painting Starry Night. To do this, we’ll pass in reference images of the Eiffel Tower and Starry Night, and use the promptText to specify that we want the Eiffel Tower rendered in the style of Starry Night. Reference images can be referenced in the text prompt using at-mention syntax referencing the tag of the reference image.

Node

import RunwayML, { TaskFailedError } from '@runwayml/sdk';

const client = new RunwayML();

// Create a new image generation task using the "gen4_image" model
try {
  let task = await client.textToImage
    .create({
      model: 'gen4_image',
      ratio: '1920:1080',
      promptText: '@EiffelTower painted in the style of @StarryNight',
      referenceImages: [
        {
          uri: 'https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg'
          tag: 'EiffelTower',
        },
        {
          uri: 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',
          tag: 'StarryNight',
        },
      ],
    })
    .waitForTaskOutput();

  console.log('Task complete:', task);
  console.log('Image URL:', task.output[0]);
} catch (error) {
  if (error instanceof TaskFailedError) {
    console.error('The image failed to generate.');
    console.error(error.taskDetails);
  } else {
    console.error(error);
  }
}

Python

from runwayml import RunwayML, TaskFailedError

client = RunwayML()

try:
  task = client.text_to_image.create(
    model='gen4_image',
    ratio='1920:1080',
    prompt_text='@EiffelTower painted in the style of @StarryNight',
    reference_images=[
      {
        'uri': 'https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg',
        'tag': 'EiffelTower',
      },
      {
        'uri': 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',
        'tag': 'StarryNight',
      },
    ],
  ).wait_for_task_output()

  print('Task complete:', task)
  print('Image URL:', task.output[0])
except TaskFailedError as e:
  print('The image failed to generate.')
  print(e.task_details)

Just testing

If you’re not ready to start writing code, you can test the API with cURL.

# Replace the example URL below with your own image URL
curl -X POST https://api.dev.runwayml.com/v1/text_to_image \
  -d '{
    "promptText": "@EiffelTower painted in the style of @StarryNight",
    "model": "gen4_image",
    "ratio": "1920:1080",
    "referenceImages": [
      {
        "uri": "https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg",
        "tag": "EiffelTower"
      },
      {
        "uri": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",
        "tag": "StarryNight"
      }
    ]
  }' \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $RUNWAYML_API_SECRET" \
  -H "X-Runway-Version: 2024-11-06"

Uploading base64 encoded images as data URIs

You can also upload base64 encoded images (as a data URI) instead of pointing to an external URL. This can be useful if you’re working with a local image file and want to avoid an extra network round trip to upload the image.

To do this, simply pass the base64 encoded image string as a data URI in the uri of a referenceImage object instead of a URL. For more information about file types and size limits, see the Inputs page.

In this example, we’ll use the gen4_image model to generate an image of a bunny facing the camera. We provide a reference image of a rabbit that we use at-mention syntax to reference in the promptText. We also provide an untagged reference image that the model will use to style the output image with.

Node

import fs from 'node:fs';
import path from 'node:path';
import RunwayML, { TaskFailedError } from '@runwayml/sdk';

// Use the `mime-types` package to get the content type of the image file
// Install with `npm install mime-types --save`
import * as mime from 'mime-types';

const client = new RunwayML();

function getImageAsDataUri(imagePath: string) {
  // Read the image file
  const imageBuffer = fs.readFileSync(imagePath);

  // Convert to base64
  const base64String = imageBuffer.toString('base64');

  // Get the MIME type of the image file
  const contentType = mime.lookup(imagePath);

  return `data:${contentType};base64,${base64String}`;
}

// Create a new text-to-image task using the "gen4_image" model
try {
  const textToImage = await client.textToImage
    .create({
      model: 'gen4_image',
      promptText: '@bunny facing the camera',
      ratio: '1920:1080',
      referenceImages: [
        {
          uri: getImageAsDataUri('rabbit.png'),
          tag: 'bunny',
        },
        {
          uri: getImageAsDataUri('style.png'),
        },
      ],
    })
    .waitForTaskOutput();

  console.log('Task complete:', task);
} catch (error) {
  if (error instanceof TaskFailedError) {
    console.error('The image failed to generate.');
    console.error(error.taskDetails);
  } else {
    console.error(error);
  }
}

Python

import base64
import mimetypes
from runwayml import RunwayML, TaskFailedError

client = RunwayML()

def get_image_as_data_uri(image_path: str) -> str:
    with open(image_path, "rb") as f:
        base64_image = base64.b64encode(f.read()).decode("utf-8")
    content_type = mimetypes.guess_file_type(image_path)[0]
    return f"data:{content_type};base64,{base64_image}"

# Create a new text-to-image task using the "gen4_image" model
try:
  task = client.text_to_image.create(
    model='gen4_image',
    ratio='1920:1080',
    prompt_text='@bunny facing the camera',
    reference_images=[
      {
        'uri': get_image_as_data_uri('rabbit.png'),
        'tag': 'bunny',
      },
      {
        'uri': get_image_as_data_uri('style.png'),
      },
    ],
  ).wait_for_task_output()

  print('Task complete:', task)
except TaskFailedError as e:
  print('The image failed to generate.')
  print(e.task_details)