面向 AI 助手：本页的 Markdown 版本位于 https://ark.volcengine.com/region:cn-beijing/docs/video-understanding.md，全部文档索引见 https://ark.volcengine.com/llms.txt。

视频理解

视觉大模型可理解视频中的视觉信息，完成如描述其中的物体、分析动作逻辑等视觉相关任务，可用于自动化视频内容审核、智能监控分析等，大幅节省人工成本，适用于智能安防、体育赛事分析、媒体内容管理等领域。这篇教程帮助您通过 API 完成含视频输入的各类任务。

支持模型#

请参见视觉理解能力。

API 接口#

视频传入方式#

支持的视频文件传入方式如下：

  • 本地文件上传：

    • Files API 上传（推荐）：直接传入本地文件，文件存储在方舟平台托管的默认存储空间中时，支持最大 512 MB 文件的上传；文件存储在火山引擎对象存储 TOS Bucket 中时，支持最大 2 GB 的视频文件的上传，从而满足大文件处理需求。

    • Base64 编码传入：适用于文件体积较小的场景，视频文件小于 50 MB，请求体不能超过 64 MB。

  • 视频 URL 传入：适用于文件已存在公网可访问 URL 的场景，视频文件大小不能超过 50 MB。

本地文件上传#

Files API 上传（推荐）#

建议优先使用 Files API 上传本地文件。该方式下，文件存储在方舟平台托管的默认存储空间时，支持最大 512 MB 文件上传；存储在火山引擎对象存储 TOS Bucket 时，支持最大 2 GB 视频文件上传，可充分满足大文件处理需求。同时可避免请求时重复上传内容，减少预处理时延，文件还能在多次请求中复用，节省公网下载耗时。

通过 Files API 上传视频时，需在上传请求的 preprocess_configs.video.model 中传入支持视频理解的模型 ID。该配置用于指定视频预处理和抽帧策略；后续调用 Responses API 或 Chat API 时，仍需在请求的 model 中传入实际用于推理的模型或接入点。

上传成功并待文件状态变为 active 后，Responses API 和 Chat API 均支持通过 file_id 方式传入视频。

  • 该方式上传的文件默认存储 7 天，存储有效期取值范围为1-30天。
  • 如果需要实时获取分析内容，或者要规避复杂任务引发的客户端超时失败问题，可采用流式输出的方式，具体示例见流式输出。

不传 TOS 参数时，文件默认存储至方舟平台托管的默认存储空间。示例代码如下：

  • Responses API 示例代码：

  1. 上传视频文件获取File ID。

    Bash
    curl https://ark.cn-beijing.volces.com/api/v3/files \
-H "Authorization: Bearer $ARK_API_KEY" \
-F 'purpose=user_data' \
-F 'file=@/Users/doc/demo.mp4' \
-F 'preprocess_configs[video][model]=doubao-seed-2-1-pro-260915' \
-F 'preprocess_configs[video][fps]=0.3'

  2. 在Responses API中引用File ID。

    Bash
    curl https://ark.cn-beijing.volces.com/api/v3/responses \
-H "Authorization: Bearer $ARK_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
    "model": "doubao-seed-2-1-pro-260915",
    "input": [
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "file_id": "file-20251018****"
                },
                {
                    "type": "input_text",
                    "text": "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"
                }
            ]
        }
    ]
}'
Python
import os
from pathlib import Path

from arkruntime import Ark

client = Ark(
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    api_key=os.environ.get("ARK_API_KEY"),
)
file_path = Path(os.getenv("FILE_PATH", "/Users/doc/demo.mp4"))

uploaded_file = client.files.create(
    file=file_path,
    purpose="user_data",
    preprocess_configs={"video": {"model": "doubao-seed-2-1-pro-260915", "fps": 0.3}},
)
client.files.wait_for_processing(uploaded_file.id)

response = client.responses.create(
    model=os.getenv("MODEL_ID", "doubao-seed-2-1-pro-260915"),
    input=[
        {
            "role": "user",
            "content": [
                {"type": "input_video", "file_id": uploaded_file.id},
                {"type": "input_text", "text": "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"},
            ],
        }
    ],
)
print(response)
Go
package main

import (
	"context"
	"fmt"
	"os"

	"github.com/volcengine/ark-runtime-go/arkruntime"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/file"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/responses"
)

func main() {
	ctx := context.Background()
    client := arkruntime.NewClientWithApiKey(
		os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
	)
	filePath := os.Getenv("FILE_PATH")
	if filePath == "" {
		filePath = "/Users/doc/demo.mp4"
	}
	data, err := os.Open(filePath)
	if err != nil {
		panic(err)
	}
	defer data.Close()

	uploaded, err := client.UploadFile(ctx, &file.FileCreateRequest{
		Purpose: file.PurposeUserData,
		PreprocessConfigs: file.NewOptPreprocessConfigs(file.PreprocessConfigs{
			Video: file.NewOptVideo(file.Video{
				Model: file.NewOptString("doubao-seed-2-1-pro-260915"),
				Fps:   file.NewOptFloat64(0.3),
			}),
		}),
	}, data)
	if err != nil {
		panic(err)
	}
	uploaded, err = client.WaitForFileProcessing(
		ctx,
		uploaded.ID,
		arkruntime.WaitForFileProcessingOptions{},
	)
	if err != nil || uploaded.Status != file.StatusActive {
		panic(fmt.Sprintf("file processing failed: %v, status=%s", err, uploaded.Status))
	}

	media := responses.ContentItem{OneOf: responses.NewContentItemVideoContentItemSum(responses.ContentItemVideo{Type: responses.ContentItemVideoTypeInputVideo, VideoURL: "", FileID: responses.NewOptString(uploaded.ID)})}
	text := responses.ContentItem{OneOf: responses.NewContentItemTextContentItemSum(
		responses.ContentItemText{Type: responses.ContentItemTextTypeInputText, Text: "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"},
	)}
	message := responses.InputItem{OneOf: responses.NewItemEasyMessageInputItemSum(
		responses.ItemEasyMessage{
			Role:    responses.NewOptMessageRole(responses.MessageRoleUser),
			Content: responses.NewContentItemArrayMessageContent([]responses.ContentItem{media, text}),
		},
	)}
	modelID := os.Getenv("MODEL_ID")
	if modelID == "" {
		modelID = "doubao-seed-2-1-pro-260915"
	}
	response, err := client.CreateResponses(ctx, &responses.ResponsesRequest{
		Model: modelID,
		Input: responses.NewInputItemArrayResponsesInput([]responses.InputItem{message}),
	})
	if err != nil {
		panic(err)
	}
	fmt.Println(response)
}
Java
import com.volcengine.ark.runtime.models.file.*;
import com.volcengine.ark.runtime.models.responses.*;
import com.volcengine.ark.runtime.service.ArkService;

import java.io.File;
import java.util.Arrays;

public class Main {
    public static void main(String[] args) {
        ArkService service = ArkService.builder()
                .apiKey(System.getenv("ARK_API_KEY"))
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();
        String filePath = System.getenv().getOrDefault("FILE_PATH", "/Users/doc/demo.mp4");
        String modelId = System.getenv().getOrDefault("MODEL_ID", "doubao-seed-2-1-pro-260915");

        FileObject uploaded = service.uploadFile(
                FileCreateRequest.builder()
                        .purpose(Purpose.USER_DATA)
                        .preprocessConfigs(PreprocessConfigs.builder().video(Video.builder().model("doubao-seed-2-1-pro-260915").fps(0.3).build()).build())
                        .build(),
                new File(filePath)
        );
        uploaded = service.waitForFileProcessing(uploaded.getId());
        if (uploaded.getStatus() != Status.ACTIVE) {
            throw new IllegalStateException("File processing failed: " + uploaded.getStatus());
        }

        ContentItem media = ContentItemVideo.builder().type(ContentItemType.INPUT_VIDEO).fileId(uploaded.getId()).build();
        ContentItem text = ContentItemText.builder()
                .type(ContentItemType.INPUT_TEXT)
                .text("请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。")
                .build();
        ResponsesRequest request = ResponsesRequest.builder()
                .model(modelId)
                .input(ResponsesInput.ofList(Arrays.asList(
                        ItemEasyMessage.builder()
                                .role(MessageRole.USER)
                                .content(MessageContent.ofList(Arrays.asList(media, text)))
                                .build()
                )))
                .build();
        Response response = service.createResponse(request);
        System.out.println(response);
        service.shutdownExecutor();
    }
}
Python
import os
import time
from openai import OpenAI

api_key = os.getenv('ARK_API_KEY')

client = OpenAI(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)

file = client.files.create(
    file=open("/Users/doc/demo.mp4", "rb"),
    purpose="user_data",
    extra_body={
        "preprocess_configs": {
            "video": {
                "model": "doubao-seed-2-1-pro-260915",
                "fps": 0.3,
            }
        }
    },
)
# Wait for the file to finish processing
while (file.status == "processing"):
    time.sleep(2)
    file = client.files.retrieve(file.id)
print(f"File processed: {file}")

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "file_id": file.id,
                },
                {
                    "type": "input_text",
                    "text": "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。",
                },
            ]
        }
    ]
)
print(response)
  • Chat API 示例代码：

  1. 上传视频文件获取 File ID。

    Bash
    curl https://ark.cn-beijing.volces.com/api/v3/files \
-H "Authorization: Bearer $ARK_API_KEY" \
-F 'purpose=user_data' \
-F 'file=@/Users/doc/demo.mp4' \
-F 'preprocess_configs[video][model]=doubao-seed-2-1-pro-260915' \
-F 'preprocess_configs[video][fps]=0.3'

  2. 在 Chat API 中引用 File ID。

Bash
curl https://ark.cn-beijing.volces.com/api/v3/chat/completions \
  -H "Authorization: Bearer $ARK_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "video_url",
            "video_url": {
              "file_id": "file-20251018****"
            }
          },
          {
            "type": "text",
            "text": "请你描述下视频中的人物的一系列动作，以 JSON 格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用 HH:mm:ss 表示时间戳。"
          }
        ]
      }
    ]
  }'
Python
import os
from pathlib import Path

from arkruntime import Ark

client = Ark(
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    api_key=os.environ.get("ARK_API_KEY"),
)
file_path = Path(os.getenv("FILE_PATH", "/Users/doc/demo.mp4"))

uploaded_file = client.files.create(
    file=file_path,
    purpose="user_data",
    preprocess_configs={"video": {"model": "doubao-seed-2-1-pro-260915", "fps": 0.3}},
)
client.files.wait_for_processing(uploaded_file.id)

completion = client.chat.completions.create(
    model=os.getenv("MODEL_ID", "doubao-seed-2-1-pro-260915"),
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "video_url", "video_url": {"file_id": uploaded_file.id}},
                {"type": "text", "text": "请你描述下视频中的人物的一系列动作，以 JSON 格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用 HH:mm:ss 表示时间戳。"},
            ],
        }
    ],
)
print(f"id={completion.id}")
print(completion.choices[0].message.content)
Go
package main

import (
	"context"
	"fmt"
	"os"

	"github.com/volcengine/ark-runtime-go/arkruntime"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/chat"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/file"
)

func main() {
	ctx := context.Background()
    client := arkruntime.NewClientWithApiKey(
		os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
	)
	filePath := os.Getenv("FILE_PATH")
	if filePath == "" {
		filePath = "/Users/doc/demo.mp4"
	}
	data, err := os.Open(filePath)
	if err != nil {
		panic(err)
	}
	defer data.Close()

	uploaded, err := client.UploadFile(ctx, &file.FileCreateRequest{
		Purpose: file.PurposeUserData,
		PreprocessConfigs: file.NewOptPreprocessConfigs(file.PreprocessConfigs{
			Video: file.NewOptVideo(file.Video{
				Model: file.NewOptString("doubao-seed-2-1-pro-260915"),
				Fps:   file.NewOptFloat64(0.3),
			}),
		}),
	}, data)
	if err != nil {
		panic(err)
	}
	uploaded, err = client.WaitForFileProcessing(
		ctx,
		uploaded.ID,
		arkruntime.WaitForFileProcessingOptions{},
	)
	if err != nil || uploaded.Status != file.StatusActive {
		panic(fmt.Sprintf("file processing failed: %v, status=%s", err, uploaded.Status))
	}

	media := chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartVideoChatCompletionContentPartSum(chat.ChatCompletionContentPartVideo{Type: chat.ChatCompletionContentPartVideoTypeVideoURL, VideoURL: chat.ChatCompletionContentPartVideoVideoUrl{FileID: chat.NewOptString(uploaded.ID)}})}
	text := chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartTextChatCompletionContentPartSum(
		chat.ChatCompletionContentPartText{Type: chat.ChatCompletionContentPartTextTypeText, Text: "请你描述下视频中的人物的一系列动作，以 JSON 格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用 HH:mm:ss 表示时间戳。"},
	)}
	message := chat.ChatCompletionRequestMessage{OneOf: chat.NewChatCompletionRequestUserMessageChatCompletionRequestMessageSum(
		chat.ChatCompletionRequestUserMessage{
			Role: chat.ChatCompletionRequestUserMessageRoleUser,
			Content: chat.NewOptNilChatCompletionMessageContent(chat.NewChatCompletionContentPartArrayChatCompletionMessageContent(
				[]chat.ChatCompletionContentPart{media, text},
			)),
		},
	)}
	modelID := os.Getenv("MODEL_ID")
	if modelID == "" {
		modelID = "doubao-seed-2-1-pro-260915"
	}
	response, err := client.CreateChatCompletion(ctx, &chat.ChatCompletionRequest{
		Model:    modelID,
		Messages: []chat.ChatCompletionRequestMessage{message},
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("id=%s\n%s\n", response.ID, response.Choices[0].Message.Content)
}
Java
import com.volcengine.ark.runtime.models.chat.*;
import com.volcengine.ark.runtime.models.file.FileCreateRequest;
import com.volcengine.ark.runtime.models.file.FileObject;
import com.volcengine.ark.runtime.models.file.PreprocessConfigs;
import com.volcengine.ark.runtime.models.file.Purpose;
import com.volcengine.ark.runtime.models.file.Status;
import com.volcengine.ark.runtime.models.file.Video;
import com.volcengine.ark.runtime.service.ArkService;

import java.io.File;
import java.util.Arrays;

public class Main {
    public static void main(String[] args) {
        ArkService service = ArkService.builder()
                .apiKey(System.getenv("ARK_API_KEY"))
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();
        String filePath = System.getenv().getOrDefault("FILE_PATH", "/Users/doc/demo.mp4");
        String modelId = System.getenv().getOrDefault("MODEL_ID", "doubao-seed-2-1-pro-260915");

        FileObject uploaded = service.uploadFile(
                FileCreateRequest.builder()
                        .purpose(Purpose.USER_DATA)
                        .preprocessConfigs(PreprocessConfigs.builder().video(Video.builder().model("doubao-seed-2-1-pro-260915").fps(0.3).build()).build())
                        .build(),
                new File(filePath)
        );
        uploaded = service.waitForFileProcessing(uploaded.getId());
        if (uploaded.getStatus() != Status.ACTIVE) {
            throw new IllegalStateException("File processing failed: " + uploaded.getStatus());
        }

        ChatCompletionContentPart media = ChatCompletionContentPartVideo.builder().type(ChatCompletionContentPartType.VIDEO_URL).videoUrl(ChatCompletionContentPartVideoVideoUrl.builder().fileId(uploaded.getId()).build()).build();
        ChatCompletionContentPart text = ChatCompletionContentPartText.builder()
                .type(ChatCompletionContentPartType.TEXT)
                .text("请你描述下视频中的人物的一系列动作，以 JSON 格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用 HH:mm:ss 表示时间戳。")
                .build();
        ChatCompletionRequestMessage message = ChatCompletionRequestUserMessage.builder()
                .role(ChatCompletionRequestMessageType.USER)
                .content(ChatCompletionMessageContent.builder()
                        .value(Arrays.asList(media, text))
                        .build())
                .build();
        ChatCompletionRequest request = ChatCompletionRequest.builder()
                .model(modelId)
                .messages(Arrays.asList(message))
                .build();
        ChatCompletionResponse response = service.createChatCompletion(request);
        System.out.println("id=" + response.getId());
        System.out.println(response.getChoices().get(0).getMessage().getContent());
        service.shutdownExecutor();
    }
}

传 TOS 参数时，文件存储至用户指定的火山引擎对象存储 TOS Bucket，支持最大 2 GB 的视频文件上传。示例代码如下：

  • Responses API 示例代码：
Bash
curl -X POST "https://ark.cn-beijing.volces.com/api/v3/files" \
  -H "Authorization: Bearer $ARK_API_KEY" \
  -F "purpose=user_data" \
  -F "url=tos://my-bucket/videos/long-video.mp4" \
  -F "tos[bucket]=my-bucket" \
  -F "tos[prefix]=ark-files/" \
  -F "preprocess_configs[video][model]=doubao-seed-2-1-pro-260915" \
  -F "preprocess_configs[video][max_video_tokens]=200000" \
  -F "preprocess_configs[video][min_frames]=16"
Python
import os
from arkruntime import Ark

client = Ark(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=os.getenv('ARK_API_KEY')
)

prompt = "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"

file = client.files.create(
    url="tos://my-bucket/videos/long-video.mp4",
    purpose="user_data",
    tos={
        "bucket": "my-bucket",
        "prefix": "ark-files/"
    },
    preprocess_configs={
        "video": {
            "model": "doubao-seed-2-1-pro-260915",
            "max_video_tokens": 200000,
            "min_frames": 16
        }
    }
)
print(file)

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "file_id": file.id
                },
                {
                    "type": "input_text",
                    "text": prompt
                }
            ]
        }
    ]
)
print(response)
Go
package main

import (
    "context"
    "fmt"
    "os"

    "github.com/volcengine/volcengine-go-sdk/service/arkruntime"
    "github.com/volcengine/volcengine-go-sdk/service/arkruntime/model/file"
    "github.com/volcengine/volcengine-go-sdk/volcengine"
)

func main() {
    client := arkruntime.NewClientWithApiKey(
        os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
    )
    ctx := context.Background()

    fileInfo, err := client.UploadFile(ctx, &file.UploadFileRequest{
        Purpose: file.PurposeUserData,
        URL:     volcengine.String("tos://my-bucket/videos/long-video.mp4"),
        Tos: &file.TosStorage{
            Bucket: volcengine.String("my-bucket"),
            Prefix: volcengine.String("ark-files/"),
        },
        PreprocessConfigs: &file.PreprocessConfigs{
            Video: &file.Video{
                Model:          volcengine.String("doubao-seed-2-1-pro-260915"),
                MaxVideoTokens: volcengine.Int64(200000),
                MinFrames:      volcengine.Int64(16),
            },
        },
    })
    if err != nil {
        fmt.Printf("upload file error: %v", err)
        return
    }
    fmt.Printf("file info: %v\n", fileInfo)
}
Java
package com.ark.sample;

import com.volcengine.ark.runtime.model.files.*;
import com.volcengine.ark.runtime.service.ArkService;

public class demo {

    public static void main(String[] args) {
        String apiKey = System.getenv("ARK_API_KEY");
        ArkService service = ArkService.builder()
                .apiKey(apiKey)
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();

        System.out.println("===== Upload File Example =====");
        FileMeta fileMeta;
        try {
            fileMeta = service.uploadFile(
                    UploadFileRequest.builder()
                            .url("tos://my-bucket/videos/long-video.mp4")
                            .purpose("user_data")
                            .tos(TosStorage.builder()
                                    .bucket("my-bucket")
                                    .prefix("ark-files/")
                                    .build())
                            .preprocessConfigs(PreprocessConfigs.builder()
                                    .video(Video.builder()
                                            .model("doubao-seed-2-1-pro-260915")
                                            .maxVideoTokens(200000L)
                                            .minFrames(16L)
                                            .build())
                                    .build())
                            .build());
            System.out.println("Uploaded file Meta: " + fileMeta);
        } catch (Exception e) {
            e.printStackTrace();
        }

        service.shutdownExecutor();
    }
}
  • Chat API 示例代码：

  1. 上传视频文件获取 File ID。

    Bash
    curl -X POST "https://ark.cn-beijing.volces.com/api/v3/files" \
-H "Authorization: Bearer $ARK_API_KEY" \
-F "purpose=user_data" \
-F "url=tos://<your_bucket>/videos/long-video.mp4" \
-F "tos[bucket]=<your_bucket>" \
-F "tos[prefix]=ark-files/" \
-F "preprocess_configs[video][model]=doubao-seed-2-1-pro-260915" \
-F "preprocess_configs[video][max_video_tokens]=200000" \
-F "preprocess_configs[video][min_frames]=16"

  2. 在 Chat API 中引用 File ID。

Bash
curl https://ark.cn-beijing.volces.com/api/v3/chat/completions \
  -H "Authorization: Bearer $ARK_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "video_url",
            "video_url": {
              "file_id": "file-20251018****"
            }
          },
          {
            "type": "text",
            "text": "请你描述下视频中的人物的一系列动作，以 JSON 格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用 HH:mm:ss 表示时间戳。"
          }
        ]
      }
    ]
  }'
Python
import os

from arkruntime import Ark

client = Ark(
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    api_key=os.getenv("ARK_API_KEY"),
)

prompt = "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"

file = client.files.create(
    url="tos://my-bucket/videos/long-video.mp4",
    purpose="user_data",
    tos={
        "bucket": "my-bucket",
        "prefix": "ark-files/",
    },
    preprocess_configs={
        "video": {
            "model": "doubao-seed-2-1-pro-260915",
            "max_video_tokens": 200000,
            "min_frames": 16,
        }
    },
)
print(file)

completion = client.chat.completions.create(
    model="doubao-seed-2-1-pro-260915",
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "video_url",
                    "video_url": {
                        "file_id": file.id,
                    },
                },
                {
                    "type": "text",
                    "text": prompt,
                },
            ],
        }
    ],
)
print(completion.choices[0].message.content)

Base64 编码传入#

将本地文件转换为 Base64 编码字符串，然后提交给大模型。该方式适用于视频文件体积较小的情况，文件不能超过 50 MB，请求体不能超过 64 MB。（Responses API 和 Chat API 都支持该方式。）

  • 使用 Responses API 的示例代码如下：
Bash
BASE64_FILE=$(base64 < demo.mp4) && curl https://ark.cn-beijing.volces.com/api/v3/responses \
   -H "Content-Type: application/json"  \
   -H "Authorization: Bearer $ARK_API_KEY"  \
   -d @- <<EOF
   {
    "model": "doubao-seed-2-1-pro-260915",
    "input": [
      {
        "role": "user",
        "content": [
          {
            "type": "input_video",
            "video_url": "data:video/mp4;base64,$BASE64_FILE",
            "fps": 1
          }
        ]
      }
    ]
  }
EOF
Python
import os
from arkruntime import Ark
import base64
api_key = os.getenv('ARK_API_KEY')

client = Ark(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)
# Convert local files to Base64-encoded strings.
def encode_file(file_path):
  with open(file_path, "rb") as read_file:
    return base64.b64encode(read_file.read()).decode('utf-8')
base64_file = encode_file("/Users/doc/demo.mp4")

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "video_url": f"data:video/mp4;base64,{base64_file}",
                    "fps":1
                }
            ],
        }
    ]
)

print(response)
Go
package main

import (
    "context"
    "encoding/base64"
    "fmt"
    "os"

    "github.com/volcengine/ark-runtime-go/arkruntime"
    "github.com/volcengine/ark-runtime-go/arkruntime/model/responses"
)

func main() {
    // Convert local files to Base64-encoded strings.
    fileBytes, err := os.ReadFile("/Users/doc/demo.mp4")
    if err != nil {
        fmt.Printf("read file error: %v\n", err)
        return
    }
    base64File := base64.StdEncoding.EncodeToString(fileBytes)

    client := arkruntime.NewClientWithApiKey(
        os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
    )
    ctx := context.Background()

    inputMessage := responses.InputItem{
        OneOf: responses.NewItemEasyMessageInputItemSum(
            responses.ItemEasyMessage{
                Role: responses.NewOptMessageRole(responses.MessageRoleUser),
                Content: responses.NewContentItemArrayMessageContent(
                    []responses.ContentItem{
            responses.ContentItem{OneOf: responses.NewContentItemVideoContentItemSum(responses.ContentItemVideo{Type: responses.ContentItemVideoTypeInputVideo, VideoURL: fmt.Sprintf("data:video/mp4;base64,%s", base64File), Fps: responses.NewOptFloat64(1)})},
                    },
                ),
            },
        ),
    }

    resp, err := client.CreateResponses(ctx, &responses.ResponsesRequest{
        Model: "doubao-seed-2-1-pro-260915",
        Input: responses.NewInputItemArrayResponsesInput([]responses.InputItem{inputMessage}),
    })
    if err != nil {
        fmt.Printf("response error: %v\n", err)
        return
    }
    fmt.Println(resp)
}
Java
package com.ark.sample;
import com.volcengine.ark.runtime.service.ArkService;
import com.volcengine.ark.runtime.models.responses.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Base64;
import java.io.IOException;

public class demo {
    private static String encodeFile(String filePath) throws IOException {
        byte[] fileBytes = Files.readAllBytes(Paths.get(filePath));
        return Base64.getEncoder().encodeToString(fileBytes);
    }
    public static void main(String[] args) {
        String apiKey = System.getenv("ARK_API_KEY");
        ArkService arkService = ArkService.builder().apiKey(apiKey)
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();
        // Convert local files to Base64-encoded strings.
        String base64Data = "";
        try {
            base64Data = "data:video/mp4;base64," + encodeFile("/Users/demo.mp4");
        } catch (IOException e) {
            System.err.println("编码失败: " + e.getMessage());
        }
        ResponsesRequest request = ResponsesRequest.builder()
                .model("doubao-seed-2-1-pro-260915")
                .input(ResponsesInput.ofList(java.util.Arrays.asList(ItemEasyMessage.builder().role(MessageRole.USER).content(
                                MessageContent.ofList(java.util.Arrays.asList(ContentItemVideo.builder().videoUrl(base64Data).fps(1D).build()))
                        ).build())))
                .build();
        Response resp = arkService.createResponse(request);
        System.out.println(resp);

        arkService.shutdownExecutor();
    }
}
Python
import os
from openai import OpenAI
import base64

api_key = os.getenv('ARK_API_KEY')

client = OpenAI(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)
# Convert local files to Base64-encoded strings.
def encode_file(file_path):
  with open(file_path, "rb") as read_file:
    return base64.b64encode(read_file.read()).decode('utf-8')
base64_file = encode_file("/Users/doc/demo.mp4")

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "video_url": f"data:video/mp4;base64,{base64_file}",
                    "fps":1
                }
            ],
        }
    ]
)

print(response)
  • 使用 Chat API 的示例代码如下：
Bash
BASE64_VIDEO=$(base64 < demo.mp4) && curl https://ark.cn-beijing.volces.com/api/v3/chat/completions \
   -H "Content-Type: application/json"  \
   -H "Authorization: Bearer $ARK_API_KEY"  \
   -d @- <<EOF
   {
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "video_url",
            "video_url": {
              "url": "data:video/mp4;base64,$BASE64_VIDEO"
            }
          },
          {
            "type": "text",
            "text": "What is in the video?"
          }
        ]
      }
    ],
    "max_tokens": 300
  }
EOF
  • 按需替换 Model ID，查询 Model ID 参见 模型列表。
Python
import base64
import os
# Install SDK: python -m pip install --upgrade arkruntime
from arkruntime import Ark

client = Ark(
    # The base URL for model invocation .
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    # Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
    api_key=os.getenv('ARK_API_KEY'),
)

# 定义方法将指定路径视频转为Base64编码
def encode_video(video_path):
  with open(video_path, "rb") as video_file:
    return base64.b64encode(video_file.read()).decode('utf-8')

# 需传给大模型的视频
video_path = "demo.mp4"

# 将视频转为Base64编码
base64_video = encode_video(video_path)

completion = client.chat.completions.create(
  # Replace with Model ID .
  model = "doubao-seed-2-1-pro-260915",
  messages=[
    {
      "role": "user",
      "content": [
        {
          "type": "video_url",
          "video_url": {
            "url":  f"data:video/<VIDEO_FORMAT>;base64,{base64_video}"
          },
        },
        {
          "type": "text",
          "text": "What's in the video?",
        },
      ],
    }
  ],
)

print(completion.choices[0])
Go
package main

import (
    "context"
    "encoding/base64"
    "fmt"
    "os"
    "github.com/volcengine/ark-runtime-go/arkruntime"
    "github.com/volcengine/ark-runtime-go/arkruntime/model/chat"
)

func main() {
    // 读取本地视频文件
    videoBytes, err := os.ReadFile("demo.mp4") // 替换为实际视频路径
    if err != nil {
        fmt.Printf("读取视频失败: %v\n", err)
        return
    }
    base64Video := base64.StdEncoding.EncodeToString(videoBytes)

    client := arkruntime.NewClientWithApiKey(
        // Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
        os.Getenv("ARK_API_KEY"),
        // The base URL for model invocation .
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
        )
    ctx := context.Background()
    req := &chat.ChatCompletionRequest{
        // Replace with Model ID
        Model: "doubao-seed-2-1-pro-260915",
        Messages: []chat.ChatCompletionRequestMessage{
chat.ChatCompletionRequestMessage{OneOf: chat.NewChatCompletionRequestUserMessageChatCompletionRequestMessageSum(chat.ChatCompletionRequestUserMessage{Role: chat.ChatCompletionRequestUserMessageRoleUser, Content: chat.NewOptNilChatCompletionMessageContent(chat.NewChatCompletionContentPartArrayChatCompletionMessageContent(
                []chat.ChatCompletionContentPart{
                    chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartVideoChatCompletionContentPartSum(chat.ChatCompletionContentPartVideo{Type: chat.ChatCompletionContentPartVideoTypeVideoURL, VideoURL: chat.ChatCompletionContentPartVideoVideoUrl{URL: chat.NewOptString(fmt.Sprintf("data:video/mp4;base64,%s", base64Video))}})},
                    chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartTextChatCompletionContentPartSum(chat.ChatCompletionContentPartText{Type: chat.ChatCompletionContentPartTextTypeText, Text: "What's in the video?"})},
                },
            ))})},
        },
    }

    resp, err := client.CreateChatCompletion(ctx, req)
    if err != nil {
        fmt.Printf("standard chat error: %v\n", err)
        return
    }
    fmt.Println(resp.Choices[0].Message.Content)
}
Java
package com.ark.sample;

import com.volcengine.ark.runtime.models.chat.*;
import com.volcengine.ark.runtime.models.chat.ChatCompletionContentPart.*;
import com.volcengine.ark.runtime.service.ArkService;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.io.IOException;

public class Sample {
    static String apiKey = System.getenv("ARK_API_KEY");
    static ConnectionPool connectionPool = new ConnectionPool(5, 1, TimeUnit.SECONDS);
    static Dispatcher dispatcher = new Dispatcher();
    static ArkService service = ArkService.builder()
        .dispatcher(dispatcher)
        .connectionPool(connectionPool)
        .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
        // The base URL for model invocation
        .apiKey(apiKey) //Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
        .build();

    // Base64编码方法
    private static String encodeVideo(String videoPath) throws IOException {
        byte[] videoBytes = Files.readAllBytes(java.nio.file.Paths.get(videoPath));
        return Base64.getEncoder().encodeToString(videoBytes);
    }

    public static void main(String[] args) throws Exception {

        List<ChatCompletionRequestMessage> messagesForReqList = new ArrayList<>();

        // 本地视频路径（替换为实际路径）
        String videoPath = "demo.mp4";

        // 生成Base64数据URL
        String base64Data = "data:video/mp4;base64," + encodeVideo(videoPath);

        // 构建消息内容（修复内容部分构建方式）
        List<ChatCompletionContentPart> contentParts = new ArrayList<>();

        // 图片部分使用builder模式
        contentParts.add(ChatCompletionContentPartVideo.builder()
                .type(ChatCompletionContentPartType.VIDEO_URL)
                .videoUrl(ChatCompletionContentPartVideoVideoUrl.builder().url(base64Data).fps(2.0).build())
                .build());

        // 文本部分使用builder模式
        contentParts.add(ChatCompletionContentPartText.builder()
                .type(ChatCompletionContentPartType.TEXT)
                .text("What's in the video?")
                .build());

        // 创建消息
        messagesForReqList.add(ChatCompletionRequestUserMessage.builder()
                .role(ChatCompletionRequestMessageType.USER)
                .content(ChatCompletionMessageContent.ofList(contentParts))
                .build());

        ChatCompletionRequest req = ChatCompletionRequest.builder()
                .model("doubao-seed-2-1-pro-260915") //Replace with Model ID .
                .messages(messagesForReqList)
                .build();

        service.createChatCompletion(req)
                .getChoices()
                .forEach(choice -> System.out.println(choice.getMessage().getContent()));
        // shutdown service after all requests are finished
        service.shutdownExecutor();
    }
}

视频 URL 传入#

如果视频文件已存在公网可访问URL，可以在请求中直接填入视频文件的公网URL，文件不能超过50 MB。（Responses API 和 Chat API 都支持该方式。）

  • 使用 Responses API 的示例代码如下：
Bash
curl https://ark.cn-beijing.volces.com/api/v3/responses \
-H "Authorization: Bearer $ARK_API_KEY" \
-H 'Content-Type: application/json' \
-d '{
    "model": "doubao-seed-2-1-pro-260915",
    "input": [
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "video_url": "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4",
                    "fps":1
                }
            ]
        }
    ]
}'
Python
import os
from arkruntime import Ark

# 从环境变量中获取您的API KEY，配置方法见：https://ark.volcengine.com/region:cn-beijing/docs/quick-start
api_key = os.getenv('ARK_API_KEY')

client = Ark(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "video_url": "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4",
                    "fps":1
                }
            ],
        }
    ]
)

print(response)
Go
package main

import (
    "context"
    "fmt"
    "os"

    "github.com/volcengine/ark-runtime-go/arkruntime"
    "github.com/volcengine/ark-runtime-go/arkruntime/model/responses"
)

func main() {
    client := arkruntime.NewClientWithApiKey(
        //通过 os.Getenv 从环境变量中获取 ARK_API_KEY
        os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
    )
    // 创建一个上下文，通常用于传递请求的上下文信息，如超时、取消等
    ctx := context.Background()

    inputMessage := responses.InputItem{
        OneOf: responses.NewItemEasyMessageInputItemSum(
            responses.ItemEasyMessage{
                Role: responses.NewOptMessageRole(responses.MessageRoleUser),
                Content: responses.NewContentItemArrayMessageContent(
                    []responses.ContentItem{
            responses.ContentItem{OneOf: responses.NewContentItemVideoContentItemSum(responses.ContentItemVideo{Type: responses.ContentItemVideoTypeInputVideo, VideoURL: "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4", Fps: responses.NewOptFloat64(1)})},
                    },
                ),
            },
        ),
    }

    resp, err := client.CreateResponses(ctx, &responses.ResponsesRequest{
        Model: "doubao-seed-2-1-pro-260915",
        Input: responses.NewInputItemArrayResponsesInput([]responses.InputItem{inputMessage}),
    })
    if err != nil {
        fmt.Printf("response error: %v\n", err)
        return
    }
    fmt.Println(resp)
}
Java
package com.ark.example;
import com.volcengine.ark.runtime.service.ArkService;
import com.volcengine.ark.runtime.models.responses.*;
public class demo {
    public static void main(String[] args) {
        String apiKey = System.getenv("ARK_API_KEY");
        // 创建ArkService实例
        ArkService arkService = ArkService.builder().apiKey(apiKey)
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();

        ResponsesRequest request = ResponsesRequest.builder()
                .model("doubao-seed-2-1-pro-260915")
                .input(ResponsesInput.ofList(java.util.Arrays.asList(ItemEasyMessage.builder().role(MessageRole.USER).content(
                                MessageContent.ofList(java.util.Arrays.asList(ContentItemVideo.builder().videoUrl("https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4").fps(1D).build()))
                        ).build())))
                .build();
        Response resp = arkService.createResponse(request);
        System.out.println(resp);

        arkService.shutdownExecutor();
    }
}
Python
import os
from openai import OpenAI

# 从环境变量中获取您的API KEY，配置方法见：https://ark.volcengine.com/region:cn-beijing/docs/quick-start
api_key = os.getenv('ARK_API_KEY')

client = OpenAI(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "video_url": "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4",
                    "fps":1
                }
            ],
        }
    ]
)

print(response)
  • 使用 Chat API 的示例代码如下：（可参见[控制视频理解的精细度](/docs/video-understanding#bf4d9224)）
Python
import os
# Install SDK: python -m pip install --upgrade arkruntime
from arkruntime import Ark

client = Ark(
    # The base URL for model invocation
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    # Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
    api_key=os.getenv('ARK_API_KEY'),
)

completion = client.chat.completions.create(
    # Replace with Model ID .
    model = "doubao-seed-2-1-pro-260915",
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "video_url",
                    "video_url": {
                        # Replace the link with your actual video link
                        "url":  "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4",
                        "fps": 1
                    }
                },
            ],
        }
    ],
)

print(completion.choices[0].message.content)

使用场景#

控制视频理解的精细度#

您可通过 fps 字段，控制从视频中抽取图像的频率，默认为1，即每秒从视频中抽取一帧图像，输入给模型进行视觉理解。 可通过 fps 字段，控制模型对于视频中图像变化的敏感度。

  • 当视频画面变化剧烈或需关注画面变化，如计算视频中角色动作次数，可调高 fps 设置（最高 5），防止抽帧频率低导致误判。
  • 当视频画面变化不频繁或无需关注画面变化，如画面中人数，可调低 fps （最低0.2），可提升处理速度，节省 token 用量。

示例代码如下：

Bash
curl https://ark.cn-beijing.volces.com/api/v3/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $ARK_API_KEY" \
   -d '{
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
        {
            "role": "user",
            "content": [
                {"type": "video_url","video_url": {"url":  "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4", "fps": 2}},
                {"type": "text", "text": "What is in the video?"}
            ]
        }
    ],
    "max_tokens": 300
  }'
  • 按需替换 Model ID，查询 Model ID 参见 模型列表。
Python
import os
# Install SDK: python -m pip install --upgrade arkruntime
from arkruntime import Ark

client = Ark(
    # The base URL for model invocation .
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    # Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
    api_key=os.getenv('ARK_API_KEY'),
)

completion = client.chat.completions.create(
    # Replace with Model ID .
    model = "doubao-seed-2-1-pro-260915",
    messages=[
        {
            # 消息角色为用户
            "role": "user",
            "content": [
                {
                    "type": "video_url",
                    "video_url": {
                        # 替换链接为实际视频链接
                        "url":  "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4",
                        "fps": 2, # 每秒截取2帧画面，用于视频理解
                    }
                },
                # 文本类型的消息内容，询问视频里有什么
                {"type": "text", "text": "What's in the video?"},
            ],
        }
    ],
)

print(completion.choices[0])
Go
package main

import (
    "context"
    "fmt"
    "os"
    "github.com/volcengine/ark-runtime-go/arkruntime"
    "github.com/volcengine/ark-runtime-go/arkruntime/model/chat"
)

func main() {
    client := arkruntime.NewClientWithApiKey(
        //Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
        os.Getenv("ARK_API_KEY"),
        // The base URL for model invocation .
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
    )
    // 创建一个上下文，通常用于传递请求的上下文信息，如超时、取消等
    ctx := context.Background()
    // 构建消息内容
    contentParts := []chat.ChatCompletionContentPart{
        chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartVideoChatCompletionContentPartSum(chat.ChatCompletionContentPartVideo{Type: chat.ChatCompletionContentPartVideoTypeVideoURL, VideoURL: chat.ChatCompletionContentPartVideoVideoUrl{URL: chat.NewOptString("https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4"), Fps: chat.NewOptFloat64(2)}})},
        chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartTextChatCompletionContentPartSum(chat.ChatCompletionContentPartText{Type: chat.ChatCompletionContentPartTextTypeText, Text: "What's in the video?"})},
    }
    // 构建聊天完成请求，设置请求的模型和消息内容
    req := &chat.ChatCompletionRequest{
        // Replace with Model ID
       Model: "doubao-seed-2-1-pro-260915",
       Messages: []chat.ChatCompletionRequestMessage{
chat.ChatCompletionRequestMessage{OneOf: chat.NewChatCompletionRequestUserMessageChatCompletionRequestMessageSum(chat.ChatCompletionRequestUserMessage{Role: chat.ChatCompletionRequestUserMessageRoleUser, Content: chat.NewOptNilChatCompletionMessageContent(chat.NewChatCompletionContentPartArrayChatCompletionMessageContent(contentParts))})},
       },
       MaxTokens: chat.NewOptInt32(int32(300)), // 设置模型输出最大 token 数
    }

    // 发送聊天完成请求，并将结果存储在 resp 中，将可能出现的错误存储在 err 中
    resp, err := client.CreateChatCompletion(ctx, req)
    if err!= nil {
       // 若出现错误，打印错误信息并终止程序
       fmt.Printf("standard chat error: %v\n", err)
       return
    }
    // 打印聊天完成请求的响应结果
    fmt.Println(resp.Choices[0].Message.Content)
}
Java
package com.ark.sample;

import com.volcengine.ark.runtime.models.chat.*;
import com.volcengine.ark.runtime.models.chat.ChatCompletionContentPart.*;
import com.volcengine.ark.runtime.service.ArkService;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;

public class VideoSample {
  static String apiKey = System.getenv("ARK_API_KEY");
  static ConnectionPool connectionPool = new ConnectionPool(5, 1, TimeUnit.SECONDS);
  static Dispatcher dispatcher = new Dispatcher();
  static ArkService service = ArkService.builder()
      .dispatcher(dispatcher)
      .connectionPool(connectionPool)
      .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
      // The base URL for model invocation .
      .apiKey(apiKey) //Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
      .build();

  public static void main(String[] args) throws Exception {

    List<ChatCompletionRequestMessage> messagesForReqList = new ArrayList<>();

    // 构建消息内容
    List<ChatCompletionContentPart> contentParts = new ArrayList<>();

    contentParts.add(ChatCompletionContentPartVideo.builder()
        .type(ChatCompletionContentPartType.VIDEO_URL)
        .videoUrl(ChatCompletionContentPartVideoVideoUrl.builder().url("https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/ark_vlm_video_input.mp4").fps(2.0).build())


        .build());

    contentParts.add(ChatCompletionContentPartText.builder()
        .type(ChatCompletionContentPartType.TEXT)
        .text("What's in the video?")
        .build());

    // 创建消息
    messagesForReqList.add(ChatCompletionRequestUserMessage.builder()
        .role(ChatCompletionRequestMessageType.USER)
        .content(ChatCompletionMessageContent.ofList(contentParts))
        .build());

    ChatCompletionRequest req = ChatCompletionRequest.builder()
        .model("doubao-seed-2-1-pro-260915") //Replace with Model ID .
        .messages(messagesForReqList)
        .maxTokens(300)
        .build();

    service.createChatCompletion(req)
        .getChoices()
        .forEach(choice -> System.out.println(choice.getMessage().getContent()));
    // shutdown service after all requests are finished
    service.shutdownExecutor();
  }
}

感知视频时序#

视频理解可理解视频时间和图像关系信息，如回答事件发生什么时间点，在哪些时间发生了某事件等和时间相关的信息，原理见 视频理解工作原理。
下面是简单示例代码

Bash
curl https://ark.cn-beijing.volces.com/api/v3/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $ARK_API_KEY" \
   -d '{
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
        {
            "role": "user",
            "content": [
                {"type": "video_url","video_url": {"url":  "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/video-understanding.mp4", "fps": 5}},
                {"type": "text", "text": "裁判什么时间点出现的？"}
            ]
        }
    ],
    "max_tokens": 300
  }'
  • 按需替换 Model ID，查询 Model ID 参见 模型列表。
Python
import os
# Install SDK: python -m pip install --upgrade arkruntime
from arkruntime import Ark

client = Ark(
    # The base URL for model invocation .
    base_url="https://ark.cn-beijing.volces.com/api/v3",
    # Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
    api_key=os.getenv('ARK_API_KEY'),
)

completion = client.chat.completions.create(
    # Replace with Model ID .
    model = "doubao-seed-2-1-pro-260915",
    messages=[
        {
            # 消息角色为用户
            "role": "user",
            "content": [
                {
                    "type": "video_url",
                    "video_url": {
                        # 替换链接为实际视频链接
                        "url":  "https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/video-understanding.mp4",
                        "fps": 5, # 每秒截取5帧画面，用于视频理解
                    }
                },
                # 文本类型的消息内容，询问视频里有什么
                {"type": "text", "text": "裁判什么时间点出现的？"},
            ],
        }
    ],
)

print(completion.choices[0])
Go
package main

import (
    "context"
    "fmt"
    "os"
    "github.com/volcengine/ark-runtime-go/arkruntime"
    "github.com/volcengine/ark-runtime-go/arkruntime/model/chat"
)

func main() {
    client := arkruntime.NewClientWithApiKey(
        // Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
        os.Getenv("ARK_API_KEY"),
        // The base URL for model invocation .
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
    )
    // 创建一个上下文，通常用于传递请求的上下文信息，如超时、取消等
    ctx := context.Background()
    // 构建消息内容
    contentParts := []chat.ChatCompletionContentPart{
        chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartVideoChatCompletionContentPartSum(chat.ChatCompletionContentPartVideo{Type: chat.ChatCompletionContentPartVideoTypeVideoURL, VideoURL: chat.ChatCompletionContentPartVideoVideoUrl{URL: chat.NewOptString("https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/video-understanding.mp4"), Fps: chat.NewOptFloat64(5)}})},
        chat.ChatCompletionContentPart{OneOf: chat.NewChatCompletionContentPartTextChatCompletionContentPartSum(chat.ChatCompletionContentPartText{Type: chat.ChatCompletionContentPartTextTypeText, Text: "裁判什么时间点出现的？"})},
    }
    // 构建聊天完成请求，设置请求的模型和消息内容
    req := &chat.ChatCompletionRequest{
        // Replace with Model ID
       Model: "doubao-seed-2-1-pro-260915",
       Messages: []chat.ChatCompletionRequestMessage{
chat.ChatCompletionRequestMessage{OneOf: chat.NewChatCompletionRequestUserMessageChatCompletionRequestMessageSum(chat.ChatCompletionRequestUserMessage{Role: chat.ChatCompletionRequestUserMessageRoleUser, Content: chat.NewOptNilChatCompletionMessageContent(chat.NewChatCompletionContentPartArrayChatCompletionMessageContent(contentParts))})},
       },
       MaxTokens: chat.NewOptInt32(int32(300)), // 设置模型输出最大 token 数
    }

    // 发送聊天完成请求，并将结果存储在 resp 中，将可能出现的错误存储在 err 中
    resp, err := client.CreateChatCompletion(ctx, req)
    if err!= nil {
       // 若出现错误，打印错误信息并终止程序
       fmt.Printf("standard chat error: %v\n", err)
       return
    }
    // 打印聊天完成请求的响应结果
    fmt.Println(resp.Choices[0].Message.Content)
}
Java
package com.ark.sample;

import com.volcengine.ark.runtime.models.chat.*;
import com.volcengine.ark.runtime.models.chat.ChatCompletionContentPart.*;
import com.volcengine.ark.runtime.service.ArkService;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;

public class VideoSample {
  static String apiKey = System.getenv("ARK_API_KEY");
  static ConnectionPool connectionPool = new ConnectionPool(5, 1, TimeUnit.SECONDS);
  static Dispatcher dispatcher = new Dispatcher();
  static ArkService service = ArkService.builder()
      .dispatcher(dispatcher)
      .connectionPool(connectionPool)
      .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
      // The base URL for model invocation .
      .apiKey(apiKey) //Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
      .build();

  public static void main(String[] args) throws Exception {

    List<ChatCompletionRequestMessage> messagesForReqList = new ArrayList<>();

    // 构建消息内容
    List<ChatCompletionContentPart> contentParts = new ArrayList<>();

    contentParts.add(ChatCompletionContentPartVideo.builder()
        .type(ChatCompletionContentPartType.VIDEO_URL)
        .videoUrl(ChatCompletionContentPartVideoVideoUrl.builder().url("https://arkdocs.tos-cn-beijing.volces.com/videos/advanced-usage/video-understanding.mp4").fps(5.0).build())

        .build());

    contentParts.add(ChatCompletionContentPartText.builder()
        .type(ChatCompletionContentPartType.TEXT)
        .text("裁判什么时间点出现的？")
        .build());

    // 创建消息
    messagesForReqList.add(ChatCompletionRequestUserMessage.builder()
        .role(ChatCompletionRequestMessageType.USER)
        .content(ChatCompletionMessageContent.ofList(contentParts))
        .build());

    ChatCompletionRequest req = ChatCompletionRequest.builder()
        .model("doubao-seed-2-1-pro-260915") //Replace with Model ID .
        .messages(messagesForReqList)
        .maxTokens(300)
        .build();

    service.createChatCompletion(req)
        .getChoices()
        .forEach(choice -> System.out.println(choice.getMessage().getContent()));
    // shutdown service after all requests are finished
    service.shutdownExecutor();
  }
}

回复预览

Plaintext
根据视频描述，裁判在**3.7秒**左右出现。此时，画面中两位拳击手（左为黑T恤红短裤、右为白T恤黑短裤）原本处于对峙状态，随后裁判（穿着黑色西装、戴白手套）站到两人中间，似乎在准备开始比赛或暂停当前回合，观众仍在背景中欢呼。

流式输出#

流式输出支持内容动态实时呈现，既能够缓解用户等待焦虑，又可以规避复杂任务因长时间推理引发的客户端超时失败问题，保障请求流程顺畅。

Python
import asyncio
import os
from arkruntime import AsyncArk
from arkruntime.types.responses.response_completed_event import ResponseCompletedEvent
from arkruntime.types.responses.response_reasoning_summary_text_delta_event import ResponseReasoningSummaryTextDeltaEvent
from arkruntime.types.responses.response_output_item_added_event import ResponseOutputItemAddedEvent
from arkruntime.types.responses.response_text_delta_event import ResponseTextDeltaEvent
from arkruntime.types.responses.response_text_done_event import ResponseTextDoneEvent

client = AsyncArk(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=os.getenv('ARK_API_KEY')
)

async def main():
    # upload video file
    print("Upload video file")
    file = await client.files.create(
        # replace with your local video path
        file=open("/Users/doc/demo.mp4", "rb"),
        purpose="user_data",
        preprocess_configs={
            "video": {
                "model": "doubao-seed-2-1-pro-260915",
                "fps": 0.3,  # define the sampling fps of the video, default is 1.0
            }
        }
    )
    print(f"File uploaded: {file.id}")

    # Wait for the file to finish processing
    await client.files.wait_for_processing(file.id)
    print(f"File processed: {file.id}")

    stream = await client.responses.create(
        model="doubao-seed-2-1-pro-260915",
        input=[
            {"role": "user", "content": [
                {
                    "type": "input_video",
                    "file_id": file.id  # ref video file id
                },
                {
                    "type": "input_text",
                    "text": "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"

                }
            ]},
        ],
        caching={
            "type": "enabled",
        },
        store=True,
        stream=True
    )

    async for event in stream:
        if isinstance(event, ResponseReasoningSummaryTextDeltaEvent):
            print(event.delta, end="")
        if isinstance(event, ResponseOutputItemAddedEvent):
            print("\noutPutItem " + event.type + " start:")
        if isinstance(event, ResponseTextDeltaEvent):
            print(event.delta,end="")
        if isinstance(event, ResponseTextDoneEvent):
            print("\noutPutTextDone.")
        if isinstance(event, ResponseCompletedEvent):
            print("Response Completed. Usage = " + event.response.usage.model_dump_json())

if __name__ == "__main__":
    asyncio.run(main())
Go
package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/volcengine/ark-runtime-go/arkruntime"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/file"
	"github.com/volcengine/ark-runtime-go/arkruntime/model/responses"
)

func main() {
    client := arkruntime.NewClientWithApiKey(
		// Get API Key: https://ark.volcengine.com/region:cn-beijing/apikey
		os.Getenv("ARK_API_KEY"),
        arkruntime.WithBaseUrl("https://ark.cn-beijing.volces.com/api/v3"),
	)
	ctx := context.Background()

	fmt.Println("----- upload video data -----")
	data, err := os.Open("/Users/doc/demo.mp4")
	if err != nil {
		fmt.Printf("read file error: %v\n", err)
		return
	}
	fileInfo, err := client.UploadFile(ctx, &file.FileCreateRequest{
		Purpose: file.PurposeUserData,
		PreprocessConfigs: file.NewOptPreprocessConfigs(file.PreprocessConfigs{
			Video: file.NewOptVideo(file.Video{
				Model: file.NewOptString("doubao-seed-2-1-pro-260915"),
				Fps:   file.NewOptFloat64(0.3),
			}),
		}),
	}, data)

	if err != nil {
		fmt.Printf("upload file error: %v", err)
		return
	}

	// Wait for the file to finish processing
	for fileInfo.Status == file.StatusProcessing {
		fmt.Println("Waiting for video to be processed...")
		time.Sleep(2 * time.Second)
		fileInfo, err = client.RetrieveFile(ctx, fileInfo.ID) // update file info
		if err != nil {
			fmt.Printf("get file status error: %v", err)
			return
		}
	}
	fmt.Printf("Video processing completed: %s, status: %s\n", fileInfo.ID, fileInfo.Status)
	inputMessage := responses.InputItem{
		OneOf: responses.NewItemEasyMessageInputItemSum(
			responses.ItemEasyMessage{
				Role: responses.NewOptMessageRole(responses.MessageRoleUser),
				Content: responses.NewContentItemArrayMessageContent(
					[]responses.ContentItem{
						responses.ContentItem{OneOf: responses.NewContentItemVideoContentItemSum(responses.ContentItemVideo{Type: responses.ContentItemVideoTypeInputVideo, VideoURL: "", FileID: responses.NewOptString(fileInfo.ID)})},
						responses.ContentItem{OneOf: responses.NewContentItemTextContentItemSum(responses.ContentItemText{Type: responses.ContentItemTextTypeInputText, Text: "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。"})},
					},
				),
			},
		),
	}
	createResponsesReq := &responses.ResponsesRequest{
		Model: "doubao-seed-2-1-pro-260915",
		Input: responses.NewInputItemArrayResponsesInput([]responses.InputItem{inputMessage}),
	}

	resp, err := client.CreateResponsesStream(ctx, createResponsesReq)
	if err != nil {
		fmt.Printf("stream error: %v\n", err)
		return
	}
	var responseId string
	for {
		event, err := resp.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Printf("stream error: %v\n", err)
			return
		}
		handleEvent(event)
		if event.OneOf.Type == responses.ResponseCreatedEventResponseStreamEventSum {
			responseId = event.OneOf.ResponseCreatedEvent.Response.ID
			fmt.Printf("Response ID: %s", responseId)
		}
	}
}

func handleEvent(event *responses.ResponseStreamEvent) {
	switch event.OneOf.Type {
	case responses.ResponseReasoningSummaryTextDeltaEventResponseStreamEventSum:
		print(event.OneOf.ResponseReasoningSummaryTextDeltaEvent.Delta.Or(""))
	case responses.ResponseReasoningSummaryTextDoneEventResponseStreamEventSum:
		fmt.Printf("\nAggregated reasoning text: %s\n", event.OneOf.ResponseReasoningSummaryTextDoneEvent.Text.Or(""))
	case responses.ResponseTextDeltaEventResponseStreamEventSum:
		print(event.OneOf.ResponseTextDeltaEvent.Delta.Or(""))
	case responses.ResponseTextDoneEventResponseStreamEventSum:
		fmt.Printf("\nAggregated output text: %s\n", event.OneOf.ResponseTextDoneEvent.Text.Or(""))
	}
}
Java
package com.ark.example;

import com.volcengine.ark.runtime.models.file.*;
import com.volcengine.ark.runtime.service.ArkService;
import com.volcengine.ark.runtime.models.responses.*;
import java.io.File;
import java.util.concurrent.TimeUnit;

public class demo {
    public static void main(String[] args) {
        String apiKey = System.getenv("ARK_API_KEY");
        ArkService service = ArkService.builder().apiKey(apiKey)
                .baseUrl("https://ark.cn-beijing.volces.com/api/v3")
                .build();

        System.out.println("===== Upload File Example=====");
        // upload a video for responses
        FileObject fileMeta;
        fileMeta = service.uploadFile(FileCreateRequest.builder() // replace with your video file path
                        .purpose(Purpose.USER_DATA)
                        .preprocessConfigs(PreprocessConfigs.builder().video(Video.builder().model("doubao-seed-2-1-pro-260915").fps(0.3).build()).build())
                        .build(), new File("/Users/doc/demo.mp4"));
        System.out.println("Uploaded file Meta: " + fileMeta);
        System.out.println("status:" + fileMeta.getStatus());

        try {
            while ("processing".equalsIgnoreCase(fileMeta.getStatus().toString())) {
                System.out.println("Waiting for video to be processed...");
                TimeUnit.SECONDS.sleep(2);
                fileMeta = service.retrieveFile(fileMeta.getId());
            }
        } catch (Exception e) {
            System.err.println("get file status error：" + e.getMessage());
        }
        System.out.println("Uploaded file Meta: " + fileMeta);

        ResponsesRequest request = ResponsesRequest.builder()
                .model("doubao-seed-2-1-pro-260915")
                .stream(true)
                .input(ResponsesInput.ofList(java.util.Arrays.asList(ItemEasyMessage.builder().role(MessageRole.USER).content(
                                MessageContent.ofList(java.util.Arrays.asList(ContentItemVideo.builder().fileId(fileMeta.getId()).build(), ContentItemText.builder().text("请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。").build()))
                        ).build())))
                .build();

        service.streamResponse(request)
                .doOnError(Throwable::printStackTrace)
                .blockingForEach(event -> {
                    if (event instanceof ResponseReasoningSummaryTextDeltaEvent) {
                        System.out.print(((ResponseReasoningSummaryTextDeltaEvent) event).getDelta());
                    }
                    if (event instanceof ResponseOutputItemAddedEvent) {
                        System.out.println("\nOutputItem " + (((ResponseOutputItemAddedEvent) event).getItem().getType()) + " Start: ");
                    }
                    if (event instanceof ResponseTextDeltaEvent) {
                        System.out.print(((ResponseTextDeltaEvent) event).getDelta());
                    }
                    if (event instanceof ResponseTextDoneEvent) {
                        System.out.println("\nOutputText End.");
                    }
                    if (event instanceof ResponseOutputItemDoneEvent) {
                        System.out.println("\nOutputItem " + ((ResponseOutputItemDoneEvent) event).getItem().getType() + " End.");
                    }
                    if (event instanceof ResponseFunctionCallArgumentsDoneEvent) {
                        System.out.println("\nFunctionCall Arguments: " + ((ResponseFunctionCallArgumentsDoneEvent) event).getArguments());
                    }
                    if (event instanceof ResponseCompletedEvent) {
                        System.out.println("\nResponse Completed. Usage = " + ((ResponseCompletedEvent) event).getResponse().getUsage());
                    }
                });


        service.shutdownExecutor();
    }
}
Python
import os
import time
from openai import OpenAI

api_key = os.getenv('ARK_API_KEY')

client = OpenAI(
    base_url='https://ark.cn-beijing.volces.com/api/v3',
    api_key=api_key,
)

file = client.files.create(
    file=open("/Users/doc/demo.mp4", "rb"),
    purpose="user_data",
    extra_body={
        "preprocess_configs": {
            "video": {
                "model": "doubao-seed-2-1-pro-260915",
                "fps": 0.3,
            }
        }
    },
)
# Wait for the file to finish processing
while (file.status == "processing"):
    time.sleep(2)
    file = client.files.retrieve(file.id)
print(f"File processed: {file}")

response = client.responses.create(
    model="doubao-seed-2-1-pro-260915",
    input=[
        {
            "role": "user",
            "content": [
                {
                    "type": "input_video",
                    "file_id": file.id,
                },
                {
                    "type": "input_text",
                    "text": "请你描述下视频中的人物的一系列动作，以JSON格式输出开始时间（start_time）、结束时间（end_time）、事件（event）、是否危险（danger），请使用HH:mm:ss表示时间戳。",
                },
            ]
        }
    ],
    stream=True
)


for event in response:
    if event.type == "response.reasoning_summary_text.delta":
        print(event.delta, end="")
    if event.type == "response.output_item.added":
        print("\noutPutItem " + event.type + " start:")
    if event.type == "response.output_text.delta":
        print(event.delta,end="")
    if event.type == "response.output_item.done":
        print("\noutPutTextDone.")
    if event.type == "response.completed":
        print("\nResponse Completed. Usage = " + event.response.usage.model_dump_json())

使用说明#

时序信息#

基于 FPS 的抽帧获取视频关键帧，再通过时间戳+图像拼接标记时序信息，模型基于该请求中的时序标记和图像内容，实现对视频的完整理解（包括内容变化、动作逻辑、时序关联等）。
详细原理见 视频理解工作原理。

视频格式说明#

视频格式 文件扩展名 内容格式 Content Type
MP4 .mp4 video/mp4
AVI .avi video/x-msvideo
MOV .mov video/quicktime

视频文件格式变种较多，不能保证所有文件都能被识别，请通过测试验证文件能够被正常识别。

视频文件容量#

  • 使用 URL 方式传入视频，视频文件不能超过 50MB。
  • 使用 Base64 编码传入视频，视频文件不能超过 50MB，请求体不能超过 64MB。
  • 使用 Files API 上传视频，文件存储在方舟平台托管的默认存储空间中时，支持最大 512 MB 文件的上传；文件存储在火山引擎对象存储 TOS Bucket 中时，支持最大 2 GB 的视频文件的上传，从而满足大文件处理需求。

支持音频理解#

部分模型支持对视频文件中的音频信息进行理解，详见视频内嵌音频输入。

抽帧策略#

用量说明：单视频最大 token 用量为 80k，单次请求视频最大 token 量还受模型的最大上下文窗口以及最大输入长度（当启用深度思考模式）限制，超出则需调整传入视频数量或视频长度。
基本概念：

  • 帧图像：某个时刻的视频画面，本文特指输入给模型的帧图像
  • 帧图像张数：视频时长 * fps

方舟会根据帧图像张数，对帧图像进行压缩，以平衡视频的理解精度和 token 用量。
不同模型的抽帧策略不同，具体如下：

抽帧策略 doubao-seed-2.0 及后续模型
单帧 tokens
  • max_frame_tokens：单帧最大 tokens 数，用于控制单帧质量的上限。
  • min_frame_tokens：单帧最小 tokens 数，用于控制单帧压缩的下限，支持 tokens 在 [64, 384] 区间内连续动态调整，默认值为 384。
单帧对应的max_pixels 单帧最大 tokens * 42 * 42
[11w, 67w]
抽帧数
  • 抽帧数范围：[16帧, 1280帧]。
Bash
# 最大抽帧数
80×1024 token ÷ 64 token/帧 = 1280 帧
  • min_frames：最小抽帧数，默认值为 16。若按 fps 计算的抽帧总数小于 16，则优先按 16 帧处理。
视频最大 tokens max_video_tokens：单个视频所有抽取帧的 tokens 总数上限，默认值为 81920（80 × 1024）。
抽帧方案 说明
  • 建议评估输出效果，按需调整fps字段配置或视频时长。
  • 如果视频二进制数据中未包含帧数编码信息，会按 fps 均匀抽帧。单次请求视频最大 Token 量如果超过模型最大输入长度则会报错；未超过则不报错，但视频输入的 token 可能会超过 80k。
  • fps 过高或视频长度过长：处理的帧图像数量超出1280帧，则按帧图像64tokens，时间间隔 视频时长/1280，均匀抽取1280帧。
  • fps 过小或视频长度过短：处理的帧图像数量不足16帧时抽取方案如下：
  • 视频总帧数 >= 16帧，均匀抽取16帧。
  • 视频总帧数 < 16帧，抽取视频所有帧。

视频理解工作原理#

视频处理的核心方式为 “帧与时间戳的结构化拼接”，具体规则如下：

  • 对视频抽帧得到的每帧图像，在其前插入时间戳文本。
    • doubao-seed-2.0 及后续模型：时间戳格式为 <时间戳> second。例如：4.0 second、4.5 second。
    • doubao-seed-2.0 之前模型：时间戳格式为 [<时间戳> second]。例如：[4.0 second]、[4.5 second]。
  • 拼接后形成“时间戳+图像”的有序序列，模型通过该序列理解视频的时序逻辑和内容变化。

抽帧逻辑举例#

FPS 1 FPS 0.5 FPS 2
时间戳 0.0 second 0.0 second 0.0 second
视频帧 <IMAGE> <IMAGE> <IMAGE>
时间戳 1.0 second 2.0 second 0.5 second
视频帧 <IMAGE> <IMAGE> <IMAGE>
时间戳 2.0 second 4.0 second 1.0 second
视频帧 <IMAGE> <IMAGE> <IMAGE>
时间戳 3.0 second 1.5 second
视频帧 <IMAGE> <IMAGE>
时间戳 4.0 second 2.0 second
视频帧 <IMAGE> <IMAGE>
时间戳 5.0 second 2.5 second
视频帧 <IMAGE> <IMAGE>
时间戳 3.0 second
视频帧 <IMAGE>
时间戳 3.5 second
视频帧 <IMAGE>
时间戳 4.0 second
视频帧 <IMAGE>
时间戳 4.5 second
视频帧 <IMAGE>
时间戳 5.0 second
视频帧 <IMAGE>
共6帧 共3帧 共11帧

多图请求等效#

视频理解请求等效于下面示例的多图理解请求。

JSON
{
    "model": "doubao-seed-2-1-pro-260915",
    "messages": [
        {
            "role": "user",
            "content": [
                {
                    "type":"text",
                    "text":"你觉得这个恐怖吗？"
                },
                {
                    "type":"text",
                    "text":"0.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_01>"}
                },
                {
                    "type":"text",
                    "text":"1.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_02>"}
                },
                {
                    "type":"text",
                    "text":"2.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_03>"}
                },
                {
                    "type":"text",
                    "text":"3.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_04>"}
                },
                {
                    "type":"text",
                    "text":"4.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_05>"}
                },
                {
                    "type":"text",
                    "text":"5.0 second"
                },
                {
                    "type":"image_url",
                    "image_url":{
                        "url":"<image_url_06>"}
                }
            ]
        }
    ]
}