GUI Agent 能力

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59PURE_GUI_FUNCTIONS = """{"name": "click", "description": "Mouse left single click action.", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: <point>x y</point>"}}, "required": ["point"]}}
{"name": "drag", "description": "Mouse left button drag action.", "parameters": {"type": "object", "properties": {"start_point": {"type": "string", "description": "Drag start point. The format is: <point>x y</point>"}, "end_point": {"type": "string", "description": "Drag end point. The format is: <point>x y</point>"}}, "required": ["start_point", "end_point"]}}
{"name": "hotkey", "description": "Press hotkey.", "parameters": {"type": "object", "properties": {"key": {"type": "string", "description": "Hotkeys you want to press. Split keys with a space and use lowercase."}}, "required": ["key"]}}
{"name": "left_double", "description": "Mouse left double click action.", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: <point>x y</point>"}}, "required": ["point"]}}
{"name": "right_single", "description": "Mouse right single click action.", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Click coordinates. The format is: <point>x y</point>"}}, "required": ["point"]}}
{"name": "scroll", "description": "Scroll action.", "parameters": {"type": "object", "properties": {"point": {"type": "string", "description": "Scroll start position. If not specified, default to execute on the current mouse position. The format is: <point>x y</point>"}, "direction": {"type": "string", "description": "Scroll direction.", "enum": ["up", "down", "left", "right"]}}, "required": ["direction", "point"]}}
{"name": "type", "description": "Type content.", "parameters": {"type": "object", "properties": {"content": {"type": "string", "description": "Type content. If you want to submit your input, use \\n at the end of content."}}, "required": ["content"]}}
{"name": "wait", "description": "Wait for a while.", "parameters": {"type": "object", "properties": {"time": {"type": "integer", "description": "Wait time in seconds."}}, "required": []}}"""

# 动作定义解析成 schema 列表,供官方解析器 parse_xml_action_65 做软校验。
PURE_GUI_SCHEMAS = functions_block_to_schemas(PURE_GUI_FUNCTIONS)

# think_token 是一个约定的哨兵串,把思维链和答案分隔开。线上就是这个常量。
THINK_TOKEN = "think_never_used_51bce0c785ca2f68081bfa7d91973934"

TASK_SP = """You are provided with a task description, a history of previous actions, and corresponding screenshots. Your goal is to perform the next action to complete the task. Please note that if performing the same action multiple times results in a static screen with no changes, you should attempt a modified or alternative action."""

FUNCTION_SP = f"""## Function Definition

- You have access to the following functions:
{PURE_GUI_FUNCTIONS}


- To call a function, use the following structure without any suffix:

<{THINK_TOKEN}> reasoning process </{THINK_TOKEN}>
<seed:tool_call><function name="example_function_name"><parameter name="example_parameter_1" string="false|true">value_1</parameter></function></seed:tool_call>

## Important Notes
- All required parameters must be explicitly provided.
- The string attribute within the parameter tag specifies the type of the parameter value. For instance, if the value is an integer, number, or boolean, set string="false". If the value is a string, set string="true". Example:
<parameter name="parameter1" string="false">1</parameter>
<parameter name="parameter1" string="false">1.0</parameter>
<parameter name="parameter2" string="false">true</parameter>
<parameter name="parameter3" string="true">This is a test 1.</parameter>
- For complex parameter types, such as lists or objects, set string="false" and provide the parameter value in JSON format. Example:
<parameter name="parameter4" string="false">{{"key1": "value1", "key2": "value2"}}</parameter>
<parameter name="parameter5" string="false">[1, 2, 3]</parameter>"""


def build_system_prompt() -> list[dict]:
    """纯 GUI 主线的两段式 system prompt(任务框架 + 动作定义)。"""
    return [
        {"role": "system", "content": TASK_SP},
        {"role": "system", "content": FUNCTION_SP},
    ]


def build_user_message(task: str, data_url: str) -> dict:
    """构造一条多模态 user 消息:文本任务 + 一张截图。"""
    return {
        "role": "user",
        "content": [
            {"type": "text", "text": task},
            {"type": "image_url", "image_url": {"url": data_url}},
        ],
    }

messages = build_system_prompt() + [build_user_message(task, data_url)]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39def extract_point(value: str) -> tuple[float, float] | None:
    """从 `"<point>x y</point>"` 里取出 (x, y) 的 0–1000 相对坐标;取不到返回 None。"""
    if not isinstance(value, str):
        return None
    inner = value.replace("<point>", "").replace("</point>", "").strip()
    parts = inner.split()
    if len(parts) == 2:
        try:
            return float(parts[0]), float(parts[1])
        except ValueError:
            return None
    return None

def rel_point_to_pixel(point: tuple[float, float], image_size: tuple[int, int]) -> tuple[int, int]:
    """把 0–1000 相对坐标映射回真实像素坐标。"""
    x, y = point
    w, h = image_size
    return round(x / 1000.0 * w), round(y / 1000.0 * h)

actions = parse_actions(reply.content, PURE_GUI_SCHEMAS)
    if not actions:
        # 没有任何 <seed:tool_call> 动作块 = 模型以纯文本收尾,视为交互结束。
        print("响应里没有 <seed:tool_call> 动作:模型以纯文本作答,视为交互结束(默认结束方式)。")
        print("最终答复文本:\n")
        answer = reply.content.split(f"</{THINK_TOKEN}>")[-1].strip()
        print(answer or reply.content)
        return 0

    print(f"解析出 {len(actions)} 个动作(官方结构:function + parameters):\n")
    for i, act in enumerate(actions):
        name, params = act["function"], act.get("parameters", {})
        print(f"[{i}] name={name!r}  parameters={params}")
        # 坐标类参数(point / start_point / end_point)从 "<point>x y</point>" 取 0–1000 值并映射像素。
        for pname, pval in params.items():
            rel = extract_point(pval)
            if rel is not None:
                px, py = rel_point_to_pixel(rel, size)
                print(f"      {pname}: 相对坐标 {rel} (0–1000)  →  真实像素 ({px}, {py})")
    return 0

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55def build_click_function_prompt(decimals: int) -> str:
    if decimals >= 1:
        coord_note = (
            "- Coordinates are normalized to the range [0, 1000] and may include one decimal place "
            "(e.g. <point>123.4 567.8</point>) to allow finer, sub-pixel grounding granularity.\n"
        )
    else:
        coord_note = "- Coordinates must be integer points normalized to the range [0, 1000].\n"
    return (
        "## Function Definition\n\n"
        "- You have access to the following functions:\n"
        f"{CLICK_FUNCTION_SPEC}\n\n"
        "- To call a function, use the following structure without any suffix:\n\n"
        f"<{THINK_TAG}> reasoning process </{THINK_TAG}>\n"
        '<seed:tool_call><function name="click">'
        '<parameter name="point" string="true"><point>x y</point></parameter>'
        "</function></seed:tool_call>\n"
        "## Important Notes\n"
        "- All required parameters must be explicitly provided.\n"
        f"{coord_note}"
        "- The origin is the top-left of the full screenshot.\n"
    )

def build_user_prompt_template(decimals: int) -> str:
    if decimals >= 1:
        coord_line = (
            "Coordinates are normalized to [0, 1000] relative to the image and may use one decimal "
            "place for finer precision. "
        )
    else:
        coord_line = "Coordinates must be integer normalized coordinates in [0, 1000] relative to the image. "
    return (
        "You are a GUI click grounding agent. You are given a screenshot and a task. "
        "Return exactly one click tool call to match the requested operation. "
        + coord_line
        + "## User Instruction: What element matches the following task: {instruction}"
    )

# Defaults (integer coords); overridden in main() when --decimals is set
SYSTEM_CLICK_FUNCTION_PROMPT = build_click_function_prompt(0)
USER_PROMPT_TEMPLATE = build_user_prompt_template(0)

def build_ark_messages(instruction, base64_image):
    """火山 Ark(OpenAI 兼容)标准多模态消息:接入点自带 chat 模板,无需特殊 token。"""
    return [
        {"role": "system", "content": SYSTEM_CLICK_FUNCTION_PROMPT},
        {
            "role": "user",
            "content": [
                {"type": "text", "text": USER_PROMPT_TEMPLATE.format(instruction=instruction)},
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}},
            ],
        },
    ]