chore: cli esm bug and sdk docs (#122)

* fix: cli * release: publish beta packages * chore: esm * release: publish beta packages * chore: sdk
bytedance · Feb 20, 2025 · d169e46 · d169e46
1 parent 756ccc4
commit d169e46
Show file tree

Hide file tree

Showing 23 changed files with 191 additions and 42 deletions.
diff --git a/.changeset/config.json b/.changeset/config.json
@@ -2,11 +2,7 @@
   "$schema": "https://unpkg.com/@changesets/[email protected]/schema.json",
   "changelog": "@changesets/cli/changelog",
   "commit": false,
-  "fixed": [
-    [
-      "@ui-tars/*"
-    ]
-  ],
+  "fixed": [],
   "linked": [],
   "access": "restricted",
   "baseBranch": "main",

diff --git a/.changeset/pre.json b/.changeset/pre.json
@@ -3,15 +3,17 @@
   "tag": "beta",
   "initialVersions": {
     "ui-tars-desktop": "0.0.6",
-    "@ui-tars/action-parser": "1.2.0-beta.6",
-    "@ui-tars/cli": "1.2.0-beta.8",
-    "@ui-tars/electron-ipc": "1.2.0-beta.6",
-    "@ui-tars/operator-nut-js": "1.2.0-beta.8",
-    "@ui-tars/sdk": "1.2.0-beta.8",
-    "@ui-tars/shared": "1.2.0-beta.6",
-    "@ui-tars/utio": "1.2.0-beta.6"
+    "@ui-tars/action-parser": "1.2.0-beta.10",
+    "@ui-tars/cli": "1.2.0-beta.10",
+    "@ui-tars/electron-ipc": "1.2.0-beta.10",
+    "@ui-tars/operator-nut-js": "1.2.0-beta.10",
+    "@ui-tars/sdk": "1.2.0-beta.10",
+    "@ui-tars/shared": "1.2.0-beta.10",
+    "@ui-tars/utio": "1.2.0-beta.10"
   },
   "changesets": [
-    "selfish-humans-drive"
+    "selfish-humans-drive",
+    "strange-schools-help",
+    "witty-points-rescue"
   ]
 }
diff --git a/.changeset/strange-schools-help.md b/.changeset/strange-schools-help.md
@@ -0,0 +1,5 @@
+---
+'@ui-tars/cli': patch
+---
+
+chore: node-fetch
diff --git a/.changeset/witty-points-rescue.md b/.changeset/witty-points-rescue.md
@@ -0,0 +1,6 @@
+---
+'@ui-tars/cli': patch
+'@ui-tars/sdk': patch
+---
+
+update
diff --git a/docs/sdk.md b/docs/sdk.md
@@ -48,15 +48,27 @@ classDiagram
     Operator <|.. MobileOperator
 ```
 
-## Installation
+## Try it out
 
 ```bash
-npm install @ui-tars/sdk@beta
+npx @ui-tars/cli start
 ```
 
-> Note: Later, we will release the stable version `@ui-tars/sdk@latest`.
+Input your UI-TARS Model Service Config(`baseURL`, `apiKey`, `model`), then you can control your computer with CLI.
 
-## Quick Start
+```
+Need to install the following packages:
+Ok to proceed? (y) y
+
+│
+◆  Input your instruction
+│  _ Open Chrome
+└
+```
+
+<video src="https://github.com/user-attachments/assets/991c6063-d474-40a7-8bbc-f5700d95977a" height="300" />
+
+## Agent Execution Process
 
 ```mermaid
 flowchart LR
@@ -164,7 +176,46 @@ stateDiagram-v2
 
 ### Operator Interface
 
-When implementing a custom operator, you need to implement two core methods:
+When implementing a custom operator, you need to implement two core methods: `screenshot()` and `execute()`.
+
+#### Initialize
+
+`npm init` to create a new operator package, configuration is as follows:
+
+```json
+{
+  "name": "your-operator-tool",
+  "version": "1.0.0",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsup --watch",
+    "prepare": "npm run build",
+    "build": "tsup",
+    "test": "vitest"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "jimp": "^1.6.0"
+  },
+  "peerDependencies": {
+    "@ui-tars/sdk": "latest"
+  },
+  "devDependencies": {
+    "@ui-tars/sdk": "latest",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.2"
+  }
+}
+```
 
 #### screenshot()
 
@@ -268,6 +319,8 @@ const agent = new GUIAgent({
 
 ### Planning
 
+You can combine planning/reasoning models (such as OpenAI-o1, DeepSeek-R1) to implement complex GUIAgent logic for planning, reasoning, and execution:
+
 ```ts
 const guiAgent = new GUIAgent({
   // ... other config

diff --git a/packages/action-parser/CHANGELOG.md b/packages/action-parser/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @ui-tars/action-parser
 
+## 1.2.0-beta.10
+
+### Patch Changes
+
+- @ui-tars/shared@1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/action-parser/package.json b/packages/action-parser/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/action-parser",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.10",
   "description": "Action parser SDK for UI-TARS",
   "repository": {
     "type": "git",

diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -1,5 +1,20 @@
 # @ui-tars/cli
 
+## 1.2.0-beta.11
+
+### Patch Changes
+
+- chore: node-fetch
+
+## 1.2.0-beta.10
+
+### Patch Changes
+
+- update
+- Updated dependencies
+  - @ui-tars/sdk@1.2.0-beta.10
+  - @ui-tars/operator-nut-js@1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/cli",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.11",
   "description": "CLI for UI-TARS",
   "repository": {
     "type": "git",
@@ -34,12 +34,15 @@
   "dependencies": {
     "commander": "^13.1.0",
     "jimp": "1.6.0",
+    "js-yaml": "^4.1.0",
     "@clack/prompts": "^0.10.0",
     "@ui-tars/operator-nut-js": "workspace:*",
-    "@ui-tars/sdk": "workspace:*"
+    "@ui-tars/sdk": "workspace:*",
+    "node-fetch": "^2.7.0"
   },
   "devDependencies": {
     "tsup": "^8.3.5",
+    "@types/js-yaml": "^4.0.9",
     "typescript": "^5.7.2",
     "vitest": "^3.0.2"
   }

diff --git a/packages/cli/src/cli/commands.ts b/packages/cli/src/cli/commands.ts
@@ -13,7 +13,7 @@ export const run = () => {
   program
     .command('start')
     .description('starting the ui-tars agent...')
-    .option('-p, --presets', 'Model Config Presets')
+    .option('-p, --presets <url>', 'Model Config Presets')
     .action(async (options: CliOptions) => {
       try {
         await start(options);

diff --git a/packages/cli/src/cli/start.ts b/packages/cli/src/cli/start.ts
@@ -2,13 +2,15 @@
  * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
  * SPDX-License-Identifier: Apache-2.0
  */
-import { GUIAgent } from '@ui-tars/sdk';
-import * as p from '@clack/prompts';
-// import { inspect } from 'node:util';
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
 
+import fetch from 'node-fetch';
+import { GUIAgent } from '@ui-tars/sdk';
+import * as p from '@clack/prompts';
+import yaml from 'js-yaml';
+
 import { NutJSOperator } from '@ui-tars/operator-nut-js';
 
 export interface CliOptions {
@@ -24,14 +26,27 @@ export const start = async (options: CliOptions) => {
     model: '',
   };
 
-  try {
-    if (fs.existsSync(CONFIG_PATH)) {
+  if (options.presets) {
+    const response = await fetch(options.presets);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch preset: ${response.status}`);
+    }
+
+    const yamlText = await response.text();
+    const preset = yaml.load(yamlText) as any;
+
+    config.apiKey = preset?.vlmApiKey;
+    config.baseURL = preset?.vlmBaseUrl;
+    config.model = preset?.vlmModelName;
+  } else if (fs.existsSync(CONFIG_PATH)) {
+    try {
       config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf-8'));
+    } catch (error) {
+      console.warn('read config file failed', error);
+      return;
     }
-  } catch (error) {
-    console.warn('read config file failed', error);
-    return;
   }
+
   if (!config.baseURL || !config.apiKey || !config.model) {
     const configAnswers = await p.group(
       {

diff --git a/packages/electron-ipc/CHANGELOG.md b/packages/electron-ipc/CHANGELOG.md
@@ -1,5 +1,7 @@
 # @ui-tars/electron-ipc
 
+## 1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/electron-ipc/package.json b/packages/electron-ipc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/electron-ipc",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.10",
   "description": "Type-safe Electron inter-process communication for UI-TARS",
   "repository": {
     "type": "git",

diff --git a/packages/operators/nut-js/CHANGELOG.md b/packages/operators/nut-js/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @ui-tars/operator-nut-js
 
+## 1.2.0-beta.10
+
+### Patch Changes
+
+- Updated dependencies
+  - @ui-tars/sdk@1.2.0-beta.10
+  - @ui-tars/shared@1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/operators/nut-js/package.json b/packages/operators/nut-js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/operator-nut-js",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.10",
   "description": "Operator Nut JS SDK for UI-TARS",
   "repository": {
     "type": "git",

diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @ui-tars/sdk
 
+## 1.2.0-beta.10
+
+### Patch Changes
+
+- update
+  - @ui-tars/action-parser@1.2.0-beta.10
+  - @ui-tars/shared@1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/sdk/README.md b/packages/sdk/README.md
@@ -8,10 +8,10 @@ It provides a flexible framework to create agents that can interact with graphic
 
 ```mermaid
 classDiagram
-    class GUIAgent {
-        +model
-        +operator
-        +signal
+    class GUIAgent~T extends Operator~ {
+        +model: UITarsModel
+        +operator: T
+        +signal: AbortSignal
         +onData
         +run()
     }
@@ -21,6 +21,7 @@ classDiagram
     }
 
     class Operator {
+        <<interface>>
         +screenshot()
         +execute()
     }
@@ -30,9 +31,21 @@ classDiagram
         +execute()
     }
 
+    class WebOperator {
+        +screenshot()
+        +execute()
+    }
+
+    class MobileOperator {
+        +screenshot()
+        +execute()
+    }
+
     GUIAgent --> UITarsModel
-    GUIAgent --> Operator
-    Operator <|-- NutJSOperator
+    GUIAgent ..> Operator
+    Operator <|.. NutJSOperator
+    Operator <|.. WebOperator
+    Operator <|.. MobileOperator
 ```
 
 ## Installation
@@ -55,7 +68,6 @@ flowchart LR
     Prediction --> Agent
     Agent --> Operator
     Operator --> Action[Execute]
-    Action --> Agent
 ```
 
 ### Basic Usage

diff --git a/packages/sdk/package.json b/packages/sdk/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/sdk",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.10",
   "description": "A powerful cross-platform(ANY device/platform) toolkit for building GUI automation agents for UI-TARS",
   "repository": {
     "type": "git",

diff --git a/packages/shared/CHANGELOG.md b/packages/shared/CHANGELOG.md
@@ -1,5 +1,7 @@
 # @ui-tars/shared
 
+## 1.2.0-beta.10
+
 ## 1.2.0-beta.9
 
 ### Patch Changes

diff --git a/packages/shared/package.json b/packages/shared/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars/shared",
-  "version": "1.2.0-beta.9",
+  "version": "1.2.0-beta.10",
   "description": "Shared types for UI-TARS",
   "repository": {
     "type": "git",