Building Custom AI Agents

Overview

RbxGenie’s HTTP API enables you to build custom AI agents, automation scripts, or integrations with any language or framework. This guide shows how to create agents that can intelligently interact with Roblox Studio.

Custom agents can be built in any language that supports HTTP requests: Python, JavaScript, Go, Rust, etc.

Basic Agent Structure

A minimal agent needs three components:

HTTP client — To call the RbxGenie daemon API
Tool execution logic — To invoke tools and handle responses
Decision-making — To determine which tools to call (AI model, rule-based, etc.)

Python Example

Simple Agent

A basic agent that creates a part and sets its color:

Simple Agent

import requests
import json

DAEMON_URL = "http://127.0.0.1:7766"

def call_tool(tool_name, args):
    """Call a RbxGenie tool and return the result."""
    response = requests.post(
        f"{DAEMON_URL}/tool/{tool_name}",
        json=args,
        timeout=120
    )
    data = response.json()
    
    if not data.get("ok"):
        raise Exception(f"Tool failed: {data.get('error')}")
    
    return data.get("result")

def main():
    # Create a part in Workspace
    result = call_tool("create_object_with_properties", {
        "path": "Workspace",
        "className": "Part",
        "properties": {
            "Name": "AgentPart",
            "BrickColor": "Bright blue",
            "Position": {"type": "Vector3", "value": [0, 10, 0]},
            "Size": {"type": "Vector3", "value": [4, 1, 2]}
        }
    })
    
    print(f"Created part: {result}")
    
    # Set transparency
    call_tool("set_property", {
        "path": "Workspace.AgentPart",
        "property": "Transparency",
        "value": 0.5
    })
    
    print("Set transparency to 0.5")

if __name__ == "__main__":
    main()

AI-Powered Agent

An agent that uses OpenAI to decide which tools to call:

AI Agent

import requests
import json
from openai import OpenAI

DAEMON_URL = "http://127.0.0.1:7766"
client = OpenAI()

TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "get_file_tree",
            "description": "Get the file/instance tree of the place",
            "parameters": {
                "type": "object",
                "properties": {
                    "root": {"type": "string", "description": "Root instance path"},
                    "depth": {"type": "number", "description": "Max depth to traverse"}
                }
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "create_object_with_properties",
            "description": "Create a new instance with properties",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {"type": "string", "description": "Parent path"},
                    "className": {"type": "string", "description": "Class name"},
                    "properties": {"type": "object", "description": "Properties to set"}
                },
                "required": ["path", "className"]
            }
        }
    },
    # Add more tools as needed...
]

def call_tool(tool_name, args):
    """Call a RbxGenie tool."""
    response = requests.post(
        f"{DAEMON_URL}/tool/{tool_name}",
        json=args,
        timeout=120
    )
    data = response.json()
    
    if not data.get("ok"):
        return {"error": data.get("error")}
    
    return data.get("result")

def run_agent(user_request):
    """Run the AI agent with tool calling."""
    messages = [
        {"role": "system", "content": "You are an AI assistant that can interact with Roblox Studio using tools."},
        {"role": "user", "content": user_request}
    ]
    
    while True:
        response = client.chat.completions.create(
            model="gpt-4",
            messages=messages,
            tools=TOOLS,
            tool_choice="auto"
        )
        
        message = response.choices[0].message
        messages.append(message)
        
        # If no tool calls, return the final message
        if not message.tool_calls:
            print(f"Agent: {message.content}")
            break
        
        # Execute tool calls
        for tool_call in message.tool_calls:
            tool_name = tool_call.function.name
            args = json.loads(tool_call.function.arguments)
            
            print(f"Calling {tool_name} with {args}")
            result = call_tool(tool_name, args)
            
            # Add tool result to messages
            messages.append({
                "role": "tool",
                "tool_call_id": tool_call.id,
                "content": json.dumps(result)
            })

if __name__ == "__main__":
    run_agent("Create a red part in Workspace at position (5, 10, 0)")

JavaScript/Node.js Example

Simple Agent

const fetch = require('node-fetch');

const DAEMON_URL = 'http://127.0.0.1:7766';

async function callTool(toolName, args) {
  const response = await fetch(`${DAEMON_URL}/tool/${toolName}`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(args)
  });
  
  const data = await response.json();
  
  if (!data.ok) {
    throw new Error(`Tool failed: ${data.error}`);
  }
  
  return data.result;
}

async function main() {
  // Get place info
  const placeInfo = await callTool('get_place_info', {});
  console.log('Place:', placeInfo);
  
  // Search for all scripts
  const scripts = await callTool('search_objects', {
    className: 'Script',
    root: 'ServerScriptService',
    maxResults: 10
  });
  console.log('Found scripts:', scripts);
  
  // Create a GUI
  await callTool('create_object_with_properties', {
    path: 'StarterGui',
    className: 'ScreenGui',
    properties: {
      Name: 'MyGui',
      ResetOnSpawn: false
    }
  });
  
  await callTool('create_object_with_properties', {
    path: 'StarterGui.MyGui',
    className: 'TextLabel',
    properties: {
      Name: 'Title',
      Text: 'Hello from Agent!',
      Size: { type: 'UDim2', value: [0, 200, 0, 50] },
      Position: { type: 'UDim2', value: [0.5, -100, 0, 10] }
    }
  });
  
  console.log('Created GUI');
}

main().catch(console.error);

TypeScript Agent with Anthropic

AI Agent (TypeScript)

import Anthropic from '@anthropic-ai/sdk';
import fetch from 'node-fetch';

const DAEMON_URL = 'http://127.0.0.1:7766';
const anthropic = new Anthropic();

interface ToolResult {
  ok: boolean;
  result?: any;
  error?: string;
}

async function callTool(toolName: string, args: Record<string, any>): Promise<any> {
  const response = await fetch(`${DAEMON_URL}/tool/${toolName}`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(args)
  });
  
  const data = await response.json() as ToolResult;
  
  if (!data.ok) {
    throw new Error(`Tool failed: ${data.error}`);
  }
  
  return data.result;
}

const tools: Anthropic.Tool[] = [
  {
    name: 'get_file_tree',
    description: 'Get the file/instance tree of the Roblox place',
    input_schema: {
      type: 'object',
      properties: {
        root: { type: 'string', description: 'Root instance path' },
        depth: { type: 'number', description: 'Max depth to traverse' }
      }
    }
  },
  {
    name: 'create_object_with_properties',
    description: 'Create a new instance with properties',
    input_schema: {
      type: 'object',
      properties: {
        path: { type: 'string', description: 'Parent instance path' },
        className: { type: 'string', description: 'Roblox class name' },
        properties: { type: 'object', description: 'Properties to set' }
      },
      required: ['path', 'className']
    }
  },
  // Add more tools...
];

async function runAgent(userRequest: string) {
  const messages: Anthropic.MessageParam[] = [
    { role: 'user', content: userRequest }
  ];
  
  while (true) {
    const response = await anthropic.messages.create({
      model: 'claude-3-5-sonnet-20241022',
      max_tokens: 4096,
      tools,
      messages
    });
    
    console.log('Response:', response.content);
    
    // Check if we're done
    if (response.stop_reason === 'end_turn') {
      break;
    }
    
    // Execute tool calls
    if (response.stop_reason === 'tool_use') {
      const toolResults: Anthropic.MessageParam[] = [];
      
      for (const block of response.content) {
        if (block.type === 'tool_use') {
          console.log(`Calling ${block.name} with`, block.input);
          
          try {
            const result = await callTool(block.name, block.input as Record<string, any>);
            
            toolResults.push({
              role: 'user',
              content: [{
                type: 'tool_result',
                tool_use_id: block.id,
                content: JSON.stringify(result)
              }]
            });
          } catch (error: any) {
            toolResults.push({
              role: 'user',
              content: [{
                type: 'tool_result',
                tool_use_id: block.id,
                content: `Error: ${error.message}`,
                is_error: true
              }]
            });
          }
        }
      }
      
      messages.push({ role: 'assistant', content: response.content });
      messages.push(...toolResults);
    }
  }
}

runAgent('Create a blue part in Workspace and tag it with "Collectible"');

Agent Design Patterns

1. Exploration Agent

Discover and analyze the game structure:

Exploration Agent

def explore_game():
    """Explore and summarize the game structure."""
    # Get high-level structure
    structure = call_tool("get_project_structure", {})
    print("Project structure:", structure)
    
    # Get detailed summary
    summary = call_tool("summarize_game", {})
    print("Game summary:", summary)
    
    # Find all scripts
    scripts = call_tool("search_objects", {
        "className": "Script",
        "maxResults": 100
    })
    print(f"Found {len(scripts)} scripts")
    
    # Analyze each script
    for script_path in scripts:
        source = call_tool("get_script_source", {"path": script_path})
        # Perform analysis (e.g., complexity, dependencies, etc.)

2. Automation Agent

Perform batch operations:

Automation Agent

def batch_modify_parts():
    """Find and modify all parts with a specific tag."""
    # Get all tagged instances
    tagged = call_tool("get_tagged", {"tag": "Destructible"})
    print(f"Found {len(tagged)} destructible parts")
    
    # Mass set properties
    call_tool("mass_set_property", {
        "paths": tagged,
        "property": "Material",
        "value": {"type": "Enum", "value": "Enum.Material.Wood"}
    })
    
    call_tool("mass_set_property", {
        "paths": tagged,
        "property": "Color",
        "value": {"type": "Color3", "value": [0.6, 0.4, 0.2]}
    })
    
    print("Applied wood material and color to all destructibles")

3. Testing Agent

Run automated tests in play mode:

Testing Agent

def test_player_spawn():
    """Test that players spawn correctly."""
    test_code = """
    local Players = game:GetService("Players")
    local player = Players:GetPlayers()[1]
    
    if not player then
        error("No player found")
    end
    
    if not player.Character then
        error("Player has no character")
    end
    
    local humanoid = player.Character:FindFirstChildOfClass("Humanoid")
    if not humanoid then
        error("Character has no Humanoid")
    end
    
    print("Player spawn test passed")
    return true
    """
    
    result = call_tool("run_script_in_play_mode", {
        "code": test_code,
        "timeout": 10,
        "mode": "play"
    })
    
    print("Test result:", result)
    return result.get("success", False)

4. Code Generation Agent

Generate and inject scripts:

Code Generation Agent

def generate_datastore_script(data_key: str):
    """Generate a DataStore script for a specific key."""
    script_code = f'''
local DataStoreService = game:GetService("DataStoreService")
local dataStore = DataStoreService:GetDataStore("{data_key}")

local function saveData(player, data)
    local success, err = pcall(function()
        dataStore:SetAsync(player.UserId, data)
    end)
    
    if not success then
        warn("Failed to save data for", player.Name, ":", err)
    end
end

local function loadData(player)
    local success, data = pcall(function()
        return dataStore:GetAsync(player.UserId)
    end)
    
    if success then
        return data
    else
        warn("Failed to load data for", player.Name)
        return nil
    end
end

return {{
    Save = saveData,
    Load = loadData
}}
'''
    
    # Create the script
    call_tool("create_object", {
        "path": "ServerScriptService",
        "className": "ModuleScript",
        "name": f"{data_key}DataStore"
    })
    
    # Set the source
    call_tool("set_script_source", {
        "path": f"ServerScriptService.{data_key}DataStore",
        "source": script_code
    })
    
    print(f"Generated DataStore script for {data_key}")

Advanced Techniques

Error Recovery

def robust_tool_call(tool_name, args, max_retries=3):
    """Call a tool with automatic retry on failure."""
    for attempt in range(max_retries):
        try:
            return call_tool(tool_name, args)
        except Exception as e:
            if attempt == max_retries - 1:
                raise
            print(f"Retry {attempt + 1}/{max_retries}: {e}")
            time.sleep(1)

Parallel Execution

import concurrent.futures

def parallel_property_changes(instances, property_name, value):
    """Change properties on multiple instances in parallel."""
    def set_prop(path):
        return call_tool("set_property", {
            "path": path,
            "property": property_name,
            "value": value
        })
    
    # Note: RbxGenie processes commands sequentially, but this
    # demonstrates how to prepare requests in parallel
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        results = list(executor.map(set_prop, instances))
    
    return results

State Management

class StudioState:
    """Track state changes in Roblox Studio."""
    
    def __init__(self):
        self.created_instances = []
        self.modified_properties = []
    
    def create_object(self, path, className, name=None):
        result = call_tool("create_object", {
            "path": path,
            "className": className,
            "name": name
        })
        
        instance_path = result.get("path")
        self.created_instances.append(instance_path)
        return result
    
    def rollback(self):
        """Delete all created instances."""
        for path in reversed(self.created_instances):
            try:
                call_tool("delete_object", {"path": path})
            except Exception as e:
                print(f"Failed to delete {path}: {e}")
        
        self.created_instances.clear()

Best Practices

Use batch operations

When modifying multiple instances, use mass_set_property, mass_create_objects, etc. instead of individual calls. This reduces network overhead and improves performance.

Handle timeouts gracefully

Tool calls timeout after 120 seconds. Design your agent to handle timeouts (retry, skip, or fail gracefully).

Validate paths before use

Use get_instance_properties or search_objects to verify instance paths exist before modifying them.

Log tool calls

Keep a log of all tool calls and results for debugging and analysis.

Implement rate limiting

Avoid overwhelming Studio with rapid-fire requests. Add delays between non-critical operations.

Use proper error handling

Always check the ok field in responses and handle errors appropriately.

Example: Complete Build Agent

A full agent that builds a simple obby:

Obby Builder Agent

import random

def build_obby(num_platforms=10):
    """Build a simple obby with multiple platforms."""
    print("Building obby...")
    
    # Create obby folder
    call_tool("create_object", {
        "path": "Workspace",
        "className": "Folder",
        "name": "Obby"
    })
    
    platforms = []
    x_pos = 0
    
    for i in range(num_platforms):
        # Random platform properties
        size_x = random.uniform(4, 8)
        size_z = random.uniform(4, 8)
        gap = random.uniform(5, 10)
        y_pos = random.uniform(5, 15)
        
        # Create platform
        call_tool("create_object_with_properties", {
            "path": "Workspace.Obby",
            "className": "Part",
            "properties": {
                "Name": f"Platform{i+1}",
                "Size": {"type": "Vector3", "value": [size_x, 1, size_z]},
                "Position": {"type": "Vector3", "value": [x_pos, y_pos, 0]},
                "Anchored": True,
                "BrickColor": random.choice(["Bright red", "Bright blue", "Bright green"]),
                "Material": {"type": "Enum", "value": "Enum.Material.SmoothPlastic"}
            }
        })
        
        platforms.append(f"Workspace.Obby.Platform{i+1}")
        x_pos += gap + size_x
    
    # Add checkpoint tag to every 3rd platform
    for i in range(2, len(platforms), 3):
        call_tool("add_tag", {
            "path": platforms[i],
            "tag": "Checkpoint"
        })
    
    # Create finish line
    call_tool("create_object_with_properties", {
        "path": "Workspace.Obby",
        "className": "Part",
        "properties": {
            "Name": "Finish",
            "Size": {"type": "Vector3", "value": [10, 20, 1]},
            "Position": {"type": "Vector3", "value": [x_pos + 10, 10, 0]},
            "Anchored": True,
            "BrickColor": "Bright yellow",
            "Material": {"type": "Enum", "value": "Enum.Material.Neon"},
            "Transparency": 0.5
        }
    })
    
    call_tool("add_tag", {
        "path": "Workspace.Obby.Finish",
        "tag": "Finish"
    })
    
    print(f"Obby complete! Created {num_platforms} platforms.")

if __name__ == "__main__":
    build_obby(15)

Next Steps

HTTP API Reference

Complete API documentation

Tools Reference

Browse all 46 tools

Claude Desktop

Use MCP with Claude Desktop

Cursor

Use MCP with Cursor IDE

Get Started

Architecture

Integration

Guides

Examples

Building Custom AI Agents

Overview

Basic Agent Structure

Python Example

Simple Agent

AI-Powered Agent

JavaScript/Node.js Example

Simple Agent

TypeScript Agent with Anthropic

Agent Design Patterns

1. Exploration Agent

2. Automation Agent

3. Testing Agent

4. Code Generation Agent

Advanced Techniques

Error Recovery

Parallel Execution

State Management

Best Practices

Example: Complete Build Agent

Next Steps

HTTP API Reference

Tools Reference

Claude Desktop

Cursor

Get Started

Architecture

Integration

Guides

Examples

Documentation Index

​Overview

​Basic Agent Structure

​Python Example

​Simple Agent

​AI-Powered Agent

​JavaScript/Node.js Example

​Simple Agent

​TypeScript Agent with Anthropic

​Agent Design Patterns

​1. Exploration Agent

​2. Automation Agent

​3. Testing Agent

​4. Code Generation Agent

​Advanced Techniques

​Error Recovery

​Parallel Execution

​State Management

​Best Practices

​Example: Complete Build Agent

​Next Steps

HTTP API Reference

Tools Reference

Claude Desktop

Cursor

Overview

Basic Agent Structure

Python Example

Simple Agent

AI-Powered Agent

JavaScript/Node.js Example

Simple Agent

TypeScript Agent with Anthropic

Agent Design Patterns

1. Exploration Agent

2. Automation Agent

3. Testing Agent

4. Code Generation Agent

Advanced Techniques

Error Recovery

Parallel Execution

State Management

Best Practices

Example: Complete Build Agent

Next Steps