Created
April 22, 2026 22:06
-
-
Save AlexMerzlikin/a810877fb26c2325536213295ddb84c3 to your computer and use it in GitHub Desktop.
Sample of MCP custom tool that lets an LLM control gameplay
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| using System; | |
| using System.Collections.Generic; | |
| using System.Threading.Tasks; | |
| using MCPForUnity.Editor.Helpers; | |
| using MCPForUnity.Editor.Tools; | |
| using Newtonsoft.Json.Linq; | |
| using UnityEditor; | |
| using UnityEngine; | |
| namespace HumanFactory.Editor | |
| { | |
| /// <summary> | |
| /// MCP custom tool that lets an LLM control Human Factory gameplay. | |
| /// | |
| /// Tool name : game_play | |
| /// Actions : simulate_click | simulate_drag | get_game_state | wait_frames | |
| /// | |
| /// Input actions require Unity Play mode to be active. | |
| /// Access via batch_execute: { "tool": "game_play", "params": { "action": "...", ... } } | |
| /// </summary> | |
| [McpForUnityTool("game_play", | |
| Description = | |
| "Controls Human Factory gameplay via simulated mouse input. " + | |
| "Actions: " + | |
| "simulate_click (x, y, OR name) — tap at screen coordinates or click UI element by name; " + | |
| "simulate_drag (startX, startY, endX, endY, steps=15) — draw a line by dragging; " + | |
| "get_game_state — returns isPlaying, isGameActive, currentScore, targetScore, humanCount; " + | |
| "wait_frames (count=30) — wait N frames for physics to settle. " + | |
| "Input actions require Play mode.")] | |
| public static class GamePlayTool | |
| { | |
| public static async Task<object> HandleCommand(JObject @params) | |
| { | |
| if (@params == null) | |
| return new ErrorResponse("Parameters cannot be null."); | |
| var p = new ToolParams(@params); | |
| var actionResult = p.GetRequired("action"); | |
| if (!actionResult.IsSuccess) | |
| return new ErrorResponse(actionResult.ErrorMessage); | |
| string action = actionResult.Value.ToLowerInvariant(); | |
| switch (action) | |
| { | |
| case "simulate_click": return await SimulateClick(p, @params); | |
| case "simulate_drag": return await SimulateDrag(p); | |
| case "get_game_state": return GetGameState(); | |
| case "wait_frames": return await WaitFrames(p); | |
| default: | |
| return new ErrorResponse( | |
| $"Unknown action '{action}'. Supported: simulate_click, simulate_drag, get_game_state, wait_frames."); | |
| } | |
| } | |
| private static async Task<object> SimulateClick(ToolParams p, JObject rawParams) | |
| { | |
| if (!EditorApplication.isPlaying) | |
| return new ErrorResponse("simulate_click requires Play mode."); | |
| string name = rawParams["name"]?.ToString(); | |
| if (!string.IsNullOrEmpty(name)) | |
| { | |
| var go = GameObject.Find(name); | |
| if (go == null) return new ErrorResponse($"UI element '{name}' not found."); | |
| var btn = go.GetComponent<UnityEngine.UI.Button>(); | |
| if (btn != null) | |
| { | |
| btn.onClick.Invoke(); | |
| Debug.Log($"[GameplayBot] MCP simulate_click invoked UI button: {name}"); | |
| return new SuccessResponse($"Simulated UI click on {name}"); | |
| } | |
| return new ErrorResponse($"'{name}' does not have a Button component."); | |
| } | |
| float x = p.GetFloat("x") ?? 0f; | |
| float y = p.GetFloat("y") ?? 0f; | |
| Debug.Log($"[GameplayBot] MCP simulate_click at ({x},{y})"); | |
| var token = new BotCompletionToken(); | |
| var sim = GetOrCreateSimulator(); | |
| if (sim == null) return new ErrorResponse("Could not create GameplayInputSimulator."); | |
| sim.EnqueueClick(new Vector2(x, y), token); | |
| return await WaitForToken(token, 10.0); | |
| } | |
| private static async Task<object> SimulateDrag(ToolParams p) | |
| { | |
| if (!EditorApplication.isPlaying) | |
| return new ErrorResponse("simulate_drag requires Play mode."); | |
| float sx = p.GetFloat("startX") ?? p.GetFloat("start_x") ?? 0f; | |
| float sy = p.GetFloat("startY") ?? p.GetFloat("start_y") ?? 0f; | |
| float ex = p.GetFloat("endX") ?? p.GetFloat("end_x") ?? 0f; | |
| float ey = p.GetFloat("endY") ?? p.GetFloat("end_y") ?? 0f; | |
| int steps = p.GetInt("steps") ?? 15; | |
| Debug.Log($"[GameplayBot] MCP simulate_drag ({sx},{sy}) → ({ex},{ey}) steps={steps}"); | |
| var token = new BotCompletionToken(); | |
| var sim = GetOrCreateSimulator(); | |
| if (sim == null) return new ErrorResponse("Could not create GameplayInputSimulator."); | |
| sim.EnqueueDrag(new Vector2(sx, sy), new Vector2(ex, ey), steps, token); | |
| return await WaitForToken(token, 15.0); | |
| } | |
| private static object GetGameState() | |
| { | |
| if (!EditorApplication.isPlaying) | |
| { | |
| return new SuccessResponse("Editor is not in Play mode.", | |
| new Dictionary<string, object> { { "isPlaying", false } }); | |
| } | |
| var gm = GameManager.Instance; | |
| var humans = GameObject.FindObjectsByType<HumanController>(FindObjectsSortMode.None); | |
| int humanCount = humans?.Length ?? 0; | |
| if (gm == null) | |
| { | |
| Debug.Log("[GameplayBot] get_game_state: GameManager not found."); | |
| return new SuccessResponse("Play mode active. GameManager not yet found.", | |
| new Dictionary<string, object> | |
| { | |
| { "isPlaying", true }, | |
| { "gameManagerFound", false }, | |
| { "humanCount", humanCount } | |
| }); | |
| } | |
| var data = new Dictionary<string, object> | |
| { | |
| { "isPlaying", true }, | |
| { "gameManagerFound", true }, | |
| { "isGameActive", gm.IsGameActive }, | |
| { "currentScore", gm.currentScore }, | |
| { "targetScore", gm.targetScore }, | |
| { "humanCount", humanCount } | |
| }; | |
| Debug.Log( | |
| $"[GameplayBot] get_game_state: {gm.currentScore}/{gm.targetScore} active={gm.IsGameActive} humans={humanCount}"); | |
| return new SuccessResponse( | |
| $"Score: {gm.currentScore}/{gm.targetScore}, active={gm.IsGameActive}, humans={humanCount}.", data); | |
| } | |
| private static async Task<object> WaitFrames(ToolParams p) | |
| { | |
| if (!EditorApplication.isPlaying) | |
| return new ErrorResponse("wait_frames requires Play mode."); | |
| int count = Mathf.Clamp(p.GetInt("count") ?? 30, 1, 600); | |
| Debug.Log($"[GameplayBot] MCP wait_frames {count}"); | |
| var token = new BotCompletionToken(); | |
| var sim = GetOrCreateSimulator(); | |
| if (sim == null) return new ErrorResponse("Could not create GameplayInputSimulator."); | |
| sim.EnqueueWait(count, token); | |
| return await WaitForToken(token, 30.0); | |
| } | |
| /// <summary> | |
| /// Polls BotCompletionToken every 50ms until done or timeout. | |
| /// This works reliably because Unity coroutines set IsDone on the main thread, | |
| /// and our Task.Delay loop checks it from the async context. | |
| /// </summary> | |
| private static async Task<object> WaitForToken(BotCompletionToken token, double timeoutSeconds) | |
| { | |
| var startTime = EditorApplication.timeSinceStartup; | |
| while (!token.IsDone) | |
| { | |
| if (EditorApplication.timeSinceStartup - startTime > timeoutSeconds) | |
| return new ErrorResponse($"Action timed out after {timeoutSeconds}s."); | |
| await Task.Delay(50); | |
| } | |
| return token.Success | |
| ? (object)new SuccessResponse(token.Message) | |
| : new ErrorResponse(token.Message); | |
| } | |
| private static GameplayInputSimulator GetOrCreateSimulator() | |
| { | |
| if (GameplayInputSimulator.Instance != null) | |
| return GameplayInputSimulator.Instance; | |
| var found = GameObject.FindFirstObjectByType<GameplayInputSimulator>(); | |
| if (found != null) return found; | |
| try | |
| { | |
| var go = new GameObject("[GameplayBot] InputSimulator"); | |
| var sim = go.AddComponent<GameplayInputSimulator>(); | |
| Debug.Log("[GameplayBot] Auto-created GameplayInputSimulator."); | |
| return sim; | |
| } | |
| catch (Exception ex) | |
| { | |
| Debug.LogError($"[GameplayBot] Failed to create simulator: {ex.Message}"); | |
| return null; | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment