{ "openapi": "3.1.1", "info": { "title": "Screenshot to Steps API", "version": "1.0.0", "summary": "Turn a screenshot of an app or website into numbered steps for what you want to do.", "description": "Every endpoint performs the same calculation as the human page and returns inputs, result, units, assumptions, sources, warnings and computedAt. MCP endpoint: /mcp." }, "servers": [ { "url": "https://screenshottosteps.usefulweb.site" } ], "paths": { "/api/v1/calculate": { "post": { "operationId": "calculate", "summary": "Turn a screenshot of an app or website into numbered steps for what you want to do.", "description": "Upload a screenshot (or paste the visible labels) and say what you want to achieve. The text on screen is read with OCR, each label is classified as button, input field, navigation, link, checkbox or heading with its position, and the elements relevant to your goal are turned into numbered instructions.\n\nMethodology:\n- OCR: tesseract (English) with word positions; words with confidence below 30 are dropped; words on one line are split into separate elements at gaps wider than 1.8× the text height.\n- Roles are assigned by rules: known action words → button; trailing colon or known field names → input field; short labels in the top 12% or known section names → navigation; URLs/\"forgot…\" → link; ☐/[ ] → checkbox; text 1.5× taller than median → heading.\n- Your goal is split into keywords and expanded with synonyms (e.g. password → security, login). Steps: open the best-matching navigation item (if no matching field is visible), fill visible input fields, tick relevant checkboxes, then click the primary button (Save/Continue/OK…).\n- Position is reported as a 3×3 grid region (top/middle/bottom × left/center/right).\n\nSteps are derived only from text visible in the screenshot; icons without labels and hidden menus cannot be seen. Never upload screenshots that show passwords or card numbers.", "tags": [ "internet" ], "x-mcp-tool": "screenshot_to_steps", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScreenshotToStepsInput" }, "examples": { "example1": { "summary": "Change password (labels pasted)", "value": { "goal": "Change my password", "uiText": "Account settings\nProfile\nSecurity\nNotifications\nCurrent password:\nNew password:\nConfirm password:\n[ ] Sign out of all other devices\nCancel\nSave" } }, "example2": { "summary": "Find the settings", "value": { "goal": "turn off notifications", "uiText": "Home\nSearch\nNotifications\nMessages\nSettings\nWhat is happening?\nPost" } } } } } }, "responses": { "200": { "description": "Calculation result envelope.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScreenshotToStepsResult" } } } }, "400": { "description": "Malformed JSON body.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "422": { "description": "Invalid input (see errors[]).", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "429": { "description": "Rate limit exceeded (see Retry-After).", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } } } }, "/api/v1/tool": { "get": { "operationId": "getToolDefinition", "summary": "Inputs, output schema and examples for this tool", "responses": { "200": { "description": "Tool definition.", "content": { "application/json": { "schema": { "type": "object" } } } } } } } }, "components": { "schemas": { "Problem": { "type": "object", "description": "RFC 9457 problem details. 422 responses include per-field `errors`.", "required": [ "type", "title", "status", "detail" ], "properties": { "type": { "type": "string" }, "title": { "type": "string" }, "status": { "type": "integer" }, "detail": { "type": "string" }, "code": { "type": "string" }, "errors": { "type": "array", "items": { "type": "object", "required": [ "field", "message" ], "properties": { "field": { "type": "string" }, "message": { "type": "string" } } } } } }, "ScreenshotToStepsInput": { "type": "object", "additionalProperties": false, "properties": { "goal": { "title": "What do you want to do?", "type": "string", "maxLength": 300 }, "file": { "title": "Screenshot", "description": "PNG or JPG, max 5 MiB. Read with OCR on our server; not stored. Upload as {filename, contentType?, contentBase64}. Max 5120 KiB. Accepts .png,.jpg,.jpeg.", "type": "object", "additionalProperties": false, "required": [ "filename", "contentBase64" ], "properties": { "filename": { "type": "string" }, "contentType": { "type": "string" }, "contentBase64": { "type": "string" } } }, "uiText": { "title": "Or: visible labels, one per line (top to bottom)", "type": "string", "maxLength": 20000 } }, "required": [ "goal" ] }, "ScreenshotToStepsResult": { "type": "object", "additionalProperties": false, "required": [ "schemaVersion", "tool", "computedAt", "inputs", "summary", "result", "units", "assumptions", "sources", "warnings", "presentation", "links" ], "properties": { "schemaVersion": { "type": "string", "const": "1.0" }, "tool": { "type": "object", "additionalProperties": false, "required": [ "id", "name", "version" ], "properties": { "id": { "type": "string" }, "name": { "type": "string" }, "version": { "type": "string" } } }, "computedAt": { "type": "string", "format": "date-time" }, "inputs": { "type": "object", "description": "Normalised inputs actually used, including applied defaults." }, "summary": { "type": "string" }, "result": { "type": "object", "additionalProperties": false, "required": [ "goal", "method", "matchedGoal", "goalTerms", "steps", "elements" ], "properties": { "goal": { "type": "string" }, "method": { "type": "string", "enum": [ "tesseract-ocr", "pasted-labels" ] }, "matchedGoal": { "type": "boolean" }, "goalTerms": { "type": "array", "items": { "type": "string" } }, "steps": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "n", "action", "elementId" ], "properties": { "n": { "type": "integer" }, "action": { "type": "string" }, "elementId": { "type": [ "string", "null" ] } } } }, "elements": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "id", "text", "role", "region", "box", "confidence" ], "properties": { "id": { "type": "string" }, "text": { "type": "string" }, "role": { "type": "string", "enum": [ "button", "field", "navigation", "heading", "link", "checkbox", "text" ] }, "region": { "type": "string" }, "box": { "type": [ "object", "null" ], "additionalProperties": false, "required": [ "left", "top", "width", "height" ], "properties": { "left": { "type": "number" }, "top": { "type": "number" }, "width": { "type": "number" }, "height": { "type": "number" } } }, "confidence": { "type": [ "number", "null" ] } } } } } }, "units": { "type": "object", "additionalProperties": { "type": "string" } }, "assumptions": { "type": "array", "items": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "title" ], "properties": { "title": { "type": "string" }, "url": { "type": "string" }, "publisher": { "type": "string" }, "retrievedAt": { "type": "string" }, "note": { "type": "string" } } } }, "warnings": { "type": "array", "items": { "type": "string" } }, "presentation": { "type": "array", "items": { "type": "object" }, "description": "Display blocks used by the human page (same data)." }, "links": { "type": "object", "additionalProperties": false, "required": [ "human", "api", "openapi", "methodology" ], "properties": { "human": { "type": "string", "format": "uri" }, "api": { "type": "string", "format": "uri" }, "openapi": { "type": "string", "format": "uri" }, "methodology": { "type": "string", "format": "uri" } } } } } } } }