{ "openapi": "3.1.0", "info": { "title": "ScrapeGoat Hosted API", "version": "1.0.0", "description": "Authenticated fetch and account-owned request history. Costs are integer micro-USD. This is not a drop-in competitor API.", "license": { "name": "Proprietary" } }, "jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema", "servers": [ { "url": "/", "description": "Use the origin of your ScrapeGoat deployment" } ], "security": [ { "bearerAuth": [] } ], "tags": [ { "name": "Scraping" }, { "name": "History" } ], "paths": { "/v1/fetch": { "post": { "tags": [ "Scraping" ], "operationId": "fetch", "summary": "Fetch and validate one public URL", "description": "Bearer key required. Reserves the provider budget plus a 20% fee. Failed attempts can be billable. Admitted IDs cannot be replayed. Maximum JSON request body: 65536 bytes. Maximum response: 16 MiB. Limits: 60 requests/minute and 5 pending requests per account.", "parameters": [ { "name": "X-Request-ID", "in": "header", "required": false, "description": "Canonical req_ ID or UUID v4 from a reverse proxy. UUIDs map deterministically to canonical IDs. Use the returned X-Request-ID for history. Repeated IDs return 409.", "schema": { "oneOf": [ { "$ref": "#/components/schemas/RequestID" }, { "type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89aAbB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" } ] } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FetchRequest" } } } }, "responses": { "200": { "description": "Validated content. Provider cost remains in the body; the charge including the service fee is in the response header.", "headers": { "X-Request-ID": { "description": "Stable request ID used by the body, logs, history, and support.", "schema": { "$ref": "#/components/schemas/RequestID" } }, "X-Charged-Micro-USD": { "description": "Settled charge including fee; absent when pending.", "schema": { "type": "integer", "minimum": 0 } }, "X-Billing-Status": { "description": "Cost remains uncertain and reserved funds remain held.", "schema": { "const": "pending" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FetchResponse" } } } }, "400": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "401": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "402": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "404": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "408": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "409": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "413": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "415": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "421": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "422": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "429": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "500": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "502": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "503": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "504": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } } } } }, "/v1/requests/{id}": { "get": { "tags": [ "History" ], "operationId": "getRequest", "summary": "Read account-owned request metadata", "description": "Page content and target query values are not stored. An uncertain admitted request is not replayed.", "parameters": [ { "in": "path", "name": "id", "required": true, "schema": { "$ref": "#/components/schemas/RequestID" } } ], "responses": { "200": { "description": "Account-owned metadata and attempts", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HostedRequest" } } } }, "401": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "404": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "421": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "500": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } }, "503": { "description": "Hosted authentication, budget, target, rate, or core routing error. A pending billing status requires history inspection before retry.", "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/HostedError" }, { "$ref": "#/components/schemas/ErrorResponse" } ] } } } } } } } }, "components": { "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "sr_ API key" } }, "schemas": { "Attempt": { "type": "object", "additionalProperties": false, "required": [ "number", "route_id", "provider_id", "started_at", "ended_at", "latency_ms", "outcome", "cost" ], "properties": { "number": { "type": "integer", "minimum": 1, "maximum": 20 }, "route_id": { "$ref": "#/components/schemas/RouteID" }, "provider_id": { "$ref": "#/components/schemas/ProviderID" }, "started_at": { "type": "string", "format": "date-time" }, "ended_at": { "type": "string", "format": "date-time" }, "latency_ms": { "$ref": "#/components/schemas/Milliseconds" }, "outcome": { "type": "string", "enum": [ "valid", "transport_failure", "unusable_status", "challenge", "validation_failure", "provider_failure", "body_limit", "canceled" ] }, "provider_status_code": { "type": "integer", "minimum": 100, "maximum": 599 }, "target_status_code": { "type": "integer", "minimum": 100, "maximum": 599 }, "cost": { "$ref": "#/components/schemas/Cost" }, "validation": { "$ref": "#/components/schemas/ValidationEvidence" }, "failure": { "$ref": "#/components/schemas/ErrorDetail" }, "provider_request_id": { "type": "string", "maxLength": 256 } } }, "BinaryAsset": { "type": "object", "additionalProperties": false, "required": [ "media_type" ], "oneOf": [ { "required": [ "data_base64" ], "not": { "required": [ "url" ] } }, { "required": [ "url" ], "not": { "required": [ "data_base64" ] } } ], "properties": { "media_type": { "type": "string", "enum": [ "image/png", "image/jpeg", "image/webp" ] }, "data_base64": { "type": "string", "contentEncoding": "base64", "contentMediaType": "application/octet-stream" }, "url": { "type": "string", "format": "uri", "pattern": "^https://", "maxLength": 2048 } } }, "Budget": { "type": "object", "additionalProperties": false, "properties": { "cost_ceiling_micro_usd": { "allOf": [ { "$ref": "#/components/schemas/MicroUSD" } ], "minimum": 1, "default": 50000, "maximum": 1000000 }, "max_attempts": { "type": "integer", "minimum": 1, "maximum": 20, "default": 5 } }, "required": [ "cost_ceiling_micro_usd" ] }, "ByteCount": { "type": "integer", "format": "int64", "minimum": 0 }, "Capability": { "type": "string", "enum": [ "html", "javascript", "markdown", "screenshot", "schema_json", "geo", "premium_proxy", "anti_bot", "custom_headers", "sensitive_headers" ] }, "CheckKind": { "type": "string", "enum": [ "status", "size", "content_type", "challenge", "required_text", "forbidden_text", "format" ] }, "Content": { "type": "object", "additionalProperties": false, "minProperties": 1, "properties": { "html": { "type": "string" }, "markdown": { "type": "string" }, "screenshot": { "$ref": "#/components/schemas/BinaryAsset" }, "json": true } }, "Cost": { "type": "object", "additionalProperties": false, "required": [ "kind", "billing_basis" ], "properties": { "micro_usd": { "$ref": "#/components/schemas/MicroUSD" }, "kind": { "type": "string", "enum": [ "actual", "estimated", "mixed", "unknown" ] }, "billing_basis": { "type": "string", "enum": [ "per_request", "per_gb", "browser_hour", "credit", "monthly_speed", "mixed", "none", "unknown" ] }, "billable_quantity": { "type": "integer", "format": "int64", "minimum": 0 }, "billable_unit": { "type": "string", "maxLength": 64 } } }, "ErrorDetail": { "type": "object", "additionalProperties": false, "required": [ "code", "class", "message", "retryable" ], "properties": { "code": { "type": "string", "enum": [ "invalid_request", "malformed_json", "unknown_field", "trailing_json", "request_body_too_large", "unsupported_media_type", "endpoint_not_found", "method_not_allowed", "overloaded", "strategy_not_found", "request_not_found", "profile_not_found", "conflict", "capability_unavailable", "strategy_unavailable", "budget_exceeded", "validation_failed", "upstream_failure", "upstream_timeout", "upstream_response_too_large", "request_canceled", "not_ready", "internal_error" ] }, "class": { "type": "string", "enum": [ "canceled", "deadline", "temporary_dns", "dns", "connect", "connection_reset", "tls", "rate_limited", "upstream_4xx", "upstream_5xx", "provider_rejected", "decode", "body_limit", "unavailable", "capability", "budget", "validation", "not_found", "conflict", "invalid", "internal" ] }, "message": { "type": "string", "minLength": 1, "maxLength": 512 }, "retryable": { "type": "boolean" }, "route_id": { "$ref": "#/components/schemas/RouteID" }, "provider_id": { "$ref": "#/components/schemas/ProviderID" }, "retry_after_ms": { "$ref": "#/components/schemas/Milliseconds" } } }, "ErrorResponse": { "type": "object", "additionalProperties": false, "required": [ "request_id", "error" ], "properties": { "request_id": { "$ref": "#/components/schemas/RequestID" }, "error": { "$ref": "#/components/schemas/ErrorDetail" } } }, "FetchRequest": { "type": "object", "additionalProperties": false, "required": [ "url" ], "allOf": [ { "if": { "required": [ "formats" ], "properties": { "formats": { "contains": { "const": "json" } } } }, "then": { "required": [ "json_schema" ], "properties": { "json_schema": { "minProperties": 1 } } }, "else": { "not": { "required": [ "json_schema" ] } } } ], "properties": { "url": { "type": "string", "format": "uri", "minLength": 8, "maxLength": 8192, "description": "Public HTTP(S) URL on port 80 or 443; maximum 8192 UTF-8 bytes. Private addresses are rejected." }, "formats": { "type": "array", "items": { "$ref": "#/components/schemas/Format" }, "minItems": 1, "maxItems": 4, "uniqueItems": true, "default": [ "markdown" ] }, "headers": { "$ref": "#/components/schemas/HeaderMap" }, "requirements": { "$ref": "#/components/schemas/RouteRequirements" }, "validation": { "$ref": "#/components/schemas/ValidationPolicy" }, "budget": { "$ref": "#/components/schemas/Budget", "description": "If omitted: 50000 micro-USD and 3 attempts. A supplied budget requires cost_ceiling_micro_usd." }, "strategy": { "oneOf": [ { "const": "auto" }, { "$ref": "#/components/schemas/RouteID" } ], "default": "auto" }, "json_schema": { "type": "object", "description": "Draft 2020-12 JSON Schema. Required exactly when `json` is requested." } } }, "FetchResponse": { "type": "object", "additionalProperties": false, "required": [ "request_id", "profile", "selected_strategy", "content", "attempts", "cost", "validation", "latency_ms" ], "properties": { "request_id": { "$ref": "#/components/schemas/RequestID" }, "profile": { "$ref": "#/components/schemas/ProfileKey" }, "selected_strategy": { "$ref": "#/components/schemas/RouteID" }, "content": { "$ref": "#/components/schemas/Content" }, "attempts": { "type": "array", "items": { "$ref": "#/components/schemas/Attempt" }, "minItems": 1 }, "cost": { "$ref": "#/components/schemas/Cost" }, "validation": { "$ref": "#/components/schemas/ValidationEvidence" }, "latency_ms": { "$ref": "#/components/schemas/Milliseconds" } } }, "Format": { "type": "string", "enum": [ "html", "markdown", "screenshot", "json" ] }, "Geo": { "type": "string", "description": "Lower-case ISO 3166-1 alpha-2 code, or `any`.", "pattern": "^(any|[a-z]{2})$", "default": "any" }, "HeaderMap": { "type": "object", "propertyNames": { "pattern": "^[!#$%&'*+.^_`|~0-9A-Za-z-]+$" }, "additionalProperties": { "type": "string", "maxLength": 65536 }, "maxProperties": 256, "default": {} }, "HostedAttempt": { "type": "object", "additionalProperties": false, "required": [ "number", "route", "outcome", "latency_ms", "cost_micro_usd", "cost_kind", "http_status" ], "properties": { "number": { "type": "integer" }, "route": { "type": "string" }, "outcome": { "type": "string" }, "latency_ms": { "type": "integer" }, "cost_micro_usd": { "type": [ "integer", "null" ], "minimum": 0 }, "cost_kind": { "type": "string" }, "http_status": { "type": [ "integer", "null" ] }, "error_code": { "type": "string" } } }, "HostedError": { "type": "object", "additionalProperties": false, "required": [ "error" ], "properties": { "error": { "type": "object", "additionalProperties": false, "required": [ "code", "message" ], "properties": { "code": { "type": "string" }, "message": { "type": "string" } } } } }, "HostedRequest": { "type": "object", "additionalProperties": false, "required": [ "id", "key_id", "host", "resource", "format", "started_at", "status", "http_status", "latency_ms", "route", "reserved_micro_usd", "charged_micro_usd", "provider_micro_usd", "cost_kind", "attempts", "error_code" ], "properties": { "id": { "$ref": "#/components/schemas/RequestID" }, "key_id": { "type": [ "string", "null" ] }, "host": { "type": "string" }, "resource": { "type": "string" }, "format": { "type": "string" }, "started_at": { "type": "integer", "description": "Unix milliseconds" }, "status": { "enum": [ "pending", "succeeded", "failed", "canceled" ] }, "http_status": { "type": "integer" }, "latency_ms": { "type": "integer" }, "route": { "type": "string" }, "reserved_micro_usd": { "type": "integer", "minimum": 1 }, "charged_micro_usd": { "type": [ "integer", "null" ], "minimum": 0 }, "provider_micro_usd": { "type": [ "integer", "null" ], "minimum": 0 }, "cost_kind": { "enum": [ "actual", "estimated", "unknown" ] }, "attempts": { "type": "array", "items": { "$ref": "#/components/schemas/HostedAttempt" } }, "error_code": { "type": "string" } } }, "MicroUSD": { "type": "integer", "format": "int64", "minimum": 0, "description": "Integer millionths of one USD." }, "Milliseconds": { "type": "integer", "format": "int64", "minimum": 0 }, "ProfileKey": { "type": "object", "additionalProperties": false, "required": [ "host", "resource_class", "geo", "rendering" ], "properties": { "host": { "type": "string", "minLength": 1, "maxLength": 261 }, "resource_class": { "$ref": "#/components/schemas/ResourceClass" }, "geo": { "$ref": "#/components/schemas/Geo" }, "rendering": { "$ref": "#/components/schemas/RenderingRequirement" } } }, "ProviderID": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z0-9][a-z0-9-]*$" }, "RenderingRequirement": { "type": "string", "enum": [ "none", "required" ], "default": "none" }, "RequestID": { "type": "string", "pattern": "^req_[0-7][0-9A-HJKMNP-TV-Z]{25}$", "example": "req_01K3T6F8N7M9Q2W4R6Y8B0D2F4" }, "ResourceClass": { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^/", "example": "/products/:int?color" }, "RouteID": { "type": "string", "minLength": 3, "maxLength": 128, "pattern": "^[a-z0-9][a-z0-9-]*(/[a-z0-9][a-z0-9-]*)+$", "example": "firecrawl/auto" }, "RouteRequirements": { "type": "object", "additionalProperties": false, "properties": { "geo": { "$ref": "#/components/schemas/Geo" }, "rendering": { "$ref": "#/components/schemas/RenderingRequirement" }, "required_capabilities": { "type": "array", "items": { "$ref": "#/components/schemas/Capability" }, "uniqueItems": true, "maxItems": 10, "default": [] }, "latency_slo_ms": { "allOf": [ { "$ref": "#/components/schemas/Milliseconds" } ], "minimum": 1, "maximum": 300000, "default": 10000 } }, "default": {} }, "ValidationCheck": { "type": "object", "additionalProperties": false, "required": [ "kind", "outcome", "reason" ], "properties": { "kind": { "$ref": "#/components/schemas/CheckKind" }, "outcome": { "type": "string", "enum": [ "passed", "failed", "skipped" ] }, "reason": { "type": "string", "enum": [ "status_accepted", "status_rejected", "status_unavailable", "size_ok", "body_empty", "body_too_small", "body_too_large", "content_type_not_required", "content_type_matches", "content_type_mismatch", "challenge_not_applicable", "challenge_none", "challenge_cloudflare", "challenge_captcha", "challenge_access_denied", "challenge_login_wall", "challenge_empty_shell", "challenge_javascript_required", "required_text_present", "required_text_missing", "forbidden_text_absent", "forbidden_text_present", "format_present", "format_missing" ] }, "observed": { "type": "string", "maxLength": 256 }, "expected": { "type": "string", "maxLength": 256 } } }, "ValidationEvidence": { "type": "object", "additionalProperties": false, "required": [ "valid", "checks" ], "properties": { "valid": { "type": "boolean" }, "checks": { "type": "array", "items": { "$ref": "#/components/schemas/ValidationCheck" }, "minItems": 4 } } }, "ValidationPolicy": { "type": "object", "additionalProperties": false, "properties": { "allowed_status_codes": { "type": "array", "items": { "type": "integer", "minimum": 200, "maximum": 299 }, "uniqueItems": true, "maxItems": 100, "default": [] }, "min_bytes": { "allOf": [ { "$ref": "#/components/schemas/ByteCount" } ], "default": 1 }, "max_bytes": { "allOf": [ { "$ref": "#/components/schemas/ByteCount" } ], "minimum": 1, "default": 10485760 }, "required_text": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 1024 }, "maxItems": 32, "default": [] }, "forbidden_text": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 1024 }, "maxItems": 32, "default": [] }, "content_type": { "type": "string", "pattern": "^[A-Za-z0-9!#$&^_.+-]+/(\\*|[A-Za-z0-9!#$&^_.+-]+)$", "maxLength": 127 } }, "default": {} } } } }