Create Crawl
curl --request POST \
--url https://sdk.nimbleway.com/v2/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"allow_external_links": false,
"allow_subdomains": false,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": false,
"exclude_paths": [
"/exclude-this-path",
"/and-this-path"
],
"extract_options": {
"auto_driver_configuration": {
"vx10": 2,
"vx10-pro": 0,
"vx6-fast": 1,
"vx6-stealth": 1,
"vx8": 5,
"vx8-pro": 5
},
"body": {
"key": "value"
},
"browser": "chrome",
"browser_actions": [
{
"goto": "https://example.com/login"
},
{
"wait_for_element": "#login-form"
},
{
"fill": {
"selector": "#username",
"value": "[email protected]"
}
},
{
"fill": {
"selector": "#password",
"value": "password123"
}
},
{
"click": "#submit"
},
{
"screenshot": {
"full_page": true
}
}
],
"city": "Los Angeles",
"consent_header": true,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": [
"<string>"
],
"hostOnly": true,
"httpOnly": true,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": true,
"secure": true,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"driver": "vx8",
"expected_status_codes": [
200,
201
],
"formats": [
"html"
],
"headers": {
"Accept-Language": "en-US",
"User-Agent": "CustomBot/1.0"
},
"http2": true,
"is_xhr": true,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": false,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"os": "windows",
"parse": true,
"parser": {
"myParser": {
"option1": "value1"
}
},
"referrer_type": "no-referrer",
"render": true,
"request_timeout": 30000,
"session": {
"id": "<string>",
"prefetch_userbrowser": false,
"renew_on_blocked": false,
"retry": false,
"timeout": 123
},
"skill": "dynamic-content",
"state": "CA",
"tag": "campaign-2024-q1",
"url": "<string>"
},
"ignore_query_parameters": false,
"include_paths": [
"/include-this-path",
"/and-this-path"
],
"limit": 100,
"max_discovery_depth": 3,
"name": "The best crawl ever",
"sitemap": "include"
}
'import requests
url = "https://sdk.nimbleway.com/v2/crawl"
payload = {
"url": "<string>",
"allow_external_links": False,
"allow_subdomains": False,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": False,
"exclude_paths": ["/exclude-this-path", "/and-this-path"],
"extract_options": {
"auto_driver_configuration": {
"vx10": 2,
"vx10-pro": 0,
"vx6-fast": 1,
"vx6-stealth": 1,
"vx8": 5,
"vx8-pro": 5
},
"body": { "key": "value" },
"browser": "chrome",
"browser_actions": [{ "goto": "https://example.com/login" }, { "wait_for_element": "#login-form" }, { "fill": {
"selector": "#username",
"value": "[email protected]"
} }, { "fill": {
"selector": "#password",
"value": "password123"
} }, { "click": "#submit" }, { "screenshot": { "full_page": True } }],
"city": "Los Angeles",
"consent_header": True,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": ["<string>"],
"hostOnly": True,
"httpOnly": True,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": True,
"secure": True,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"driver": "vx8",
"expected_status_codes": [200, 201],
"formats": ["html"],
"headers": {
"Accept-Language": "en-US",
"User-Agent": "CustomBot/1.0"
},
"http2": True,
"is_xhr": True,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": False,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"os": "windows",
"parse": True,
"parser": { "myParser": { "option1": "value1" } },
"referrer_type": "no-referrer",
"render": True,
"request_timeout": 30000,
"session": {
"id": "<string>",
"prefetch_userbrowser": False,
"renew_on_blocked": False,
"retry": False,
"timeout": 123
},
"skill": "dynamic-content",
"state": "CA",
"tag": "campaign-2024-q1",
"url": "<string>"
},
"ignore_query_parameters": False,
"include_paths": ["/include-this-path", "/and-this-path"],
"limit": 100,
"max_discovery_depth": 3,
"name": "The best crawl ever",
"sitemap": "include"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
allow_external_links: false,
allow_subdomains: false,
callback: {url: '<string>', events: [], headers: {}, metadata: {}},
crawl_entire_domain: false,
exclude_paths: ['/exclude-this-path', '/and-this-path'],
extract_options: {
auto_driver_configuration: {vx10: 2, 'vx10-pro': 0, 'vx6-fast': 1, 'vx6-stealth': 1, vx8: 5, 'vx8-pro': 5},
body: JSON.stringify({key: 'value'}),
browser: 'chrome',
browser_actions: [
{goto: 'https://example.com/login'},
{wait_for_element: '#login-form'},
{fill: {selector: '#username', value: '[email protected]'}},
{fill: {selector: '#password', value: 'password123'}},
{click: '#submit'},
{screenshot: {full_page: true}}
],
city: 'Los Angeles',
consent_header: true,
cookies: [
{
creation: '<string>',
domain: '<string>',
expires: '<string>',
extensions: ['<string>'],
hostOnly: true,
httpOnly: true,
lastAccessed: '<string>',
maxAge: 123,
name: '<string>',
path: '<string>',
pathIsDefault: true,
secure: true,
value: '<string>'
}
],
country: 'US',
device: 'desktop',
driver: 'vx8',
expected_status_codes: [200, 201],
formats: ['html'],
headers: {'Accept-Language': 'en-US', 'User-Agent': 'CustomBot/1.0'},
http2: true,
is_xhr: true,
locale: 'en-US',
markdown_backend: 'full_page',
method: 'GET',
network_capture: [
{
resource_type: 'document',
status_code: 349.5,
validation: false,
wait_for_requests_count: 0,
wait_for_requests_count_timeout: 123
}
],
os: 'windows',
parse: true,
parser: {myParser: {option1: 'value1'}},
referrer_type: 'no-referrer',
render: true,
request_timeout: 30000,
session: {
id: '<string>',
prefetch_userbrowser: false,
renew_on_blocked: false,
retry: false,
timeout: 123
},
skill: 'dynamic-content',
state: 'CA',
tag: 'campaign-2024-q1',
url: '<string>'
},
ignore_query_parameters: false,
include_paths: ['/include-this-path', '/and-this-path'],
limit: 100,
max_discovery_depth: 3,
name: 'The best crawl ever',
sitemap: 'include'
})
};
fetch('https://sdk.nimbleway.com/v2/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://sdk.nimbleway.com/v2/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'allow_external_links' => false,
'allow_subdomains' => false,
'callback' => [
'url' => '<string>',
'events' => [
],
'headers' => [
],
'metadata' => [
]
],
'crawl_entire_domain' => false,
'exclude_paths' => [
'/exclude-this-path',
'/and-this-path'
],
'extract_options' => [
'auto_driver_configuration' => [
'vx10' => 2,
'vx10-pro' => 0,
'vx6-fast' => 1,
'vx6-stealth' => 1,
'vx8' => 5,
'vx8-pro' => 5
],
'body' => [
'key' => 'value'
],
'browser' => 'chrome',
'browser_actions' => [
[
'goto' => 'https://example.com/login'
],
[
'wait_for_element' => '#login-form'
],
[
'fill' => [
'selector' => '#username',
'value' => '[email protected]'
]
],
[
'fill' => [
'selector' => '#password',
'value' => 'password123'
]
],
[
'click' => '#submit'
],
[
'screenshot' => [
'full_page' => true
]
]
],
'city' => 'Los Angeles',
'consent_header' => true,
'cookies' => [
[
'creation' => '<string>',
'domain' => '<string>',
'expires' => '<string>',
'extensions' => [
'<string>'
],
'hostOnly' => true,
'httpOnly' => true,
'lastAccessed' => '<string>',
'maxAge' => 123,
'name' => '<string>',
'path' => '<string>',
'pathIsDefault' => true,
'secure' => true,
'value' => '<string>'
]
],
'country' => 'US',
'device' => 'desktop',
'driver' => 'vx8',
'expected_status_codes' => [
200,
201
],
'formats' => [
'html'
],
'headers' => [
'Accept-Language' => 'en-US',
'User-Agent' => 'CustomBot/1.0'
],
'http2' => true,
'is_xhr' => true,
'locale' => 'en-US',
'markdown_backend' => 'full_page',
'method' => 'GET',
'network_capture' => [
[
'resource_type' => 'document',
'status_code' => 349.5,
'validation' => false,
'wait_for_requests_count' => 0,
'wait_for_requests_count_timeout' => 123
]
],
'os' => 'windows',
'parse' => true,
'parser' => [
'myParser' => [
'option1' => 'value1'
]
],
'referrer_type' => 'no-referrer',
'render' => true,
'request_timeout' => 30000,
'session' => [
'id' => '<string>',
'prefetch_userbrowser' => false,
'renew_on_blocked' => false,
'retry' => false,
'timeout' => 123
],
'skill' => 'dynamic-content',
'state' => 'CA',
'tag' => 'campaign-2024-q1',
'url' => '<string>'
],
'ignore_query_parameters' => false,
'include_paths' => [
'/include-this-path',
'/and-this-path'
],
'limit' => 100,
'max_discovery_depth' => 3,
'name' => 'The best crawl ever',
'sitemap' => 'include'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://sdk.nimbleway.com/v2/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://sdk.nimbleway.com/v2/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://sdk.nimbleway.com/v2/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}"
response = http.request(request)
puts response.read_body{
"account_name": "<string>",
"crawl_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"crawl_options": {
"allow_external_links": false,
"allow_subdomains": false,
"crawl_entire_domain": false,
"ignore_query_parameters": false,
"limit": 5000,
"max_discovery_depth": 5,
"sitemap": "include",
"callback": {
"url": "<string>",
"events": [
"started"
],
"headers": {},
"metadata": {}
},
"exclude_paths": [
"<string>"
],
"include_paths": [
"<string>"
]
},
"created_at": "<string>",
"status": "queued",
"updated_at": "<string>",
"url": "<string>",
"completed": 123,
"completed_at": "<string>",
"extract_options": {},
"failed": 123,
"name": "<string>",
"pending": 123,
"tasks": [
{
"status": "pending",
"task_id": "<string>",
"created_at": "<string>",
"updated_at": "<string>"
}
],
"total": 123
}Create Crawl
POST
/
v2
/
crawl
Create Crawl
curl --request POST \
--url https://sdk.nimbleway.com/v2/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"allow_external_links": false,
"allow_subdomains": false,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": false,
"exclude_paths": [
"/exclude-this-path",
"/and-this-path"
],
"extract_options": {
"auto_driver_configuration": {
"vx10": 2,
"vx10-pro": 0,
"vx6-fast": 1,
"vx6-stealth": 1,
"vx8": 5,
"vx8-pro": 5
},
"body": {
"key": "value"
},
"browser": "chrome",
"browser_actions": [
{
"goto": "https://example.com/login"
},
{
"wait_for_element": "#login-form"
},
{
"fill": {
"selector": "#username",
"value": "[email protected]"
}
},
{
"fill": {
"selector": "#password",
"value": "password123"
}
},
{
"click": "#submit"
},
{
"screenshot": {
"full_page": true
}
}
],
"city": "Los Angeles",
"consent_header": true,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": [
"<string>"
],
"hostOnly": true,
"httpOnly": true,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": true,
"secure": true,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"driver": "vx8",
"expected_status_codes": [
200,
201
],
"formats": [
"html"
],
"headers": {
"Accept-Language": "en-US",
"User-Agent": "CustomBot/1.0"
},
"http2": true,
"is_xhr": true,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": false,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"os": "windows",
"parse": true,
"parser": {
"myParser": {
"option1": "value1"
}
},
"referrer_type": "no-referrer",
"render": true,
"request_timeout": 30000,
"session": {
"id": "<string>",
"prefetch_userbrowser": false,
"renew_on_blocked": false,
"retry": false,
"timeout": 123
},
"skill": "dynamic-content",
"state": "CA",
"tag": "campaign-2024-q1",
"url": "<string>"
},
"ignore_query_parameters": false,
"include_paths": [
"/include-this-path",
"/and-this-path"
],
"limit": 100,
"max_discovery_depth": 3,
"name": "The best crawl ever",
"sitemap": "include"
}
'import requests
url = "https://sdk.nimbleway.com/v2/crawl"
payload = {
"url": "<string>",
"allow_external_links": False,
"allow_subdomains": False,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": False,
"exclude_paths": ["/exclude-this-path", "/and-this-path"],
"extract_options": {
"auto_driver_configuration": {
"vx10": 2,
"vx10-pro": 0,
"vx6-fast": 1,
"vx6-stealth": 1,
"vx8": 5,
"vx8-pro": 5
},
"body": { "key": "value" },
"browser": "chrome",
"browser_actions": [{ "goto": "https://example.com/login" }, { "wait_for_element": "#login-form" }, { "fill": {
"selector": "#username",
"value": "[email protected]"
} }, { "fill": {
"selector": "#password",
"value": "password123"
} }, { "click": "#submit" }, { "screenshot": { "full_page": True } }],
"city": "Los Angeles",
"consent_header": True,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": ["<string>"],
"hostOnly": True,
"httpOnly": True,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": True,
"secure": True,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"driver": "vx8",
"expected_status_codes": [200, 201],
"formats": ["html"],
"headers": {
"Accept-Language": "en-US",
"User-Agent": "CustomBot/1.0"
},
"http2": True,
"is_xhr": True,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": False,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"os": "windows",
"parse": True,
"parser": { "myParser": { "option1": "value1" } },
"referrer_type": "no-referrer",
"render": True,
"request_timeout": 30000,
"session": {
"id": "<string>",
"prefetch_userbrowser": False,
"renew_on_blocked": False,
"retry": False,
"timeout": 123
},
"skill": "dynamic-content",
"state": "CA",
"tag": "campaign-2024-q1",
"url": "<string>"
},
"ignore_query_parameters": False,
"include_paths": ["/include-this-path", "/and-this-path"],
"limit": 100,
"max_discovery_depth": 3,
"name": "The best crawl ever",
"sitemap": "include"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
allow_external_links: false,
allow_subdomains: false,
callback: {url: '<string>', events: [], headers: {}, metadata: {}},
crawl_entire_domain: false,
exclude_paths: ['/exclude-this-path', '/and-this-path'],
extract_options: {
auto_driver_configuration: {vx10: 2, 'vx10-pro': 0, 'vx6-fast': 1, 'vx6-stealth': 1, vx8: 5, 'vx8-pro': 5},
body: JSON.stringify({key: 'value'}),
browser: 'chrome',
browser_actions: [
{goto: 'https://example.com/login'},
{wait_for_element: '#login-form'},
{fill: {selector: '#username', value: '[email protected]'}},
{fill: {selector: '#password', value: 'password123'}},
{click: '#submit'},
{screenshot: {full_page: true}}
],
city: 'Los Angeles',
consent_header: true,
cookies: [
{
creation: '<string>',
domain: '<string>',
expires: '<string>',
extensions: ['<string>'],
hostOnly: true,
httpOnly: true,
lastAccessed: '<string>',
maxAge: 123,
name: '<string>',
path: '<string>',
pathIsDefault: true,
secure: true,
value: '<string>'
}
],
country: 'US',
device: 'desktop',
driver: 'vx8',
expected_status_codes: [200, 201],
formats: ['html'],
headers: {'Accept-Language': 'en-US', 'User-Agent': 'CustomBot/1.0'},
http2: true,
is_xhr: true,
locale: 'en-US',
markdown_backend: 'full_page',
method: 'GET',
network_capture: [
{
resource_type: 'document',
status_code: 349.5,
validation: false,
wait_for_requests_count: 0,
wait_for_requests_count_timeout: 123
}
],
os: 'windows',
parse: true,
parser: {myParser: {option1: 'value1'}},
referrer_type: 'no-referrer',
render: true,
request_timeout: 30000,
session: {
id: '<string>',
prefetch_userbrowser: false,
renew_on_blocked: false,
retry: false,
timeout: 123
},
skill: 'dynamic-content',
state: 'CA',
tag: 'campaign-2024-q1',
url: '<string>'
},
ignore_query_parameters: false,
include_paths: ['/include-this-path', '/and-this-path'],
limit: 100,
max_discovery_depth: 3,
name: 'The best crawl ever',
sitemap: 'include'
})
};
fetch('https://sdk.nimbleway.com/v2/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://sdk.nimbleway.com/v2/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'allow_external_links' => false,
'allow_subdomains' => false,
'callback' => [
'url' => '<string>',
'events' => [
],
'headers' => [
],
'metadata' => [
]
],
'crawl_entire_domain' => false,
'exclude_paths' => [
'/exclude-this-path',
'/and-this-path'
],
'extract_options' => [
'auto_driver_configuration' => [
'vx10' => 2,
'vx10-pro' => 0,
'vx6-fast' => 1,
'vx6-stealth' => 1,
'vx8' => 5,
'vx8-pro' => 5
],
'body' => [
'key' => 'value'
],
'browser' => 'chrome',
'browser_actions' => [
[
'goto' => 'https://example.com/login'
],
[
'wait_for_element' => '#login-form'
],
[
'fill' => [
'selector' => '#username',
'value' => '[email protected]'
]
],
[
'fill' => [
'selector' => '#password',
'value' => 'password123'
]
],
[
'click' => '#submit'
],
[
'screenshot' => [
'full_page' => true
]
]
],
'city' => 'Los Angeles',
'consent_header' => true,
'cookies' => [
[
'creation' => '<string>',
'domain' => '<string>',
'expires' => '<string>',
'extensions' => [
'<string>'
],
'hostOnly' => true,
'httpOnly' => true,
'lastAccessed' => '<string>',
'maxAge' => 123,
'name' => '<string>',
'path' => '<string>',
'pathIsDefault' => true,
'secure' => true,
'value' => '<string>'
]
],
'country' => 'US',
'device' => 'desktop',
'driver' => 'vx8',
'expected_status_codes' => [
200,
201
],
'formats' => [
'html'
],
'headers' => [
'Accept-Language' => 'en-US',
'User-Agent' => 'CustomBot/1.0'
],
'http2' => true,
'is_xhr' => true,
'locale' => 'en-US',
'markdown_backend' => 'full_page',
'method' => 'GET',
'network_capture' => [
[
'resource_type' => 'document',
'status_code' => 349.5,
'validation' => false,
'wait_for_requests_count' => 0,
'wait_for_requests_count_timeout' => 123
]
],
'os' => 'windows',
'parse' => true,
'parser' => [
'myParser' => [
'option1' => 'value1'
]
],
'referrer_type' => 'no-referrer',
'render' => true,
'request_timeout' => 30000,
'session' => [
'id' => '<string>',
'prefetch_userbrowser' => false,
'renew_on_blocked' => false,
'retry' => false,
'timeout' => 123
],
'skill' => 'dynamic-content',
'state' => 'CA',
'tag' => 'campaign-2024-q1',
'url' => '<string>'
],
'ignore_query_parameters' => false,
'include_paths' => [
'/include-this-path',
'/and-this-path'
],
'limit' => 100,
'max_discovery_depth' => 3,
'name' => 'The best crawl ever',
'sitemap' => 'include'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://sdk.nimbleway.com/v2/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://sdk.nimbleway.com/v2/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://sdk.nimbleway.com/v2/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"/exclude-this-path\",\n \"/and-this-path\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {\n \"vx10\": 2,\n \"vx10-pro\": 0,\n \"vx6-fast\": 1,\n \"vx6-stealth\": 1,\n \"vx8\": 5,\n \"vx8-pro\": 5\n },\n \"body\": {\n \"key\": \"value\"\n },\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"Los Angeles\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"driver\": \"vx8\",\n \"expected_status_codes\": [\n 200,\n 201\n ],\n \"formats\": [\n \"html\"\n ],\n \"headers\": {\n \"Accept-Language\": \"en-US\",\n \"User-Agent\": \"CustomBot/1.0\"\n },\n \"http2\": true,\n \"is_xhr\": true,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"os\": \"windows\",\n \"parse\": true,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 30000,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"state\": \"CA\",\n \"tag\": \"campaign-2024-q1\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"/include-this-path\",\n \"/and-this-path\"\n ],\n \"limit\": 100,\n \"max_discovery_depth\": 3,\n \"name\": \"The best crawl ever\",\n \"sitemap\": \"include\"\n}"
response = http.request(request)
puts response.read_body{
"account_name": "<string>",
"crawl_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"crawl_options": {
"allow_external_links": false,
"allow_subdomains": false,
"crawl_entire_domain": false,
"ignore_query_parameters": false,
"limit": 5000,
"max_discovery_depth": 5,
"sitemap": "include",
"callback": {
"url": "<string>",
"events": [
"started"
],
"headers": {},
"metadata": {}
},
"exclude_paths": [
"<string>"
],
"include_paths": [
"<string>"
]
},
"created_at": "<string>",
"status": "queued",
"updated_at": "<string>",
"url": "<string>",
"completed": 123,
"completed_at": "<string>",
"extract_options": {},
"failed": 123,
"name": "<string>",
"pending": 123,
"tasks": [
{
"status": "pending",
"task_id": "<string>",
"created_at": "<string>",
"updated_at": "<string>"
}
],
"total": 123
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
Request body model for the /crawl endpoint
Url to crawl.
Allows the crawler to follow links to external websites.
Example:
false
Allows the crawler to follow links to subdomains of the main domain.
Example:
false
Webhook configuration for receiving crawl results.
Show child attributes
Show child attributes
Allows the crawler to follow internal links to sibling or parent URLs, not just child paths.
Example:
false
URL pathname regex patterns that exclude matching URLs from the crawl.
Example:
["/exclude-this-path", "/and-this-path"]
Show child attributes
Show child attributes
Do not re-scrape the same path with different (or none) query parameters.
Example:
false
URL pathname regex patterns that include matching URLs in the crawl.
Example:
["/include-this-path", "/and-this-path"]
Maximum number of pages to crawl.
Required range:
1 <= x <= 10000Example:
100
Maximum depth to crawl based on discovery order.
Required range:
1 <= x <= 20Example:
3
Name of the crawl.
Example:
"The best crawl ever"
Sitemap and other methods will be used together to find URLs.
Available options:
skip, include, only Example:
"include"
Response
200 - application/json
Successful Response - Crawl Task Created
Crawl API response
Show child attributes
Show child attributes
Available options:
queued, running, succeeded, failed, canceled Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?