curl --request POST \
--url https://sdk.nimbleway.com/v2/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"allow_external_links": false,
"allow_subdomains": false,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": false,
"exclude_paths": [
"<string>"
],
"extract_options": {
"auto_driver_configuration": {},
"body": "<unknown>",
"browser": "chrome",
"browser_actions": [
{
"goto": "https://example.com/login"
},
{
"wait_for_element": "#login-form"
},
{
"fill": {
"selector": "#username",
"value": "[email protected]"
}
},
{
"fill": {
"selector": "#password",
"value": "password123"
}
},
{
"click": "#submit"
},
{
"screenshot": {
"full_page": true
}
}
],
"city": "<string>",
"consent_header": true,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": [
"<string>"
],
"hostOnly": true,
"httpOnly": true,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": true,
"secure": true,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"expected_status_codes": [
0
],
"formats": [],
"headers": {},
"http2": false,
"is_xhr": false,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": false,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"parse": false,
"parser": {
"myParser": {
"option1": "value1"
}
},
"referrer_type": "no-referrer",
"render": true,
"request_timeout": 2,
"session": {
"id": "<string>",
"prefetch_userbrowser": false,
"renew_on_blocked": false,
"retry": false,
"timeout": 123
},
"skill": "dynamic-content",
"tag": "<string>",
"url": "<string>"
},
"ignore_query_parameters": false,
"include_paths": [
"<string>"
],
"limit": 5000,
"max_discovery_depth": 5,
"name": "<string>",
"sitemap": "include"
}
'import requests
url = "https://sdk.nimbleway.com/v2/crawl"
payload = {
"url": "<string>",
"allow_external_links": False,
"allow_subdomains": False,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": False,
"exclude_paths": ["<string>"],
"extract_options": {
"auto_driver_configuration": {},
"body": "<unknown>",
"browser": "chrome",
"browser_actions": [{ "goto": "https://example.com/login" }, { "wait_for_element": "#login-form" }, { "fill": {
"selector": "#username",
"value": "[email protected]"
} }, { "fill": {
"selector": "#password",
"value": "password123"
} }, { "click": "#submit" }, { "screenshot": { "full_page": True } }],
"city": "<string>",
"consent_header": True,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": ["<string>"],
"hostOnly": True,
"httpOnly": True,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": True,
"secure": True,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"expected_status_codes": [0],
"formats": [],
"headers": {},
"http2": False,
"is_xhr": False,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": False,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"parse": False,
"parser": { "myParser": { "option1": "value1" } },
"referrer_type": "no-referrer",
"render": True,
"request_timeout": 2,
"session": {
"id": "<string>",
"prefetch_userbrowser": False,
"renew_on_blocked": False,
"retry": False,
"timeout": 123
},
"skill": "dynamic-content",
"tag": "<string>",
"url": "<string>"
},
"ignore_query_parameters": False,
"include_paths": ["<string>"],
"limit": 5000,
"max_discovery_depth": 5,
"name": "<string>",
"sitemap": "include"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
allow_external_links: false,
allow_subdomains: false,
callback: {url: '<string>', events: [], headers: {}, metadata: {}},
crawl_entire_domain: false,
exclude_paths: ['<string>'],
extract_options: {
auto_driver_configuration: {},
body: JSON.stringify('<unknown>'),
browser: 'chrome',
browser_actions: [
{goto: 'https://example.com/login'},
{wait_for_element: '#login-form'},
{fill: {selector: '#username', value: '[email protected]'}},
{fill: {selector: '#password', value: 'password123'}},
{click: '#submit'},
{screenshot: {full_page: true}}
],
city: '<string>',
consent_header: true,
cookies: [
{
creation: '<string>',
domain: '<string>',
expires: '<string>',
extensions: ['<string>'],
hostOnly: true,
httpOnly: true,
lastAccessed: '<string>',
maxAge: 123,
name: '<string>',
path: '<string>',
pathIsDefault: true,
secure: true,
value: '<string>'
}
],
country: 'US',
device: 'desktop',
expected_status_codes: [0],
formats: [],
headers: {},
http2: false,
is_xhr: false,
locale: 'en-US',
markdown_backend: 'full_page',
method: 'GET',
network_capture: [
{
resource_type: 'document',
status_code: 349.5,
validation: false,
wait_for_requests_count: 0,
wait_for_requests_count_timeout: 123
}
],
parse: false,
parser: {myParser: {option1: 'value1'}},
referrer_type: 'no-referrer',
render: true,
request_timeout: 2,
session: {
id: '<string>',
prefetch_userbrowser: false,
renew_on_blocked: false,
retry: false,
timeout: 123
},
skill: 'dynamic-content',
tag: '<string>',
url: '<string>'
},
ignore_query_parameters: false,
include_paths: ['<string>'],
limit: 5000,
max_discovery_depth: 5,
name: '<string>',
sitemap: 'include'
})
};
fetch('https://sdk.nimbleway.com/v2/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://sdk.nimbleway.com/v2/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'allow_external_links' => false,
'allow_subdomains' => false,
'callback' => [
'url' => '<string>',
'events' => [
],
'headers' => [
],
'metadata' => [
]
],
'crawl_entire_domain' => false,
'exclude_paths' => [
'<string>'
],
'extract_options' => [
'auto_driver_configuration' => [
],
'body' => '<unknown>',
'browser' => 'chrome',
'browser_actions' => [
[
'goto' => 'https://example.com/login'
],
[
'wait_for_element' => '#login-form'
],
[
'fill' => [
'selector' => '#username',
'value' => '[email protected]'
]
],
[
'fill' => [
'selector' => '#password',
'value' => 'password123'
]
],
[
'click' => '#submit'
],
[
'screenshot' => [
'full_page' => true
]
]
],
'city' => '<string>',
'consent_header' => true,
'cookies' => [
[
'creation' => '<string>',
'domain' => '<string>',
'expires' => '<string>',
'extensions' => [
'<string>'
],
'hostOnly' => true,
'httpOnly' => true,
'lastAccessed' => '<string>',
'maxAge' => 123,
'name' => '<string>',
'path' => '<string>',
'pathIsDefault' => true,
'secure' => true,
'value' => '<string>'
]
],
'country' => 'US',
'device' => 'desktop',
'expected_status_codes' => [
0
],
'formats' => [
],
'headers' => [
],
'http2' => false,
'is_xhr' => false,
'locale' => 'en-US',
'markdown_backend' => 'full_page',
'method' => 'GET',
'network_capture' => [
[
'resource_type' => 'document',
'status_code' => 349.5,
'validation' => false,
'wait_for_requests_count' => 0,
'wait_for_requests_count_timeout' => 123
]
],
'parse' => false,
'parser' => [
'myParser' => [
'option1' => 'value1'
]
],
'referrer_type' => 'no-referrer',
'render' => true,
'request_timeout' => 2,
'session' => [
'id' => '<string>',
'prefetch_userbrowser' => false,
'renew_on_blocked' => false,
'retry' => false,
'timeout' => 123
],
'skill' => 'dynamic-content',
'tag' => '<string>',
'url' => '<string>'
],
'ignore_query_parameters' => false,
'include_paths' => [
'<string>'
],
'limit' => 5000,
'max_discovery_depth' => 5,
'name' => '<string>',
'sitemap' => 'include'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://sdk.nimbleway.com/v2/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://sdk.nimbleway.com/v2/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://sdk.nimbleway.com/v2/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}"
response = http.request(request)
puts response.read_body{
"account_name": "<string>",
"crawl_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"crawl_options": {
"allow_external_links": false,
"allow_subdomains": false,
"crawl_entire_domain": false,
"ignore_query_parameters": false,
"limit": 5000,
"max_discovery_depth": 5,
"sitemap": "include",
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"exclude_paths": [
"<string>"
],
"include_paths": [
"<string>"
]
},
"created_at": "<string>",
"updated_at": "<string>",
"url": "<string>",
"completed": 123,
"completed_at": "<string>",
"extract_options": {},
"failed": 123,
"name": "<string>",
"pending": 123,
"tasks": [
{
"task_id": "<string>",
"created_at": "<string>",
"updated_at": "<string>"
}
],
"total": 123
}Create Crawl
curl --request POST \
--url https://sdk.nimbleway.com/v2/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"allow_external_links": false,
"allow_subdomains": false,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": false,
"exclude_paths": [
"<string>"
],
"extract_options": {
"auto_driver_configuration": {},
"body": "<unknown>",
"browser": "chrome",
"browser_actions": [
{
"goto": "https://example.com/login"
},
{
"wait_for_element": "#login-form"
},
{
"fill": {
"selector": "#username",
"value": "[email protected]"
}
},
{
"fill": {
"selector": "#password",
"value": "password123"
}
},
{
"click": "#submit"
},
{
"screenshot": {
"full_page": true
}
}
],
"city": "<string>",
"consent_header": true,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": [
"<string>"
],
"hostOnly": true,
"httpOnly": true,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": true,
"secure": true,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"expected_status_codes": [
0
],
"formats": [],
"headers": {},
"http2": false,
"is_xhr": false,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": false,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"parse": false,
"parser": {
"myParser": {
"option1": "value1"
}
},
"referrer_type": "no-referrer",
"render": true,
"request_timeout": 2,
"session": {
"id": "<string>",
"prefetch_userbrowser": false,
"renew_on_blocked": false,
"retry": false,
"timeout": 123
},
"skill": "dynamic-content",
"tag": "<string>",
"url": "<string>"
},
"ignore_query_parameters": false,
"include_paths": [
"<string>"
],
"limit": 5000,
"max_discovery_depth": 5,
"name": "<string>",
"sitemap": "include"
}
'import requests
url = "https://sdk.nimbleway.com/v2/crawl"
payload = {
"url": "<string>",
"allow_external_links": False,
"allow_subdomains": False,
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"crawl_entire_domain": False,
"exclude_paths": ["<string>"],
"extract_options": {
"auto_driver_configuration": {},
"body": "<unknown>",
"browser": "chrome",
"browser_actions": [{ "goto": "https://example.com/login" }, { "wait_for_element": "#login-form" }, { "fill": {
"selector": "#username",
"value": "[email protected]"
} }, { "fill": {
"selector": "#password",
"value": "password123"
} }, { "click": "#submit" }, { "screenshot": { "full_page": True } }],
"city": "<string>",
"consent_header": True,
"cookies": [
{
"creation": "<string>",
"domain": "<string>",
"expires": "<string>",
"extensions": ["<string>"],
"hostOnly": True,
"httpOnly": True,
"lastAccessed": "<string>",
"maxAge": 123,
"name": "<string>",
"path": "<string>",
"pathIsDefault": True,
"secure": True,
"value": "<string>"
}
],
"country": "US",
"device": "desktop",
"expected_status_codes": [0],
"formats": [],
"headers": {},
"http2": False,
"is_xhr": False,
"locale": "en-US",
"markdown_backend": "full_page",
"method": "GET",
"network_capture": [
{
"resource_type": "document",
"status_code": 349.5,
"validation": False,
"wait_for_requests_count": 0,
"wait_for_requests_count_timeout": 123
}
],
"parse": False,
"parser": { "myParser": { "option1": "value1" } },
"referrer_type": "no-referrer",
"render": True,
"request_timeout": 2,
"session": {
"id": "<string>",
"prefetch_userbrowser": False,
"renew_on_blocked": False,
"retry": False,
"timeout": 123
},
"skill": "dynamic-content",
"tag": "<string>",
"url": "<string>"
},
"ignore_query_parameters": False,
"include_paths": ["<string>"],
"limit": 5000,
"max_discovery_depth": 5,
"name": "<string>",
"sitemap": "include"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
allow_external_links: false,
allow_subdomains: false,
callback: {url: '<string>', events: [], headers: {}, metadata: {}},
crawl_entire_domain: false,
exclude_paths: ['<string>'],
extract_options: {
auto_driver_configuration: {},
body: JSON.stringify('<unknown>'),
browser: 'chrome',
browser_actions: [
{goto: 'https://example.com/login'},
{wait_for_element: '#login-form'},
{fill: {selector: '#username', value: '[email protected]'}},
{fill: {selector: '#password', value: 'password123'}},
{click: '#submit'},
{screenshot: {full_page: true}}
],
city: '<string>',
consent_header: true,
cookies: [
{
creation: '<string>',
domain: '<string>',
expires: '<string>',
extensions: ['<string>'],
hostOnly: true,
httpOnly: true,
lastAccessed: '<string>',
maxAge: 123,
name: '<string>',
path: '<string>',
pathIsDefault: true,
secure: true,
value: '<string>'
}
],
country: 'US',
device: 'desktop',
expected_status_codes: [0],
formats: [],
headers: {},
http2: false,
is_xhr: false,
locale: 'en-US',
markdown_backend: 'full_page',
method: 'GET',
network_capture: [
{
resource_type: 'document',
status_code: 349.5,
validation: false,
wait_for_requests_count: 0,
wait_for_requests_count_timeout: 123
}
],
parse: false,
parser: {myParser: {option1: 'value1'}},
referrer_type: 'no-referrer',
render: true,
request_timeout: 2,
session: {
id: '<string>',
prefetch_userbrowser: false,
renew_on_blocked: false,
retry: false,
timeout: 123
},
skill: 'dynamic-content',
tag: '<string>',
url: '<string>'
},
ignore_query_parameters: false,
include_paths: ['<string>'],
limit: 5000,
max_discovery_depth: 5,
name: '<string>',
sitemap: 'include'
})
};
fetch('https://sdk.nimbleway.com/v2/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://sdk.nimbleway.com/v2/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'allow_external_links' => false,
'allow_subdomains' => false,
'callback' => [
'url' => '<string>',
'events' => [
],
'headers' => [
],
'metadata' => [
]
],
'crawl_entire_domain' => false,
'exclude_paths' => [
'<string>'
],
'extract_options' => [
'auto_driver_configuration' => [
],
'body' => '<unknown>',
'browser' => 'chrome',
'browser_actions' => [
[
'goto' => 'https://example.com/login'
],
[
'wait_for_element' => '#login-form'
],
[
'fill' => [
'selector' => '#username',
'value' => '[email protected]'
]
],
[
'fill' => [
'selector' => '#password',
'value' => 'password123'
]
],
[
'click' => '#submit'
],
[
'screenshot' => [
'full_page' => true
]
]
],
'city' => '<string>',
'consent_header' => true,
'cookies' => [
[
'creation' => '<string>',
'domain' => '<string>',
'expires' => '<string>',
'extensions' => [
'<string>'
],
'hostOnly' => true,
'httpOnly' => true,
'lastAccessed' => '<string>',
'maxAge' => 123,
'name' => '<string>',
'path' => '<string>',
'pathIsDefault' => true,
'secure' => true,
'value' => '<string>'
]
],
'country' => 'US',
'device' => 'desktop',
'expected_status_codes' => [
0
],
'formats' => [
],
'headers' => [
],
'http2' => false,
'is_xhr' => false,
'locale' => 'en-US',
'markdown_backend' => 'full_page',
'method' => 'GET',
'network_capture' => [
[
'resource_type' => 'document',
'status_code' => 349.5,
'validation' => false,
'wait_for_requests_count' => 0,
'wait_for_requests_count_timeout' => 123
]
],
'parse' => false,
'parser' => [
'myParser' => [
'option1' => 'value1'
]
],
'referrer_type' => 'no-referrer',
'render' => true,
'request_timeout' => 2,
'session' => [
'id' => '<string>',
'prefetch_userbrowser' => false,
'renew_on_blocked' => false,
'retry' => false,
'timeout' => 123
],
'skill' => 'dynamic-content',
'tag' => '<string>',
'url' => '<string>'
],
'ignore_query_parameters' => false,
'include_paths' => [
'<string>'
],
'limit' => 5000,
'max_discovery_depth' => 5,
'name' => '<string>',
'sitemap' => 'include'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://sdk.nimbleway.com/v2/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://sdk.nimbleway.com/v2/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://sdk.nimbleway.com/v2/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"allow_external_links\": false,\n \"allow_subdomains\": false,\n \"callback\": {\n \"url\": \"<string>\",\n \"events\": [],\n \"headers\": {},\n \"metadata\": {}\n },\n \"crawl_entire_domain\": false,\n \"exclude_paths\": [\n \"<string>\"\n ],\n \"extract_options\": {\n \"auto_driver_configuration\": {},\n \"body\": \"<unknown>\",\n \"browser\": \"chrome\",\n \"browser_actions\": [\n {\n \"goto\": \"https://example.com/login\"\n },\n {\n \"wait_for_element\": \"#login-form\"\n },\n {\n \"fill\": {\n \"selector\": \"#username\",\n \"value\": \"[email protected]\"\n }\n },\n {\n \"fill\": {\n \"selector\": \"#password\",\n \"value\": \"password123\"\n }\n },\n {\n \"click\": \"#submit\"\n },\n {\n \"screenshot\": {\n \"full_page\": true\n }\n }\n ],\n \"city\": \"<string>\",\n \"consent_header\": true,\n \"cookies\": [\n {\n \"creation\": \"<string>\",\n \"domain\": \"<string>\",\n \"expires\": \"<string>\",\n \"extensions\": [\n \"<string>\"\n ],\n \"hostOnly\": true,\n \"httpOnly\": true,\n \"lastAccessed\": \"<string>\",\n \"maxAge\": 123,\n \"name\": \"<string>\",\n \"path\": \"<string>\",\n \"pathIsDefault\": true,\n \"secure\": true,\n \"value\": \"<string>\"\n }\n ],\n \"country\": \"US\",\n \"device\": \"desktop\",\n \"expected_status_codes\": [\n 0\n ],\n \"formats\": [],\n \"headers\": {},\n \"http2\": false,\n \"is_xhr\": false,\n \"locale\": \"en-US\",\n \"markdown_backend\": \"full_page\",\n \"method\": \"GET\",\n \"network_capture\": [\n {\n \"resource_type\": \"document\",\n \"status_code\": 349.5,\n \"validation\": false,\n \"wait_for_requests_count\": 0,\n \"wait_for_requests_count_timeout\": 123\n }\n ],\n \"parse\": false,\n \"parser\": {\n \"myParser\": {\n \"option1\": \"value1\"\n }\n },\n \"referrer_type\": \"no-referrer\",\n \"render\": true,\n \"request_timeout\": 2,\n \"session\": {\n \"id\": \"<string>\",\n \"prefetch_userbrowser\": false,\n \"renew_on_blocked\": false,\n \"retry\": false,\n \"timeout\": 123\n },\n \"skill\": \"dynamic-content\",\n \"tag\": \"<string>\",\n \"url\": \"<string>\"\n },\n \"ignore_query_parameters\": false,\n \"include_paths\": [\n \"<string>\"\n ],\n \"limit\": 5000,\n \"max_discovery_depth\": 5,\n \"name\": \"<string>\",\n \"sitemap\": \"include\"\n}"
response = http.request(request)
puts response.read_body{
"account_name": "<string>",
"crawl_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"crawl_options": {
"allow_external_links": false,
"allow_subdomains": false,
"crawl_entire_domain": false,
"ignore_query_parameters": false,
"limit": 5000,
"max_discovery_depth": 5,
"sitemap": "include",
"callback": {
"url": "<string>",
"events": [],
"headers": {},
"metadata": {}
},
"exclude_paths": [
"<string>"
],
"include_paths": [
"<string>"
]
},
"created_at": "<string>",
"updated_at": "<string>",
"url": "<string>",
"completed": 123,
"completed_at": "<string>",
"extract_options": {},
"failed": 123,
"name": "<string>",
"pending": 123,
"tasks": [
{
"task_id": "<string>",
"created_at": "<string>",
"updated_at": "<string>"
}
],
"total": 123
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
Request body model for the /crawl endpoint
Url to crawl.
Allows the crawler to follow links to external websites.
false
Allows the crawler to follow links to subdomains of the main domain.
false
Webhook configuration for receiving crawl results.
Show child attributes
Show child attributes
Allows the crawler to follow internal links to sibling or parent URLs, not just child paths.
false
URL pathname regex patterns that exclude matching URLs from the crawl.
["/exclude-this-path", "/and-this-path"]
Show child attributes
Show child attributes
Do not re-scrape the same path with different (or none) query parameters.
false
URL pathname regex patterns that include matching URLs in the crawl.
["/include-this-path", "/and-this-path"]
Maximum number of pages to crawl.
1 <= x <= 10000100
Maximum depth to crawl based on discovery order.
1 <= x <= 203
Name of the crawl.
"The best crawl ever"
Sitemap and other methods will be used together to find URLs.
skip, include, only "include"
Response
Successful Response - Crawl Task Created
Crawl API response
Show child attributes
Show child attributes
queued, running, succeeded, failed, canceled Show child attributes
Show child attributes
Show child attributes
Show child attributes
Was this page helpful?