Custom Headers & Cookies

What?

The Nimble Web API provides the ability to enhance your scraping requests by including custom headers or cookies. These can be defined in either string or object format and sent as part of your request to the target domain. This feature is crucial for accessing additional data that may be gated behind specific user settings or preferences stored in cookies, allowing you to interact with the website in a more personalized and context-aware manner.

Why?

• Unlock Access to Personalized Data: By including custom headers or cookies, you can retrieve data that is tailored to specific user session contexts, which is often otherwise inaccessible.

• Enhance Data Retrieval Capabilities: Support diverse use cases by leveraging cookies and headers to interact with websites in ways that mimic real user behavior, ensuring you capture the most relevant data.

• Improve Data Retrieval Efficiency: Reduce the need for additional scraping attempts by ensuring that your initial requests are as comprehensive and accurate as possible, saving time and resources.

Additional Information

• Supported by real-time, asynchronous, and batch requests.

• Supported Endpoints: Web

• Not supported Endpoints: Social, SERP, Maps and eCommerce.

Misconfiguration of custom headers or cookies may negatively impact the success rates of your scraping tasks, as improper use may trigger detection by the target website. This feature is intended for advanced users, so please exercise caution and ensure configurations are accurate.

Request Option

Parameter	Required	Description
url	Required	URL | The page or resource to be fetched. Note: when using a URL with a query string, encode the URL and place it at the end of the query string
headers*	Optional	String | JSON with key/value structure to pass the required headers
method	Required when headers is used to send POST request	Enum | GET, POST. Use POST to send forms to the target site
body	Required when headers is used to send POST request	Object | JSON with key/value structure or string to pass needed payload
cookies	Optional	String | The cookie string uses a syntax of key1=value1;key2=value2;... format, and does not accept a domain parameter, always defaulting to the requested URL's domain. OR List Objects | List of attach cookie objects. The cookie object allows for one or multiple cookies to be set, with each one containing three parameters - key, value, domain
cookies.key	Required when cookies is used	String | The name of the attached cookie
cookies.value	Required when cookies is used	String | The value of the attached cookie.
cookies.domain	Optional	URL | The domain with which this cookie is associated. If a domain is not defined, Nimble will use the requested URL's domain by default.

* Please do not include any cookies when sending custom headers.

Example Request - Sending POST Request with Headers

If you wish to send POST data with your target domain as part of your request, such as when submitting a form, please use the following syntax:

cURL

curl -X POST 'https://api.webit.live/api/v1/realtime/web' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
    "url": "https://www.example.com",
    "method": "POST",
    "headers": {
    "Content-Type": "application/json",
        "Some-Extra-Header": "some-extra-header"
    },
    "body": {
        "a": 1,
        "b": 2
    }
}'

Python

import requests

url = 'https://api.webit.live/api/v1/realtime/web'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "url": "https://www.example.com",
    "method": "POST",
    "headers": {
        "Content-Type": "application/json",
        "Some-Extra-Header": "some-extra-header"
    },
    "body": {
        "a": 1,
        "b": 2
    }
}

response = requests.post(url, headers=headers, json=data)

Node.js

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/realtime/web';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "url": "https://www.example.com",
  "method": "POST",
  "headers": {
    "Content-Type": "application/json",
    "Some-Extra-Header": "some-extra-header"
  },
  "body": {
    "a": 1,
    "b": 2
  }
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

Go

package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/realtime/web"
	payload := []byte(`{
		"url": "https://www.example.com",
		"method": "POST",
		"headers": {
			"Content-Type": "application/json",
			"Some-Extra-Header": "some-extra-header"
		},
		"body": {
			"a": 1,
			"b": 2
		}
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

Please note that specifying a content-type header is required when submitting a POST request with data in the body.

Example Request - Cookie Object

In the following example, a Cookie Object is used to select a particular store on the Lowe's website.

cURL

curl --location --request POST 'https://api.webit.live/api/v1/realtime/web/' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
    "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
    "country": "US",
    "cookies": [
        {
            "key": "sd",
            "value": "%7B%22id%22%3A%222209%22%2C%22zip%22%3A%2204769%22%2C%22city%22%3A%22Presque%20Isle%22%2C%22state%22%3A%22ME%22%2C%22name%22%3A%22Presque%20Isle%20Lowe's%22%2C%22region%22%3A%2218%22%7D",
            "domain": "lowes.com"
        }
    ]
}'

Python

import requests

url = 'https://api.webit.live/api/v1/realtime/web'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
    "country": "CA",
    "cookies": [
        {
            "key": "sd",
            "value": "%7B%22id%22%3A%222209%22%2C%22zip%22%3A%2204769%22%2C%22city%22%3A%22Presque%20Isle%22%2C%22state%22%3A%22ME%22%2C%22name%22%3A%22Presque%20Isle%20Lowe's%22%2C%22region%22%3A%2218%22%7D",
            "domain": "lowes.com"
        }
    ] 
}

response = requests.post(url, headers=headers, json=data)

Node.js

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/realtime/web';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
  "country": "CA",
  "cookies": [
        {
            "key": "sd",
            "value": "%7B%22id%22%3A%222209%22%2C%22zip%22%3A%2204769%22%2C%22city%22%3A%22Presque%20Isle%22%2C%22state%22%3A%22ME%22%2C%22name%22%3A%22Presque%20Isle%20Lowe's%22%2C%22region%22%3A%2218%22%7D",
            "domain": "lowes.com"
        }
    ]
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

Go

package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/realtime/web"
	payload := []byte(`{
		"url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
		"country": "CA",
		"cookies": [
		        {
		            "key": "sd",
		            "value": "%7B%22id%22%3A%222209%22%2C%22zip%22%3A%2204769%22%2C%22city%22%3A%22Presque%20Isle%22%2C%22state%22%3A%22ME%22%2C%22name%22%3A%22Presque%20Isle%20Lowe's%22%2C%22region%22%3A%2218%22%7D",
		            "domain": "lowes.com"
		        }
		    ]
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

In the above example, the key is set to sd and the value is set to an encoded version of the following location object:

{
    "id": "2209",
    "zip": "04769",
    "city": "Presque Isle",
    "state": "ME",
    "name": "Presque Isle Lowe's",
    "region": "18"
}

These values are unique to Lowe's website, and are not necessarily applicable to other websites.

Example Request - Cookie String

In the following example, a Cookie String is used to select a particular store on the Lowe's website.

cURL

curl --location --request POST 'https://api.webit.live/api/v1/realtime/web/' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
    "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
    "country": "US",
    "cookies": "key1=value1;sd=%7B%22id%22%3A%222209..."
}'

Python

import requests

url = 'https://api.webit.live/api/v1/realtime/web'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
    "country": "CA",
    "cookies": "key1=value1;sd=%7B%22id%22%3A%222209..."
}

response = requests.post(url, headers=headers, json=data)

Node.js

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/realtime/web';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
  "country": "CA",
  "cookies": "key1=value1;sd=%7B%22id%22%3A%222209..."
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

Go

package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/realtime/web"
	payload := []byte(`{
		"url": "https://www.lowes.com/pd/CRAFTSMAN-M110-140-cc-21-in-Gas-Push-Lawn-Mower-with-Briggs-Stratton-Engine/1000676311",
		"country": "CA",
		"cookies": "key1=value1;sd=%7B%22id%22%3A%222209..."
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}