Comment on page
Batch Processing
The Web API can receive batch requests with up to 1,000 URLs per batch. This is useful if you need to make many requests and would like to save time. In the following examples, we'll demonstrate real-world use cases, and show how to use batch requests most efficiently to achieve the intended goal.
In this first example, we'll collect data from several unique URLs. To do so, we set the URLs we want to collect in the
url fields of the requests object. Nimble APIs requires that a base64 encoded credential string be sent with every request to authenticate your account. For detailed examples, see Web API Authentication.
cURL
Python
Node.js
Go
curl -X POST 'https://api.webit.live/api/v1/batch/web' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
"requests": [
{ "url": "https://www.finance.com" },
{ "url": "https://www.travel.com" },
{ "url": "https://www.socialmedia.com" }
],
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}'
import requests
url = 'https://api.webit.live/api/v1/batch/web'
headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
}
data = {
"requests": [
{ "url": "https://www.finance.com" },
{ "url": "https://www.travel.com" },
{ "url": "https://www.socialmedia.com" }
],
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}
response = requests.post(url, headers=headers, json=data)
print(response.status_code)
print(response.json())
const axios = require('axios');
const url = 'https://api.webit.live/api/v1/batch/web';
const headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
};
const data = {
"requests": [
{ "url": "https://www.finance.com" },
{ "url": "https://www.travel.com" },
{ "url": "https://www.socialmedia.com" }
],
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
};
axios.post(url, data, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});
package main
import (
"bytes"
"fmt"
"net/http"
"encoding/json"
)
func main() {
url := "https://api.webit.live/api/v1/batch/web"
payload := []byte(`{
"requests": [
{ "url": "https://www.finance.com" },
{ "url": "https://www.travel.com" },
{ "url": "https://www.socialmedia.com" }
],
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}`)
headers := map[string]string{
"Authorization": "Basic <credential string>",
"Content-Type": "application/json",
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}
Parameters that are placed outside the
requests object, such as storage_type, storage_url, and callback_url , are automatically applied as defaults to all defined requests.If a parameter is set both inside and outside the
requests object, the value inside the request overrides the one outside.In this example, we'll collect multiple URLs with a different country set for each URL. To do so, we'll take advantage of the requests object, which allows us to set any parameter inside each request:
cURL
Python
Node.js
Go
curl -X POST 'https://api.webit.live/api/v1/batch/web' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
"requests": [
{ "url": "https://www.finance.com", "country": "US", "locale": "en-US" },
{ "url": "https://www.travel.com", "country": "FR", "locale": "fr" },
{ "url": "https://www.socialmedia.com", "country": "GR", "locale": "de" },
{ "url": "https://www.searchengine.com" }
],
"country": "CA",
"locale": "ca",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}'
import requests
url = 'https://api.webit.live/api/v1/batch/web'
headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
}
data = {
"requests": [
{ "url": "https://www.finance.com", "country": "US", "locale": "en-US" },
{ "url": "https://www.travel.com", "country": "FR", "locale": "fr" },
{ "url": "https://www.socialmedia.com", "country": "GR", "locale": "de" },
{ "url": "https://www.searchengine.com" }
],
"country": "CA",
"locale": "ca",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}
response = requests.post(url, headers=headers, json=data)
print(response.status_code)
print(response.json())
const axios = require('axios');
const url = 'https://api.webit.live/api/v1/batch/web';
const headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
};
const data = {
"requests": [
{ "url": "https://www.finance.com", "country": "US", "locale": "en-US" },
{ "url": "https://www.travel.com", "country": "FR", "locale": "fr" },
{ "url": "https://www.socialmedia.com", "country": "GR", "locale": "de" },
{ "url": "https://www.searchengine.com" }
],
"country": "CA",
"locale": "ca",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
};
axios.post(url, data, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});
package main
import (
"bytes"
"fmt"
"net/http"
"encoding/json"
)
func main() {
url := "https://api.webit.live/api/v1/batch/web"
payload := []byte(`{
"requests": [
{ "url": "https://www.finance.com", "country": "US", "locale": "en-US" },
{ "url": "https://www.travel.com", "country": "FR", "locale": "fr" },
{ "url": "https://www.socialmedia.com", "country": "GR", "locale": "de" },
{ "url": "https://www.searchengine.com" }
],
"country": "CA",
"locale": "ca",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}`)
headers := map[string]string{
"Authorization": "Basic <credential string>",
"Content-Type": "application/json",
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
For the above request, each URL would be requested from the corresponding country. "examplefour.com" does not have a country set in its request, and thus will default to the country defined outside the
requests object (CA - Canada). If no default country had been set, by default the request would have used a randomly selected country.
Any parameter can be defined inside and/or outside the
requests object. We can take advantage of this in some cases by setting our URL once as a default and setting various other parameters in the requests object. For example:cURL
Python
Node.js
Go
curl -X POST 'https://api.webit.live/api/v1/batch/web' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
"requests": [
{ "country": "US", "locale": "en-US" },
{ "country": "FR", "locale": "fr" },
{ "country": "GR", "locale": "de" },
],
"url": "https://www.finance.com",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}'
import requests
url = 'https://api.webit.live/api/v1/batch/web'
headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
}
data = {
"requests": [
{ "country": "US", "locale": "en-US" },
{ "country": "FR", "locale": "fr" },
{ "country": "GR", "locale": "de" }
],
"url": "https://www.finance.com",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}
response = requests.post(url, headers=headers, json=data)
print(response.status_code)
print(response.json())
const axios = require('axios');
const url = 'https://api.webit.live/api/v1/batch/web';
const headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
};
const data = {
"requests": [
{ "country": "US", "locale": "en-US" },
{ "country": "FR", "locale": "fr" },
{ "country": "GR", "locale": "de" }
],
"url": "https://www.finance.com",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
};
axios.post(url, data, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});
package main
import (
"bytes"
"fmt"
"net/http"
"encoding/json"
)
func main() {
url := "https://api.webit.live/api/v1/batch/web"
payload := []byte(`{
"requests": [
{ "country": "US", "locale": "en-US" },
{ "country": "FR", "locale": "fr" },
{ "country": "GR", "locale": "de" }
],
"url": "https://www.finance.com",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}`)
headers := map[string]string{
"Authorization": "Basic <credential string>",
"Content-Type": "application/json",
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}
In the above example, the URL "exampleone.com" would be requested three times - once from the US, once from France, and once from Germany.
Batch requests use the same parameters as asynchronous requests, with the exception of the
requests object.Parameter | Required | Description |
|---|---|---|
requests | No | Object array - Allows for defining custom parameters for each request within the bulk. Any of the parameters below can be used in an individual request. |
url | Required | The URL to be fetched.
Note: when using a URL with a query string, encode the URL and place it at the end of the query string. |
method | Optional (default = GET) | The method for requesting a URL from the target server. |
country | Optional (default = all) | Country used to access the target URL, use ISO Alpha-2 Country Codes i.e. US, DE, GB |
locale | Optional (default = EN) | String || LCID standard locale used for the URL request. |
headers* | Optional | JSON with key/value structure to pass the required headers. |
format | Optional (default = JSON) | Enum: JSON | HTML - The data response format. HTML - in case of error, returns JSON with error message. |
parse | Optional (default = false) | Enum: true | false - True - the page's content will be parsed and returned in a JSON format. False - Response will include page headers and raw data (without parsing). When using parse = true, format must be set to “JSON”. |
parser | Optional (default = null) | |
render | Optional (default = false) | Enum: true | false - enables or disables Javascript rendering on the target page. |
render_flow | Optional (default = null) | String | Define a series of actions to be performed on the page prior to data collection. See Page Interactions. |
render_options | N/A | |
storage_type | Optional | ENUM: s3 | gs - Use s3 for Amazon S3 and gs for Google Cloud Platform.
Leave blank to enable Push/Pull delivery. |
storage_url | Optional | Repository URL: s3://Your.Bucket.Name/your/object/name/prefix/ - Output will be saved to TASK_ID.json
Leave blank to enable Push/Pull delivery. |
callback_url | Optional | A url to callback once the data is delivered. The WebAPI will send a POST request to the callback_url with the task details once the task is complete (this “notification” will not include the requested data). |
All URLs in a batch request receive the same parameters (country, locale, parse, repository, etc.)
In order to use Google Cloud Storage as your destination repository, please add Nimble’s system user as a principal to the relevant bucket. To do so, navigate to the “bucket details” page in your GCP console, and click on “Permission” in the submenu.
Next, past our system user [email protected] into the “New Principals” box, select Storage Object Creator as the role, and click save.
That’s all! At this point, Nimble will be able to upload files to your chosen GCS bucket.
In order to use S3 as your destination repository, please give Nimble’s service user permission to upload files to the relevant S3 bucket. Paste the following JSON into the “Bucket Policy” (found under “Permissions”) in the AWS console.
Follow these steps:
1. Go to the “Permissions” tab on the bucket’s dashboard:
2. Scroll down to “Bucket policy” and press edit:
3. Paste the following bucket policy configuration into your bucket:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Statement1",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": [
"s3:PutObject",
"s3:PutObjectACL"
],
"Resource": "arn:aws:s3:::YOUR_BUCKET_NAME/*"
},
{
"Sid": "Statement2",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": "s3:GetBucketLocation",
"Resource": "arn:aws:s3:::YOUR_BUCKET_NAME"
}
]
}
Important: Remember to replace “YOUR_BUCKET_NAME” with your actual bucket name.
Here is what the bucket policy should look like:
4. Scroll down and press “Save changes”
If your S3 bucket is encrypted using an AWS Key Management Service (KMS) key, additional permissions to those outlined above are also needed. Specifically, Nimble's service user will need to be given permission to encrypt and decrypt objects using a KMS key. To do this, follow the steps below:
- 1.Sign in to the AWS Management Console and open the AWS Key Management Service (KMS) console.
- 2.In the navigation pane, choose "Customer managed keys".
- 3.Select the KMS key you want to modify.
- 4.Choose the "Key policy" tab, then "Switch to policy view".
- 5.Click "Edit".
- 6.Add the following statement to the existing policy JSON, ensuring it's inside the Statement array:
{
"Version": "2012-10-17",
"Id": "example-key-policy",
"Statement": [
// ... your pre-existing statements ...
{
"Sid": "Allow Nimble APIs account",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*"
},
]
}
- 7.Click "Save changes" to update the key policy.
That's it! You've now given Nimble APIs permission to encrypt and decrypt objects, enabling access to encrypted buckets.
Please add Nimble's system/service user to your GCS or S3 bucket to ensure that data can be delivered successfully.
Batch requests operate asynchronously, and treat each request as a separate task. The result of each task is stored in a file, and a notification is sent to the provided callback any time an individual task is completed.
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"batch_size": 3,
"tasks": [
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "2e508d43-8b02-4fc0-96c7-0968ab454a0c",
"state": "pending",
"output_url": "s3://Your.Repository.Path/2e508d43-8b02-4fc0-96c7-0968ab454a0c.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://api.webit.live/api/v1/tasks/2e508d43-8b02-4fc0-96c7-0968ab454a0c",
"created_at": "2022-07-24T08:09:23.205Z",
"modified_at": "2022-07-24T08:09:23.205Z",
"input": {
...
},
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "63cc3bd5-01b4-4787-90a2-f382b9960c77",
"state": "pending",
...
},
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
"state": "pending",
...
}
]
}
POST https://api.webit.live/api/v1/batches/<batch_id>/progress
Like asynchronous tasks, the status of a batch is available for 24 hours.
cURL
Python
Node.js
Go
curl -X GET 'https://api.webit.live/api/v1/batches/<batch_id>/progress' \
--header 'Authorization: Basic <credential string>'
import requests
batch_id = "<batch_id>"
url = f"https://api.webit.live/api/v1/batches/{batch_id}/progress"
headers = {
'Authorization': 'Basic <credential string>'
}
response = requests.get(url, headers=headers)
print(response.status_code)
print(response.json())
const axios = require('axios');
const batchId = "<batch_id>";
const url = `https://api.webit.live/api/v1/batches/${batchId}/progress`;
const headers = {
'Authorization': 'Basic <credential string>'
};
axios.get(url, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});
package main
import (
"fmt"
"net/http"
)
func main() {
batchID := "<batch_id>"
url := fmt.Sprintf("https://api.webit.live/api/v1/batches/%s/progress", batchID)
headers := map[string]string{
"Authorization": "Basic <credential string>",
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}
Response
The progress of a batch is reported in percentages.
{
"status": "success",
"completed": false,
"progress": 0.333333
}
Once a batch is finished, its progress will be reported as “1”.
{
"status": "success",
"completed": true,
"progress": 1
}
Once a batch has finished, it’s possible to return a summary of the completed tasks, by using the following endpoint:
GET https://api.webit.live/api/v1/batches/<batch_id>
For example:
cURL
Python
Node.js
Go
curl -X GET 'https://api.webit.live/api/v1/batches/7a07a96d-c402-4d98-a17f-4ecb390d11a3' \
--header 'Authorization: Basic <credential string>'
import requests
batch_id = "7a07a96d-c402-4d98-a17f-4ecb390d11a3"
url = f"https://api.webit.live/api/v1/batches/{batch_id}"
headers = {
'Authorization': 'Basic <credential string>'
}
response = requests.get(url, headers=headers)
print(response.status_code)
print(response.json())
const axios = require('axios');
const batchId = "7a07a96d-c402-4d98-a17f-4ecb390d11a3";
const url = `https://api.webit.live/api/v1/batches/${batchId}`;
const headers = {
'Authorization': 'Basic <credential string>'
};
axios.get(url, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});
package main
import (
"fmt"
"net/http"
)
func main() {
batchID := "7a07a96d-c402-4d98-a17f-4ecb390d11a3"
url := fmt.Sprintf("https://api.webit.live/api/v1/batches/%s", batchID)
headers := map[string]string{
"Authorization": "Basic <credential string>",
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}
The response object lists the status of the overall batch, as well as the individual tasks and their details:
Response
{
"status": "success",
"tasks": [
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "2e508d43-8b02-4fc0-96c7-0968ab454a0c",
"state": "success",
"query_time": "2023-01-01T12:00:00.007Z",
"output_url": "s3://Your.Repository.Path/2e508d43-8b02-4fc0-96c7-0968ab454a0c.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://[base_url]/api/v1/tasks/2e508d43-8b02-4fc0-96c7-0968ab454a0c",
"created_at": "2022-07-24T08:09:23.205Z",
"modified_at": "2022-07-24T08:10:27.244Z",
"input": {
...
}
},
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "63cc3bd5-01b4-4787-90a2-f382b9960c77",
"state": "success",
"query_time": "2023-01-01T12:00:00.007Z",
"output_url": "s3://Your.Repository.Path/63cc3bd5-01b4-4787-90a2-f382b9960c77.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://[base_url]/api/v1/tasks/63cc3bd5-01b4-4787-90a2-f382b9960c77",
"created_at": "2022-07-24T08:09:23.205Z",
"modified_at": "2022-07-24T08:10:27.973Z",
"input": {
...
}
},
{
"batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
"id": "4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
"state": "success",
"query_time": "2023-01-01T12:00:00.007Z",
"output_url": "s3://Your.Repository.Path/4cb39bbf-5580-4c50-8ed4-4a7905e2ec52.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://[base_url]/api/v1/tasks/4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
"created_at": "2022-07-24T08:09:23.205Z",
"modified_at": "2022-07-24T08:10:30.292Z",
"input": {
...
}
}
],
"completed": true,
"progress": 1
}
500 Error
{
"status": "error",
"task_id": "<task_id>",
"msg": "can't download the query response - please try again"
}
400 Input Error
{
"status": "failed",
"msg": error
}
Previously, batch requests used a
url field to pass multiple URLs for collection. That syntax is now deprecated, and batch requests should be updated to use the new requestssyntax defined above.Status | Description |
|---|---|
200 | OK |
400 | The requested resource could not be reached |
401 | Unauthorized/invalid credental string |
500 | Internal service error |
501 | An error was encountered by the proxy service |