Batch Processing

When collecting data on a large number of products, a batch request can help streamline the process into a more convenient workflow. Instead of launching a separate request for each product, a batch request allows you to include up to 1,000 URLs in a single request, and have the results delivered to your cloud storage solution.

In the following examples, we'll demonstrate real-world use cases, and show how to use batch requests to most efficiently achieve the intended goal.

Example one - collecting data for multiple products

In this first example, we'll collect data for multiple unique products. To do so, we set the URLs of the products we want to collect in the url fields of the requests object.

Nimble APIs requires that a base64 encoded credential string be sent with every request to authenticate your account. For detailed examples, see Web API Authentication.

curl -X POST 'https://api.webit.live/api/v1/batch/ecommerce' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{ 
    "requests": [
        { "url": "https://www.walmart.com/ip/Product-One...." },
        { "url": "https://www.walmart.com/ip/Product-Two...." },
        { "url": "https://www.walmart.com/ip/Product-Three...." }
    ],
    "vendor": "walmart",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}'

import requests

url = 'https://api.webit.live/api/v1/batch/ecommerce'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "requests": [
        { "url": "https://www.walmart.com/ip/Product-One...." },
        { "url": "https://www.walmart.com/ip/Product-Two...." },
        { "url": "https://www.walmart.com/ip/Product-Three...." }
    ],
    "vendor": "walmart",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}

response = requests.post(url, headers=headers, json=data)

print(response.status_code)
print(response.json())

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batch/ecommerce';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "requests": [
    { "url": "https://www.walmart.com/ip/Product-One...." },
    { "url": "https://www.walmart.com/ip/Product-Two...." },
    { "url": "https://www.walmart.com/ip/Product-Three...." }
  ],
  "vendor": "walmart",
  "storage_type": "s3",
  "storage_url": "s3://Your.Repository.Path/",
  "callback_url": "https://your.callback.url/path"
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

package main

import (
	"bytes"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/batch/ecommerce"
	payload := []byte(`{
		"requests": [
			{ "url": "https://www.walmart.com/ip/Product-One...." },
			{ "url": "https://www.walmart.com/ip/Product-Two...." },
			{ "url": "https://www.walmart.com/ip/Product-Three...." }
		],
		"vendor": "walmart",
		"storage_type": "s3",
		"storage_url": "s3://Your.Repository.Path/",
		"callback_url": "https://your.callback.url/path"
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

Parameters that are placed outside the requests object, such as vendor, storage_type, storage_url, and callback_url , are automatically applied as defaults to all defined requests.

If a parameter is set both inside and outside the requests object, the value inside the request overrides the one outside.

Example two - collecting products from multiple vendors

In the next example, we'll collect products from multiple vendors. To do so, we'll take advantage of the requests object, which allows us to set any parameter inside each request:

curl -X POST 'https://api.webit.live/api/v1/batch/ecommerce' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{ 
    "requests": [
        { "url": "https://www.walmart.com/ip/Product-One....", "vendor": "walmart" },
        { "url": "https://www.walmart.com/ip/Product-Two....", "vendor": "walmart" },
        { "url": "https://www.amazon.com/Other-product...."  }
    ],
    "vendor": "amazon",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}'

import requests

url = 'https://api.webit.live/api/v1/batch/ecommerce'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "requests": [
        { "url": "https://www.walmart.com/ip/Product-One....", "vendor": "walmart" },
        { "url": "https://www.walmart.com/ip/Product-Two....", "vendor": "walmart" },
        { "url": "https://www.amazon.com/Other-product...."  }
    ],
    "vendor": "amazon",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}

response = requests.post(url, headers=headers, json=data)

print(response.status_code)
print(response.json())

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batch/ecommerce';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "requests": [
        { "url": "https://www.walmart.com/ip/Product-One....", "vendor": "walmart" },
        { "url": "https://www.walmart.com/ip/Product-Two....", "vendor": "walmart" },
        { "url": "https://www.amazon.com/Other-product...."  }
    ],
    "vendor": "amazon",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

package main

import (
	"bytes"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/batch/ecommerce"
	payload := []byte(`{
		"requests": [
        		{ "url": "https://www.walmart.com/ip/Product-One....", "vendor": "walmart" },
        		{ "url": "https://www.walmart.com/ip/Product-Two....", "vendor": "walmart" },
        		{ "url": "https://www.amazon.com/Other-product...."  }
    		],
    		"vendor": "amazon",
    		"storage_type": "s3",
    		"storage_url": "s3://Your.Repository.Path/",
    		"callback_url": "https://your.callback.url/path"
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

In the above example, we've set Walmart as our desired vendor for the first two requests. On the third request, we haven't set a vendor implicitly, and so the default vendor defined outside the requests object is used instead, which in this case is Amazon.

Example three - collecting the same product from multiple countries

Any parameter can be defined inside and/or outside the requests object. We can take advantage of this in some cases by setting our URL once as a default and setting various other parameters in the requests object. For example:

curl -X POST 'https://api.webit.live/api/v1/batch/ecommerce' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{ 
    "requests": [
        { "country": "US", "locale": "en-US" },
        { "country": "FR", "locale": "fr" },
        { "country": "GR", "locale": "de" },
    ],
    "url": "https://www.walmart.com/ip/Product-One....",
    "vendor": "walmart",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}'

import requests

url = 'https://api.webit.live/api/v1/batch/ecommerce'
headers = {
    'Authorization': 'Basic <credential string>',
    'Content-Type': 'application/json'
}
data = {
    "requests": [
        { "country": "US", "locale": "en-US" },
        { "country": "FR", "locale": "fr" },
        { "country": "GR", "locale": "de" },
    ],
    "url": "https://www.walmart.com/ip/Product-One....",
    "vendor": "walmart",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
}

response = requests.post(url, headers=headers, json=data)

print(response.status_code)
print(response.json())

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batch/ecommerce';
const headers = {
  'Authorization': 'Basic <credential string>',
  'Content-Type': 'application/json'
};
const data = {
  "requests": [
        { "country": "US", "locale": "en-US" },
        { "country": "FR", "locale": "fr" },
        { "country": "GR", "locale": "de" },
    ],
    "url": "https://www.walmart.com/ip/Product-One....",
    "vendor": "walmart",
    "storage_type": "s3",
    "storage_url": "s3://Your.Repository.Path/",
    "callback_url": "https://your.callback.url/path"
};

axios.post(url, data, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

package main

import (
	"bytes"
	"fmt"
	"net/http"
	"encoding/json"
)

func main() {
	url := "https://api.webit.live/api/v1/batch/ecommerce"
	payload := []byte(`{
		"requests": [
        		{ "country": "US", "locale": "en-US" },
        		{ "country": "FR", "locale": "fr" },
        		{ "country": "GR", "locale": "de" },
    		],
    		"url": "https://www.walmart.com/ip/Product-One....",
    		"vendor": "walmart",
    		"storage_type": "s3",
    		"storage_url": "s3://Your.Repository.Path/",
    		"callback_url": "https://your.callback.url/path"
	}`)
	headers := map[string]string{
		"Authorization":  "Basic <credential string>",
		"Content-Type":   "application/json",
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

In the above example, three total requests would be made for the same Walmart product - each time from a different country and locale.

Request Options

Batch requests use the same parameters as asynchronous requests, with the exception of the requests object.

Parameter	Required	Description
requests	Optional	Allows for defining custom parameters for each request within the bulk. Any of the parameters below can be used in an individual request.
vendor	Required	String walmart, amazon
url	Required	The product or search pages to collect.
country	Optional (default = all)	String Country used to access the target URL, use ISO Alpha-2 Country Codes i.e. US, DE, GB
zip	optional	String A 5-digit US zip code. If provided, the closest store to the provided zip code is selected.
locale	Optional (default = en)	String LCID standard locale used for the URL request.
parse	Optional (default = false)	Enum: true \| false Instructs Nimble whether to structure the results into a JSON format or return the raw HTML.
storage_type	Optional	ENUM: s3 \| gs Use s3 for Amazon S3 and gs for Google Cloud Platform. Leave blank to enable Push/Pull delivery.
storage_url	Optional	Repository URL: s3://Your.Bucket.Name/your/object/name/prefix/ Leave blank to enable Push/Pull delivery.
callback_url	Optional	A url to callback once the data is delivered. The E-commerce API will send a POST request to the callback_url with the task details once the task is complete (this “notification” will not include the requested data).

Parameter

Required

Description

requests

Optional

Allows for defining custom parameters for each request within the bulk. Any of the parameters below can be used in an individual request.

vendor

Required

String walmart, amazon

url

Required

The product or search pages to collect.

country

Optional (default = all)

String Country used to access the target URL, use ISO Alpha-2 Country Codes i.e. US, DE, GB

zip

optional

String A 5-digit US zip code. If provided, the closest store to the provided zip code is selected.

locale

Optional (default = en)

String LCID standard locale used for the URL request.

parse

Optional (default = false)

Enum: true | false Instructs Nimble whether to structure the results into a JSON format or return the raw HTML.

storage_type

Optional

ENUM: s3 | gs Use s3 for Amazon S3 and gs for Google Cloud Platform. Leave blank to enable Push/Pull delivery.

storage_url

Optional

Repository URL: s3://Your.Bucket.Name/your/object/name/prefix/ Leave blank to enable Push/Pull delivery.

callback_url

Optional

A url to callback once the data is delivered. The E-commerce API will send a POST request to the

callback_url with the task details once the task is complete (this “notification” will not include the requested data).

Setting GCS/AWS access permissions

GCS Repository Configuration

In order to use Google Cloud Storage as your destination repository, please add Nimble’s system user as a principal to the relevant bucket. To do so, navigate to the “bucket details” page in your GCP console, and click on “Permission” in the submenu.

Next, past our system user [email protected] into the “New Principals” box, select Storage Object Creator as the role, and click save.

That’s all! At this point, Nimble will be able to upload files to your chosen GCS bucket.

S3 repository configuration

In order to use S3 as your destination repository, please give Nimble’s service user permission to upload files to the relevant S3 bucket. Paste the following JSON into the “Bucket Policy” (found under “Permissions”) in the AWS console.

Follow these steps:

1. Go to the “Permissions” tab on the bucket’s dashboard:

2. Scroll down to “Bucket policy” and press edit:

3. Paste the following bucket policy configuration into your bucket:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "Statement1",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::744254827463:user/webit-uploader"
            },
            "Action": [
                "s3:PutObject",
                "s3:PutObjectACL"
            ],
            "Resource": "arn:aws:s3:::YOUR_BUCKET_NAME/*"
        },
        {
            "Sid": "Statement2",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::744254827463:user/webit-uploader"
            },
            "Action": "s3:GetBucketLocation",
            "Resource": "arn:aws:s3:::YOUR_BUCKET_NAME"
        }
    ]
}

Important: Remember to replace “YOUR_BUCKET_NAME” with your actual bucket name.

Here is what the bucket policy should look like:

4. Scroll down and press “Save changes”

S3 Encrypted Buckets

If your S3 bucket is encrypted using an AWS Key Management Service (KMS) key, additional permissions to those outlined above are also needed. Specifically, Nimble's service user will need to be given permission to encrypt and decrypt objects using a KMS key. To do this, follow the steps below:

Sign in to the AWS Management Console and open the AWS Key Management Service (KMS) console.
In the navigation pane, choose "Customer managed keys".
Select the KMS key you want to modify.
Choose the "Key policy" tab, then "Switch to policy view".
Click "Edit".
Add the following statement to the existing policy JSON, ensuring it's inside the Statement array:

{
	"Version": "2012-10-17",
	"Id": "example-key-policy",
	"Statement": [
		// ... your pre-existing statements ...
		{
			"Sid": "Allow Nimble APIs account",
			"Effect": "Allow",
			"Principal": {
				"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
			},
			"Action": [
				"kms:Encrypt",
				"kms:Decrypt",
				"kms:ReEncrypt*",
				"kms:GenerateDataKey*",
				"kms:DescribeKey"
			],
			"Resource": "*"
		},
	]
}

Click "Save changes" to update the key policy.

That's it! You've now given Nimble APIs permission to encrypt and decrypt objects, enabling access to encrypted buckets.

Please add Nimble's system/service user to your GCS or S3 bucket to ensure that data can be delivered successfully.

Response

Initial Response

Batch requests operate asynchronously, and treat each request as a separate task. The result of each task is stored in a file, and a notification is sent to the provided callback any time an individual task is completed.

{
    "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
    "batch_size": 3,
    "tasks": [
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "2e508d43-8b02-4fc0-96c7-0968ab454a0c",
            "state": "pending",
            "output_url": "s3://Your.Repository.Path/2e508d43-8b02-4fc0-96c7-0968ab454a0c.json",
            "callback_url": "https://your.callback.url/path",
            "status_url": "https://api.webit.live/api/v1/tasks/2e508d43-8b02-4fc0-96c7-0968ab454a0c",
            "created_at": "2022-07-24T08:09:23.205Z",
            "modified_at": "2022-07-24T08:09:23.205Z",
            "input": {
            ...
        },
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "63cc3bd5-01b4-4787-90a2-f382b9960c77",
            "state": "pending",
            ...
        },
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
            "state": "pending",
            ...
        }
    ]
}

Checking batch progress and status

POST https://api.webit.live/api/v1/batches/<batch_id>/progress

Like asynchronous tasks, the status of a batch is available for 24 hours.

curl -X GET 'https://api.webit.live/api/v1/batches/<batch_id>/progress' \
--header 'Authorization: Basic <credential string>'

import requests

url = 'https://api.webit.live/api/v1/batches/<batch_id>/progress'
headers = {
    'Authorization': 'Basic <credential string>'
}

response = requests.get(url, headers=headers)

print(response.status_code)
print(response.json())

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batches/<batch_id>/progress';
const headers = {
  'Authorization': 'Basic <credential string>'
};

axios.get(url, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

package main

import (
	"fmt"
	"net/http"
)

func main() {
	url := "https://api.webit.live/api/v1/batches/<batch_id>/progress"
	headers := map[string]string{
		"Authorization": "Basic <credential string>",
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

Response

The progress of a batch is reported in percentages.

{
    "status": "success",
    "completed": false,
    "progress": 0.333333
}

Once a batch is finished, its progress will be reported as “1”.

{
    "status": "success",
    "completed": true,
    "progress": 1
}

Retrieving Batch Summary

Once a batch is finished, it’s possible to return a summary of the completed tasks by using the following endpoint:

GET https://api.webit.live/api/v1/batches/<batch_id>

For example:

curl -X GET 'https://api.webit.live/api/v1/batches/<batch_id>' \
--header 'Authorization: Basic <credential string>'

import requests

url = 'https://api.webit.live/api/v1/batches/<batch_id>const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batches/<batch_id>/progress';
const headers = {
  'Authorization': 'Basic <credential string>'
};

axios.get(url, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });
'
headers = {
    'Authorization': 'Basic <credential string>'
}

response = requests.get(url, headers=headers)

print(response.status_code)
print(response.json())

const axios = require('axios');

const url = 'https://api.webit.live/api/v1/batches/<batch_id>';
const headers = {
  'Authorization': 'Basic <credential string>'
};

axios.get(url, { headers })
  .then(response => {
    console.log(response.status);
    console.log(response.data);
  })
  .catch(error => {
    console.error(error);
  });

package main

import (
	"fmt"
	"net/http"
)

func main() {
	url := "https://api.webit.live/api/v1/batches/<batch_id>"
	headers := map[string]string{
		"Authorization": "Basic <credential string>",
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	for key, value := range headers {
		req.Header.Set(key, value)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.StatusCode)
	// Read the response body if needed
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
}

The response object lists the status of the overall batch, as well as the individual tasks and their details:

Response

{
    "status": "success",
    "tasks": [
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "2e508d43-8b02-4fc0-96c7-0968ab454a0c",
            "state": "success",
            "output_url": "s3://Your.Repository.Path/2e508d43-8b02-4fc0-96c7-0968ab454a0c.json",
            "callback_url": "https://your.callback.url/path",
            "status_url": "https://[base_url]/api/v1/tasks/2e508d43-8b02-4fc0-96c7-0968ab454a0c",
            "created_at": "2022-07-24T08:09:23.205Z",
            "modified_at": "2022-07-24T08:10:27.244Z",
            "input": {
                            ...
            }
        },
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "63cc3bd5-01b4-4787-90a2-f382b9960c77",
            "state": "success",
            "output_url": "s3://Your.Repository.Path/63cc3bd5-01b4-4787-90a2-f382b9960c77.json",
            "callback_url": "https://your.callback.url/path",
            "status_url": "https://[base_url]/api/v1/tasks/63cc3bd5-01b4-4787-90a2-f382b9960c77",
            "created_at": "2022-07-24T08:09:23.205Z",
            "modified_at": "2022-07-24T08:10:27.973Z",
            "input": {
                            ...
            }
         },
        {
            "batch_id": "7a07a96d-c402-4d98-a17f-4ecb390d11a3",
            "id": "4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
            "state": "success",
            "output_url": "s3://Your.Repository.Path/4cb39bbf-5580-4c50-8ed4-4a7905e2ec52.json",
            "callback_url": "https://your.callback.url/path",
            "status_url": "https://[base_url]/api/v1/tasks/4cb39bbf-5580-4c50-8ed4-4a7905e2ec52",
            "created_at": "2022-07-24T08:09:23.205Z",
            "modified_at": "2022-07-24T08:10:30.292Z",
            "input": {
                            ...
            }
        }
    ],
    "completed": true,
    "progress": 1
}

500 Error

{
          "status": "error",
        "task_id": "<task_id>",
        "msg": "can't download the query response - please try again"
}

400 Input Error

{
        "status": "failed",
        "msg": error
}

Response codes

Status	Description
200	OK
400	The requested resource could not be reached
401	Unauthorized/invalid credental string
500	Internal service error
501	An error was encountered by the proxy service

PreviousDelivery methods NextEndpoints and Response Codes

Last updated 5 months ago