Delivery methods
Nimble API supports three delivery methods:
For real-time delivery, see our page on performing a real-time URL request. To use Cloud or Push/Pull delivery, use an asynchronous request instead. Asynchronous requests also have the added benefit of running in the background, so you can continue working without waiting for the job to complete.
To send an asynchronous request, use the https://api.webit.live/api/v1/async/web endpoint, such as in the example below:
curl -X POST 'https://api.webit.live/api/v1/async/web' \
--header 'Authorization: Basic <credential string>' \
--header 'Content-Type: application/json' \
--data-raw '{
"url": "https://www.example.com",
"storage_type": "s3",
"storage_url" : "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}'import requests
url = 'https://api.webit.live/api/v1/async/web'
headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
}
data = {
"url": "https://www.example.com",
"storage_type": "s3",
"storage_url" : "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}
response = requests.post(url, headers=headers, json=data)
print(response.status_code)
print(response.json())const axios = require('axios');
const url = 'https://api.webit.live/api/v1/async/web';
const headers = {
'Authorization': 'Basic <credential string>',
'Content-Type': 'application/json'
};
const data = {
"url": "https://www.example.com",
"storage_type": "s3",
"storage_url" : "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
};
axios.post(url, data, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});package main
import (
"bytes"
"encoding/base64"
"fmt"
"net/http"
"encoding/json"
)
func main() {
url := "https://api.webit.live/api/v1/async/web"
payload := []byte(`{
"url": "https://www.example.com",
"storage_type": "s3",
"storage_url": "s3://Your.Repository.Path/",
"callback_url": "https://your.callback.url/path"
}`)
headers := map[string]string{
"Authorization": "Basic <credential string>",
"Content-Type": "application/json",
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}Asynchronous requests share the same parameters as real-time requests, but include a few additional parameters.
Parameters
storage_type
Optional
ENUM: s3 | gs - Use s3 for Amazon S3 and gs for Google Cloud Platform. Leave blank to enable Push/Pull delivery.
storage_url
Optional
Repository URL: s3://Your.Bucket.Name/your/object/name/prefix/ The output will be saved to: s3://Your.Bucket.Name/your/object/name/prefix/TASK_ID.json Leave blank to enable Push/Pull delivery.
callback_url
Optional
A URL to callback once the data is delivered. A POST request will be sent to the callback_url with the task details once the task is complete (this “notification” will not include the requested data).
Setting GCS/AWS access permissions
GCS Repository Configuration
In order to use Google Cloud Storage as your destination repository, please add Nimble’s system user as a principal to the relevant bucket. To do so, navigate to the “bucket details” page in your GCP console, and click on “Permission” in the submenu.
Next, past our system user [email protected] into the “New Principals” box, select Storage Object Creator as the role, and click save.
That’s all! At this point, Nimble will be able to upload files to your chosen GCS bucket.
S3 repository configuration
In order to use S3 as your destination repository, please give Nimble’s service user permission to upload files to the relevant S3 bucket. Paste the following JSON into the “Bucket Policy” (found under “Permissions”) in the AWS console.
Follow these steps:
1. Go to the “Permissions” tab on the bucket’s dashboard:
2. Scroll down to “Bucket policy” and press edit:
3. Paste the following bucket policy configuration into your bucket:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Statement1",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": [
"s3:PutObject",
"s3:PutObjectACL"
],
"Resource": "arn:aws:s3:::YOUR_BUCKET_NAME/*"
},
{
"Sid": "Statement2",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": "s3:GetBucketLocation",
"Resource": "arn:aws:s3:::YOUR_BUCKET_NAME"
}
]
}Important: Remember to replace “YOUR_BUCKET_NAME” with your actual bucket name.
Here is what the bucket policy should look like:
4. Scroll down and press “Save changes”
S3 Encrypted Buckets
If your S3 bucket is encrypted using an AWS Key Management Service (KMS) key, additional permissions to those outlined above are also needed. Specifically, Nimble's service user will need to be given permission to encrypt and decrypt objects using a KMS key. To do this, follow the steps below:
Sign in to the AWS Management Console and open the AWS Key Management Service (KMS) console.
In the navigation pane, choose "Customer managed keys".
Select the KMS key you want to modify.
Choose the "Key policy" tab, then "Switch to policy view".
Click "Edit".
Add the following statement to the existing policy JSON, ensuring it's inside the Statement array:
{
"Version": "2012-10-17",
"Id": "example-key-policy",
"Statement": [
// ... your pre-existing statements ...
{
"Sid": "Allow Nimble APIs account",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::744254827463:user/webit-uploader"
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*"
},
]
}Click "Save changes" to update the key policy.
That's it! You've now given Nimble APIs permission to encrypt and decrypt objects, enabling access to encrypted buckets.
Selecting a delivery method
To enable cloud delivery:
Set the storage_type parameter to either s3 or gs
Set the storage_url parameter to the bucket/folder URL of your cloud storage where you'd like the data to be saved.
To enable Push/Pull delivery:
Leave both the storage_type and storage_url fields blank. Nimble will automatically recognize that Push/Pull delivery has been selected.
Asynchronous Request Process
Unlike real-time requests, asynchronous requests do not return the result directly to the client. Instead, an asynchronous request produces a “task” that runs in the background and delivers the resulting data to a cloud storage bucket and/or callback URL. Tasks go through four stages:
pending
The task is still being processed.
uploading
The results are being uploaded to the destination repository.
success
Task was complete and stored in the destination repository.
failed
Nimble was unable to complete the task, no file was created in the repository.
Initial Response
In response to triggering an asynchronous request, the details of the created task are returned, which can later be used to check its status. The response contains the Task ID, as well as other information, and is structured as follows:
{
"status": "success",
"task": {
"id": "Task_ID",
"state": "pending",
"output_url": "s3://Your.Repository.Path/Task_ID.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://api.webit.live/api/v1/tasks/Task_ID",
"created_at": "0000-00-00T00:00:00.000Z",
"modified_at": "0000-00-00T00:00:00.000Z",
"input": {
"parse": "true",
"render": "false",
"storage_url": "s3://Your.Repository.Path/",
"storage_type": "s3",
"url": "https://www.example.com",
"callback_url": "https://your.callback.url/path"
},
"status_code": 200
}
}Checking Task Status
To check the status of an asynchronous task, use the endpoint https://api.webit.live/api/v1/tasks/<task_id>
For example:
curl -X GET 'https://api.webit.live/api/v1/tasks/Task_ID' \
--header 'Authorization: Basic <credential string>'import requests
url = 'https://api.webit.live/api/v1/tasks/Task_ID'
headers = {
'Authorization': 'Basic <credential string>'
}
response = requests.get(url, headers=headers)
print(response.status_code)
print(response.json())const axios = require('axios');
const url = 'https://api.webit.live/api/v1/tasks/Task_ID';
const headers = {
'Authorization': 'Basic <credential string>'
};
axios.get(url, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});package main
import (
"fmt"
"net/http"
)
func main() {
url := "https://api.webit.live/api/v1/tasks/Task_ID"
headers := map[string]string{
"Authorization": "Basic <credential string>",
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}The response object has the same structure as the Task Completion object that is sent to the callback_url upon task completion.
Checking Tasks List
To check the status of asynchronous tasks list, use the endpoint https://api.webit.live/api/v1/tasks/list
Parameters
limit
Optional (default = 100)
Number | List item limit
cursor
Optional
String | Cursor for pagination.
Example Request:
curl -X GET 'https://api.webit.live/api/v1/tasks/list?limit=20' \
--header 'Authorization: Basic <credential string>'import requests
url = 'https://api.webit.live/api/v1/tasks/list?limit=20'
headers = {
'Authorization': 'Basic <credential string>'
}
response = requests.get(url, headers=headers)
print(response.status_code)
print(response.json())const axios = require('axios');
const url = 'https://api.webit.live/api/v1/tasks/list?limit=20';
const headers = {
'Authorization': 'Basic <credential string>'
};
axios.get(url, { headers })
.then(response => {
console.log(response.status);
console.log(response.data);
})
.catch(error => {
console.error(error);
});package main
import (
"fmt"
"net/http"
)
func main() {
url := "https://api.webit.live/api/v1/tasks/list?limit=20"
headers := map[string]string{
"Authorization": "Basic <credential string>",
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
for key, value := range headers {
req.Header.Set(key, value)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
// Read the response body if needed
// body, err := ioutil.ReadAll(resp.Body)
// fmt.Println(string(body))
}Example Response:
{
"data": [
...
],
"pagination": {
"hasNext": true,
"nextCursor": ...,
"total": 102
}
}The response objects within data has the same structure as the Task Completion object that is sent to the callback_url upon task completion.
For pagination, run until pagination.hasNext = false or pagination.nextCursor = null
Task Completion
A POST request will be sent to the callback_url once the task is complete which contains the following information:
{
"status": "success",
"task": {
"id": "Task_ID",
"state": "success",
"output_url": "s3://Your.Repository.Path/Task_ID.json",
"callback_url": "https://your.callback.url/path",
"status_url": "https://api.webit.live/api/v1/tasks/Task_ID",
"created_at": "0000-00-00T00:00:00.000Z",
"modified_at": "0000-00-00T00:00:00.000Z",
"input": {...},
"status_code": 200
}
}Asynchronous requests also have methods for handling upload failures. For more information, see the Nimble Web API Documentation.
Last updated