Pagination

An example of how to use pagination for fetching a lot of data at once from the REST API.

Overview

If your application needs to fetch more data than a single request can handle, pagination can be used to seamlessly fetch blocks of the data, or pages, until completion. It splits a large request into smaller ones at the cost of a few extra lines of code. While this example shows how one can fetch projects with pagination, it can be adapted for other requests if desired.

Preliminaries

• Basic Auth For simplicity, we here use Basic Auth for authentication. We recommend replacing this with an OAuth2 flow production-level integrations.

• Service Account Credentials You must create and know the credentials of a Service Account. Any role will suffice.

• REST API This example utilizes our REST API to interact with our cloud. See the REST API Reference for a full list of available endpoints.

Example Code

The following points summarize the provided example code.

• Projects are fetched in groups of 100 at a time.

• When all projects have been fetched, print the total number of projects.

• Iterate through and print the display name for each project.

Environment Setup

If you wish to run the code locally, make sure you have a working runtime environment.

Python 3.11

The following packages are required to run the example code.

pip install requests==2.31.0

Node.js 20

The following modules are required by the example code and must be installed.

npm install [email protected]

Go 1.20

No additional packages are required.

Add the following environment variables as they will be used to authenticate the API.

export DT_SERVICE_ACCOUNT_KEY_ID=<YOUR_SERVICE_ACCOUNT_KEY_ID>
export DT_SERVICE_ACCOUNT_SECRET=<YOUR_SERVICE_ACCOUNT_SECRET>
export DT_SERVICE_ACCOUNT_EMAIL=<YOUR_SERVICE_ACCOUNT_EMAIL>

Source

The following code snippet implements pagination in a few languages.

Python 3.11

import os
import requests

# Authentication credentials loaded from environment.
SERVICE_ACCOUNT_KEY_ID = os.getenv('DT_SERVICE_ACCOUNT_KEY_ID', '')
SERVICE_ACCOUNT_SECRET = os.getenv('DT_SERVICE_ACCOUNT_SECRET', '')

# Shortform Disruptive REST API base url.
BASE_URL = 'https://api.d21s.com/v2'


def paginated_get(url: str, result_field: str, key_id: str, secret: str):
    results = []

    # Create a parameters dictionary that contains an empty page token.
    params: dict = {'pageToken': ''}

    # Loop until all pages have been fetched.
    print('Paging...')
    while True:
        # Send GET request for projects.
        page = requests.get(url, params=params, auth=(key_id, secret)).json()
        if 'error' in page:
            raise Exception(page)
        elif result_field not in page:
            raise KeyError(f'Field "{result_field}" not in response.')

        # Concatenate the request response to output.
        results += page[result_field]
        print(f'- Got {len(page[result_field])} results in page.')

        # Update parameters dictionary with next page token.
        if len(page['nextPageToken']) > 0:
            params['pageToken'] = page['nextPageToken']
        else:
            break

    return results


if __name__ == '__main__':
    # Make paginated requests for all available projects.
    projects = paginated_get(
        url=BASE_URL + '/projects',
        result_field='projects',
        key_id=SERVICE_ACCOUNT_KEY_ID,
        secret=SERVICE_ACCOUNT_SECRET,
    )

    # Print display name of all fetched projects.
    for i, project in enumerate(projects):
        print('{}. {}'.format(i, project['displayName']))

Node.js 20

const axios = require('axios').default

// Environment variables for authentication and target.
const serviceAccountKeyId  = process.env.DT_SERVICE_ACCOUNT_KEY_ID
const serviceAccountSecret = process.env.DT_SERVICE_ACCOUNT_SECRET

// Shortform Disruptive REST API base url.
const baseUrl = 'https://api.d21s.com/v2'

async function paginatedGet(url, resultField, keyId, secret) {
    let results = []

    // Create a parameters dictionary that contains an empty page token.
    let params = {'pageToken': ''}

    // Loop until all pages have been fetched.
    console.log('Paging...')
    while (true) {
        // Send GET request for projects.
        let page = await axios({
            method: 'GET',
            url: url,
            auth: {
                username: keyId,
                password: secret,
            },
            params: params,
        }).catch(function (error) {
            if (error.response) {
                throw new Error(
                    error.response.data.code + ' - '
                    + error.response.data.error
                )
            }
        })

        if (!(resultField in page.data)) {
            throw new Error('Field "' + resultField + '" not in response.')
        }

        // Concatenate response contents to output list.
        results.push(...page.data[resultField])
        console.log(`- ${page.data[resultField].length} projects in page.`)

        // Update parameters with next page token.
        if (page.data.nextPageToken.length > 0) {
            params.pageToken = page.data.nextPageToken
        } else {
            break
        }
    }

    return results
}

async function main () {
    // Make paginated requests for all available projects.
    let projects = []
    projects = await paginatedGet(
        baseUrl + '/projects',
        'projects',
        serviceAccountKeyId,
        serviceAccountSecret,
    )

    // Print display name of all fetched projects.
    for (let i = 0; i < projects.length; i++) {
        console.log(`${i}. ${projects[i]['displayName']}`)
    }
}

main().catch((err) => {console.log(err)});

Go 1.20

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
)

const (
	baseURL = "https://api.d21s.com/v2"
)

// Read the environment variables used for authentication
var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID")
var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET")

// Struct that represents a project fetched from the API
type Project struct {
	Name                    string `json:"name"`
	DisplayName             string `json:"displayName"`
	Inventory               bool   `json:"inventory"`
	Organization            string `json:"organization"`
	OrganizationDisplayName string `json:"organizationDisplayName"`
	Sensorcount             int    `json:"sensorCount"`
	CloudConnectorCount     int    `json:"cloudConnectorCount"`
}

// Sends a GET request to the Disruptive REST API and returns the response body.
func getRequest(endpointURL string, queryParams map[string]string) ([]byte, error) {
	// Create the HTTP GET request
	req, err := http.NewRequest("GET", endpointURL, nil)
	if err != nil {
		return nil, err
	}

	// Set the query parameters for the request
	q := req.URL.Query()
	for key, value := range queryParams {
		q.Set(key, value)
	}
	req.URL.RawQuery = q.Encode()

	// Set the Authorization header to use HTTP Basic Auth
	req.SetBasicAuth(keyID, secret)

	// Send the HTTP request with a 3 second timout in case we
	// are unable to reach the server.
	client := &http.Client{Timeout: time.Second * 3}
	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	return ioutil.ReadAll(response.Body)
}

// getAllProjects retrieves all projects that are accessible to the
// Service Account, one page at a time.
func paginatedGetProjects() ([]Project, error) {
	// Initialize empty output slice.
	projects := []Project{}

	// Define the structure of each page.
	type ProjectsPage struct {
		Projects      []Project `json:"projects"`
		NextPageToken string    `json:"nextPageToken"`
	}

	// Start with an empty page token to request the first page.
	// We'll update this variable to point to the subsequent pages.
	var pageToken = ""

	// Loop until all pages have been fetched
	for {
		// Define the query parameters for the request.
		// The "page_token" query parameter specifies the page
		// we want to get. The "page_size" parameter is included
		// just for the sake of completion. The default page size is 100.
		params := map[string]string{
			"page_token": pageToken,
			"page_size":  "100",
		}

		// Send GET request to the endpoint and get bytes back
		pageBytes, err := getRequest(baseURL+"/projects", params)
		if err != nil {
			return nil, err
		}

		// Decode pageBytes into the expected page structure
		page := ProjectsPage{}
		if err := json.Unmarshal(pageBytes, &page); err != nil {
			return nil, err
		}

		// Append the retrieved page of projects to our output slice
		projects = append(projects, page.Projects...)
		fmt.Printf("Got a page of %d projects\n", len(page.Projects))

		// Update the page token to point to the next page
		pageToken = page.NextPageToken

		// If we got an empty "next_page_token" in the response,
		// it means there are no more pages to fetch.
		if len(pageToken) == 0 {
			break
		}
	}

	return projects, nil
}

func main() {
	// Make paginated requests for all available projects.
	projects, err := paginatedGetProjects()
	if err != nil {
		log.Fatal(err)
	}

	// Print display name of all fetched projects.
	for i, project := range projects {
		fmt.Printf("%d. %s\n", i, project.DisplayName)
	}
}

Expected Output

The number of projects resulting from each page will be displayed in the stdout.

Paging...
- Got 100 projects in page.
- Got 100 projects in page.
- Got 100 projects in page.
- Got 68 projects in page.

0. my-project-A
1. my-project-B
2. my-project-C
3. ...