Pagination

An example of how to use pagination for fetching a lot of data at once from the REST API.

Overview

If your application needs to fetch more data than a single request can handle, pagination can be used to seamlessly fetch blocks of the data, or pages, until completion. It splits a large request into smaller ones at the cost of a few extra lines of code. While this example shows how one can fetch projects with pagination, it can be adapted for other requests if desired.

Preliminaries

Basic Auth For simplicity, we here use Basic Auth for authentication. We recommend replacing this with an OAuth2 flow production-level integrations.
Service Account Credentials You must create and know the credentials of a Service Account. Any role will suffice.
REST API This example utilizes our REST API to interact with our cloud. See the REST API Reference for a full list of available endpoints.

Example Code

The following points summarize the provided example code.

Projects are fetched in groups of 100 at a time.
When all projects have been fetched, print the total number of projects.
Iterate through and print the display name for each project.

Environment Setup

If you wish to run the code locally, make sure you have a working runtime environment.

The following packages are required to run the example code.

pip install requests==2.31.0

Add the following environment variables as they will be used to authenticate the API.

export DT_SERVICE_ACCOUNT_KEY_ID=<YOUR_SERVICE_ACCOUNT_KEY_ID>
export DT_SERVICE_ACCOUNT_SECRET=<YOUR_SERVICE_ACCOUNT_SECRET>
export DT_SERVICE_ACCOUNT_EMAIL=<YOUR_SERVICE_ACCOUNT_EMAIL>

Source

The following code snippet implements pagination in a few languages.

import os
import requests

# Authentication credentials loaded from environment.
SERVICE_ACCOUNT_KEY_ID = os.getenv('DT_SERVICE_ACCOUNT_KEY_ID', '')
SERVICE_ACCOUNT_SECRET = os.getenv('DT_SERVICE_ACCOUNT_SECRET', '')

# Shortform Disruptive REST API base url.
BASE_URL = 'https://api.d21s.com/v2'


def paginated_get(url: str, result_field: str, key_id: str, secret: str):
    results = []

    # Create a parameters dictionary that contains an empty page token.
    params: dict = {'pageToken': ''}

    # Loop until all pages have been fetched.
    print('Paging...')
    while True:
        # Send GET request for projects.
        page = requests.get(url, params=params, auth=(key_id, secret)).json()
        if 'error' in page:
            raise Exception(page)
        elif result_field not in page:
            raise KeyError(f'Field "{result_field}" not in response.')

        # Concatenate the request response to output.
        results += page[result_field]
        print(f'- Got {len(page[result_field])} results in page.')

        # Update parameters dictionary with next page token.
        if len(page['nextPageToken']) > 0:
            params['pageToken'] = page['nextPageToken']
        else:
            break

    return results


if __name__ == '__main__':
    # Make paginated requests for all available projects.
    projects = paginated_get(
        url=BASE_URL + '/projects',
        result_field='projects',
        key_id=SERVICE_ACCOUNT_KEY_ID,
        secret=SERVICE_ACCOUNT_SECRET,
    )

    # Print display name of all fetched projects.
    for i, project in enumerate(projects):
        print('{}. {}'.format(i, project['displayName']))

const axios = require('axios').default

// Environment variables for authentication and target.
const serviceAccountKeyId  = process.env.DT_SERVICE_ACCOUNT_KEY_ID
const serviceAccountSecret = process.env.DT_SERVICE_ACCOUNT_SECRET

// Shortform Disruptive REST API base url.
const baseUrl = 'https://api.d21s.com/v2'

async function paginatedGet(url, resultField, keyId, secret) {
    let results = []

    // Create a parameters dictionary that contains an empty page token.
    let params = {'pageToken': ''}

    // Loop until all pages have been fetched.
    console.log('Paging...')
    while (true) {
        // Send GET request for projects.
        let page = await axios({
            method: 'GET',
            url: url,
            auth: {
                username: keyId,
                password: secret,
            },
            params: params,
        }).catch(function (error) {
            if (error.response) {
                throw new Error(
                    error.response.data.code + ' - '
                    + error.response.data.error
                )
            }
        })

        if (!(resultField in page.data)) {
            throw new Error('Field "' + resultField + '" not in response.')
        }

        // Concatenate response contents to output list.
        results.push(...page.data[resultField])
        console.log(`- ${page.data[resultField].length} projects in page.`)

        // Update parameters with next page token.
        if (page.data.nextPageToken.length > 0) {
            params.pageToken = page.data.nextPageToken
        } else {
            break
        }
    }

    return results
}

async function main () {
    // Make paginated requests for all available projects.
    let projects = []
    projects = await paginatedGet(
        baseUrl + '/projects',
        'projects',
        serviceAccountKeyId,
        serviceAccountSecret,
    )

    // Print display name of all fetched projects.
    for (let i = 0; i < projects.length; i++) {
        console.log(`${i}. ${projects[i]['displayName']}`)
    }
}

main().catch((err) => {console.log(err)});

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
)

const (
	baseURL = "https://api.d21s.com/v2"
)

// Read the environment variables used for authentication
var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID")
var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET")

// Struct that represents a project fetched from the API
type Project struct {
	Name                    string `json:"name"`
	DisplayName             string `json:"displayName"`
	Inventory               bool   `json:"inventory"`
	Organization            string `json:"organization"`
	OrganizationDisplayName string `json:"organizationDisplayName"`
	Sensorcount             int    `json:"sensorCount"`
	CloudConnectorCount     int    `json:"cloudConnectorCount"`
}

// Sends a GET request to the Disruptive REST API and returns the response body.
func getRequest(endpointURL string, queryParams map[string]string) ([]byte, error) {
	// Create the HTTP GET request
	req, err := http.NewRequest("GET", endpointURL, nil)
	if err != nil {
		return nil, err
	}

	// Set the query parameters for the request
	q := req.URL.Query()
	for key, value := range queryParams {
		q.Set(key, value)
	}
	req.URL.RawQuery = q.Encode()

	// Set the Authorization header to use HTTP Basic Auth
	req.SetBasicAuth(keyID, secret)

	// Send the HTTP request with a 3 second timout in case we
	// are unable to reach the server.
	client := &http.Client{Timeout: time.Second * 3}
	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	return ioutil.ReadAll(response.Body)
}

// getAllProjects retrieves all projects that are accessible to the
// Service Account, one page at a time.
func paginatedGetProjects() ([]Project, error) {
	// Initialize empty output slice.
	projects := []Project{}

	// Define the structure of each page.
	type ProjectsPage struct {
		Projects      []Project `json:"projects"`
		NextPageToken string    `json:"nextPageToken"`
	}

	// Start with an empty page token to request the first page.
	// We'll update this variable to point to the subsequent pages.
	var pageToken = ""

	// Loop until all pages have been fetched
	for {
		// Define the query parameters for the request.
		// The "page_token" query parameter specifies the page
		// we want to get. The "page_size" parameter is included
		// just for the sake of completion. The default page size is 100.
		params := map[string]string{
			"page_token": pageToken,
			"page_size":  "100",
		}

		// Send GET request to the endpoint and get bytes back
		pageBytes, err := getRequest(baseURL+"/projects", params)
		if err != nil {
			return nil, err
		}

		// Decode pageBytes into the expected page structure
		page := ProjectsPage{}
		if err := json.Unmarshal(pageBytes, &page); err != nil {
			return nil, err
		}

		// Append the retrieved page of projects to our output slice
		projects = append(projects, page.Projects...)
		fmt.Printf("Got a page of %d projects\n", len(page.Projects))

		// Update the page token to point to the next page
		pageToken = page.NextPageToken

		// If we got an empty "next_page_token" in the response,
		// it means there are no more pages to fetch.
		if len(pageToken) == 0 {
			break
		}
	}

	return projects, nil
}

func main() {
	// Make paginated requests for all available projects.
	projects, err := paginatedGetProjects()
	if err != nil {
		log.Fatal(err)
	}

	// Print display name of all fetched projects.
	for i, project := range projects {
		fmt.Printf("%d. %s\n", i, project.DisplayName)
	}
}

Expected Output

The number of projects resulting from each page will be displayed in the stdout.

Paging...
- Got 100 projects in page.
- Got 100 projects in page.
- Got 100 projects in page.
- Got 68 projects in page.

0. my-project-A
1. my-project-B
2. my-project-C
3. ...

Last updated 2 years ago

Was this helpful?

package main import ( "encoding/json" "fmt" "io/ioutil" "log" "net/http" "os" "time" ) const ( baseURL = "https://api.d21s.com/v2" ) // Read the environment variables used for authentication var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID") var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET") // Struct that represents a project fetched from the API type Project struct { Name string `json:"name"` DisplayName string `json:"displayName"` Inventory bool `json:"inventory"` Organization string `json:"organization"` OrganizationDisplayName string `json:"organizationDisplayName"` Sensorcount int `json:"sensorCount"` CloudConnectorCount int `json:"cloudConnectorCount"` } // Sends a GET request to the Disruptive REST API and returns the response body. func getRequest(endpointURL string, queryParams map[string]string) ([]byte, error) { // Create the HTTP GET request req, err := http.NewRequest("GET", endpointURL, nil) if err != nil { return nil, err } // Set the query parameters for the request q := req.URL.Query() for key, value := range queryParams { q.Set(key, value) } req.URL.RawQuery = q.Encode() // Set the Authorization header to use HTTP Basic Auth req.SetBasicAuth(keyID, secret) // Send the HTTP request with a 3 second timout in case we // are unable to reach the server. client := &http.Client{Timeout: time.Second * 3} response, err := client.Do(req) if err != nil { return nil, err } defer response.Body.Close() return ioutil.ReadAll(response.Body) } // getAllProjects retrieves all projects that are accessible to the // Service Account, one page at a time. func paginatedGetProjects() ([]Project, error) { // Initialize empty output slice. projects := []Project{} // Define the structure of each page. type ProjectsPage struct { Projects []Project `json:"projects"` NextPageToken string `json:"nextPageToken"` } // Start with an empty page token to request the first page. // We'll update this variable to point to the subsequent pages. var pageToken = "" // Loop until all pages have been fetched for { // Define the query parameters for the request. // The "page_token" query parameter specifies the page // we want to get. The "page_size" parameter is included // just for the sake of completion. The default page size is 100. params := map[string]string{ "page_token": pageToken, "page_size": "100", } // Send GET request to the endpoint and get bytes back pageBytes, err := getRequest(baseURL+"/projects", params) if err != nil { return nil, err } // Decode pageBytes into the expected page structure page := ProjectsPage{} if err := json.Unmarshal(pageBytes, &page); err != nil { return nil, err } // Append the retrieved page of projects to our output slice projects = append(projects, page.Projects...) fmt.Printf("Got a page of %d projects\n", len(page.Projects)) // Update the page token to point to the next page pageToken = page.NextPageToken // If we got an empty "next_page_token" in the response, // it means there are no more pages to fetch. if len(pageToken) == 0 { break } } return projects, nil } func main() { // Make paginated requests for all available projects. projects, err := paginatedGetProjects() if err != nil { log.Fatal(err) } // Print display name of all fetched projects. for i, project := range projects { fmt.Printf("%d. %s\n", i, project.DisplayName) } }