Here are some examples on how to bulk download a data product using Python.
1. Create API Key
In the system, click Connections → Add Connection to create your API key.
As the message says, please make a copy of your API key and store it somewhere. Also, please hit the Save button before use.
2. Get a product path
Choose your product and Get / Subscribe → Connect to API then you can get API endpoint (product path). Make a copy of it.
3. Call the API
Example 1 - Download first page of files (up to 1000 files)
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# get results from API endpoint, using API key for authentication
results = requests.get(url=PRODUCT_API_PATH,
params={'page': 1}, # only getting 1st page of results
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
# loop through download links and save to your computer
for i, link_data in enumerate(results.json()["download_links"]):
print(f"Downloading file {i}...")
data = requests.get(link_data["link"])
open(link_data["file_name"], 'wb').write(data.content)
Example 2 - Download all files
Filtering Parameters:
page: Page number (API will return maximum of 1000 files per page), defaulted to 1.
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# loop through all API result pages, keeping track of number of downloaded files
page = 1
download_count = 0
while True:
# get results from API endpoint, using API key for authentication, for a specific page
results = requests.get(url=PRODUCT_API_PATH,
params={'page': page},
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
response_json = results.json()
# for each result page, loop through download links and save to your computer
for link_data in response_json['download_links']:
print(f"Downloading file {link_data['file_name']}...")
data = requests.get(link_data['link'])
with open(link_data['file_name'], 'wb') as file:
file.write(data.content)
download_count += 1
# only continue if there are more result pages to process
total_pages = response_json['total_pages']
if page >= total_pages:
break
page += 1
print(f"Successfully downloaded {download_count} files.")
Example 3 - Download specific files using date filters
Filtering Parameters:
partition_key_after: Retrieve file links for data at or after the specified date in YYYY-MM-DD format
partition_key_before: Retrieve file links for data at or before the specified date in YYYY-MM-DD format
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# loop through all API result pages, keeping track of number of downloaded files
page = 1
download_count = 0
while True:
# get results from API endpoint, using API key for authentication
results = requests.get(url=PRODUCT_API_PATH,
params={'page': page,
'partition_key_after': <'YYYY-MM-DD'>, # optionally set date value here
'partition_key_before': <'YYYY-MM-DD'>}, # optionally set date value here
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
response_json = results.json()
# for each result page, loop through download links and save to your computer
for link_data in response_json['download_links']:
print(f"Downloading file {link_data['file_name']}...")
data = requests.get(link_data['link'])
with open(link_data['file_name'], 'wb') as file:
file.write(data.content)
download_count += 1
# only continue if there are more result pages to process
total_pages = response_json['total_pages']
if page >= total_pages:
break
page += 1
print(f"Successfully downloaded {download_count} files.")
