This page is no longer being updated
For the latest updates and information, please refer to our new Dewey documentation page at docs.deweydata.io
The information for this page can be found on this page:
Here are some examples on how to bulk download a data product using Python.
1. Create API Key
In the system, click Connections → Add Connection to create your API key.
As the message says, please make a copy of your API key and store it somewhere. Also, please hit the Save button before use.
2. Get a product path
Choose your product and Get / Subscribe → Connect to API then you can get API endpoint (product path). Make a copy of it.
3. Call the API
Example 1 - Download first page of files (up to 1000 files)
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# get results from API endpoint, using API key for authentication
results = requests.get(url=PRODUCT_API_PATH,
params={'page': 1}, # only getting 1st page of results
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
# loop through download links and save to your computer
for i, link_data in enumerate(results.json()["download_links"]):
print(f"Downloading file {i}...")
data = requests.get(link_data["link"])
open(link_data["file_name"], 'wb').write(data.content)
Example 2 - Download all files
Filtering Parameters:
page: Page number (API will return maximum of 1000 files per page), defaulted to 1.
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# loop through all API result pages, keeping track of number of downloaded files
page = 1
download_count = 0
while True:
# get results from API endpoint, using API key for authentication, for a specific page
results = requests.get(url=PRODUCT_API_PATH,
params={'page': page},
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
response_json = results.json()
# for each result page, loop through download links and save to your computer
for link_data in response_json['download_links']:
print(f"Downloading file {link_data['file_name']}...")
data = requests.get(link_data['link'])
with open(link_data['file_name'], 'wb') as file:
file.write(data.content)
download_count += 1
# only continue if there are more result pages to process
total_pages = response_json['total_pages']
if page >= total_pages:
break
page += 1
print(f"Successfully downloaded {download_count} files.")
Example 3 - Download specific files using date filters
Filtering Parameters:
partition_key_after: Retrieve file links for data at or after the specified date in YYYY-MM-DD format
partition_key_before: Retrieve file links for data at or before the specified date in YYYY-MM-DD format
# import requests library to call API endpoint
import requests
# set key and API endpoint variables
API_KEY = <key generated by step 1>
PRODUCT_API_PATH = <URL generated by step 2>
# loop through all API result pages, keeping track of number of downloaded files
page = 1
download_count = 0
while True:
# get results from API endpoint, using API key for authentication
results = requests.get(url=PRODUCT_API_PATH,
params={'page': page,
'partition_key_after': <'YYYY-MM-DD'>, # optionally set date value here
'partition_key_before': <'YYYY-MM-DD'>}, # optionally set date value here
headers={'X-API-KEY': API_KEY,
'accept': 'application/json'
})
response_json = results.json()
# for each result page, loop through download links and save to your computer
for link_data in response_json['download_links']:
print(f"Downloading file {link_data['file_name']}...")
data = requests.get(link_data['link'])
with open(link_data['file_name'], 'wb') as file:
file.write(data.content)
download_count += 1
# only continue if there are more result pages to process
total_pages = response_json['total_pages']
if page >= total_pages:
break
page += 1
print(f"Successfully downloaded {download_count} files.")
