Explore CDSE ODATA API

Explore CDSE ODATA API#

Notebook Description#

This Jupyter Notebook demonstrates how to explore and interact with the Copernicus Data Space Ecosystem (CDSE) ODATA API. The notebook provides a step-by-step guide to querying metadata, visualizing product footprints, and downloading geospatial data from various collections.

Key Steps#

  1. Library Imports:

    • Utilizes libraries such as requests, geopandas, folium, shapely, and matplotlib for querying, processing, and visualizing geospatial data.

  2. Authentication:

    • Demonstrates how to generate an access token using a username and password for secure interaction with the ODATA API.

  3. Querying the ODATA API:

    • Searches for products in specific collections (e.g., Sentinel-2, Sentinel-1, Landsat-8) based on spatial (bounding box or AOI), temporal (date range), and cloud cover filters.

  4. Footprint Visualization:

    • Visualizes the footprints of queried products on an interactive map using folium.

  5. Data Download:

    • Downloads selected products as ZIP files and extracts them for further analysis.

  6. Customizable Parameters:

    • Allows users to specify parameters such as collection name, cloud cover limit, area of interest (AOI), and date range for tailored queries.

Use Case#

This notebook is ideal for users who want to:

  • Explore metadata and product availability in the Copernicus Data Space Ecosystem.

  • Visualize product footprints interactively on a map.

  • Download geospatial data for further analysis.

Prerequisites#

  • Python environment with the required libraries installed (requests, geopandas, folium, shapely, matplotlib).

  • A valid CDSE account with a username and password.

  • Basic understanding of Python and Jupyter Notebooks.

  • Familiarity with geospatial data concepts.

Import the required libraries#

import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import requests
import zipfile
import geopandas as gpd
import folium
from shapely import wkt
from dotenv import load_dotenv
load_dotenv("../.env")

True

Load username and password from env file#

USERNAME = os.getenv('USERNAME')
PASSWORD = os.getenv('PASSWORD')

Helping functions to create geometry from string and convert bounding box format#

def clean_footprint(footprint):
    wkt_str = footprint.split(";")[1][:-1]
    return wkt.loads(wkt_str)

def convert_bounds(bbox, invert_y=False):
    """
    Helper method for changing bounding box representation to leaflet notation

    ``(lon1, lat1, lon2, lat2) -> ((lat1, lon1), (lat2, lon2))``
    """
    x1, y1, x2, y2 = bbox
    if invert_y:
        y1, y2 = y2, y1
    return ((y1, x1), (y2, x2))

Get access token using username and password#

def get_access_token(username: str, password: str) -> str:
    data = {
        "client_id": "cdse-public",
        "username": username,
        "password": password,
        "grant_type": "password",
    }
    try:
        r = requests.post(
            "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token",
            data=data,
        )
        r.raise_for_status()
    except Exception as e:
        raise Exception(
            f"Access token creation failed. Reponse from the server was: {r.json()}"
        )
    return r.json()["access_token"]

Query the CDSE ODATA API for differet collections#

# Configuration
start_date = "2024-06-01"
end_date = "2024-09-01"
#data_collection = "LANDSAT-8-ESA"
data_collection = "SENTINEL-2"
# data_collection = "SENTINEL-1"
cloud_cover_limit = 5.00  # Maximum cloud cover percentage
aoi = "POLYGON((19.6 41.8, 22.0 41.8, 26.5 41.2, 28.3 40.3, 28.0 38.8, 26.0 36.0, 24.0 35.0, 22.0 36.0, 20.0 37.0, 19.0 38.5, 19.0 40.0, 19.6 41.8))"
# tile_id  = "30TXQ"
product_type = "S2MSI2A"
point_location = "POINT(-1.3009 44.5556)"  # Localisation du point au format WKT

#Search in the Copernicus Catalogue
response = requests.get(
    f"https://catalogue.dataspace.copernicus.eu/odata/v1/Products?$filter="
    f"Collection/Name eq '{data_collection}' and "
    f"OData.CSC.Intersects(area=geography'SRID=4326;{aoi}') and "
    f"Attributes/OData.CSC.DoubleAttribute/any(att:att/Name eq 'cloudCover' and "
    f"att/OData.CSC.DoubleAttribute/Value lt {cloud_cover_limit}) and "
    # # f"Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'tileId' and "
    # # f"att/OData.CSC.StringAttribute/Value eq '{tile_id}') and "
    # f"Attributes/OData.CSC.StringAttribute/any(att:att/Name eq 'productType' and "
    # f"att/OData.CSC.StringAttribute/Value eq '{product_type}') and "
    # # f"OData.CSC.Within(area=geography'SRID=4326;{point_location}') and "
    # f"OData.CSC.Intersects(area=geography'SRID=4326;{point_location}') and "
    f"ContentDate/Start gt {start_date}T00:00:00.000Z and "
    f"ContentDate/Start lt {end_date}T00:00:00.000Z&$top=100"
)

response.raise_for_status()
metadata = response.json()
products_df = pd.DataFrame.from_dict(metadata["value"])
print(f"Found {len(products_df)} products.")
products_df['geometry'] = products_df['Footprint'].apply(clean_footprint)
gdf = gpd.GeoDataFrame(products_df, geometry='geometry')
gdf.set_crs("EPSG:4326", inplace=True)

Found 100 products.
@odata.mediaContentType Id Name ContentType ContentLength OriginDate PublicationDate ModificationDate Online EvictionDate S3Path Checksum ContentDate Footprint GeoFootprint geometry
0 application/octet-stream d63cf75d-991d-41fa-9eb4-8f3113f034e7 S2A_MSIL2A_20240607T094041_N0510_R036_T34TCL_2... application/octet-stream 755009760 2024-06-07T13:32:15.000000Z 2024-06-07T13:44:01.038477Z 2024-11-11T12:15:47.721572Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/07/S2A_MSIL... [{'Value': '6a9af52984d62382693acde029f58f0b',... {'Start': '2024-06-07T09:40:41.024000Z', 'End'... geography'SRID=4326;POLYGON ((19.6816974043764... {'type': 'Polygon', 'coordinates': [[[19.68169... POLYGON ((19.6817 41.54318, 18.60282 41.52685,...
1 application/octet-stream b2b7f98c-02d9-47bc-a140-be722a140571 S2A_MSIL2A_20240607T094041_N0510_R036_T34SCJ_2... application/octet-stream 164282618 2024-06-07T13:31:41.000000Z 2024-06-07T13:43:31.828287Z 2024-11-11T12:15:58.747447Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/07/S2A_MSIL... [{'Value': '863c1af27581008a95dc18031b03d01c',... {'Start': '2024-06-07T09:40:41.024000Z', 'End'... geography'SRID=4326;POLYGON ((18.7416089837887... {'type': 'Polygon', 'coordinates': [[[18.74160... POLYGON ((18.74161 38.91958, 18.74314 38.92399...
2 application/octet-stream 47b03d27-f7d2-442a-9dcc-50909068f818 S2B_MSIL1C_20240630T085559_N0510_R007_T35SND_2... application/octet-stream 882138284 2024-06-30T11:29:42.000000Z 2024-06-30T11:37:28.750697Z 2024-11-09T10:48:57.344322Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L1C/2024/06/30/S2B_MSIL... [{'Value': 'aad5230e92fc1b6801a0d93f7ddca00c',... {'Start': '2024-06-30T08:55:59.024000Z', 'End'... geography'SRID=4326;POLYGON ((26.9997665456227... {'type': 'Polygon', 'coordinates': [[[26.99976... POLYGON ((26.99977 39.75027, 26.99977 38.76086...
3 application/octet-stream 6a8ac0dc-b632-44ee-ae75-ea454492ad65 S2B_MSIL2A_20240630T085559_N0510_R007_T35TNE_2... application/octet-stream 1174323525 2024-06-30T12:21:04.000000Z 2024-06-30T12:36:10.616897Z 2024-11-09T10:48:17.235509Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/30/S2B_MSIL... [{'Value': 'c7665e40df94d28fb4ec84fca32fc86d',... {'Start': '2024-06-30T08:55:59.024000Z', 'End'... geography'SRID=4326;POLYGON ((26.9997634361085... {'type': 'Polygon', 'coordinates': [[[26.99976... POLYGON ((26.99976 40.65086, 26.99977 39.66161...
4 application/octet-stream fe48c186-fdb5-4ef0-a8a8-2b8763543834 S2B_MSIL2A_20240630T085559_N0510_R007_T34SGE_2... application/octet-stream 78150988 2024-06-30T11:42:36.000000Z 2024-06-30T11:49:44.903938Z 2024-11-09T10:45:28.873467Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/30/S2B_MSIL... [{'Value': 'b882d2d3e298d53f41891a066259dba4',... {'Start': '2024-06-30T08:55:59.024000Z', 'End'... geography'SRID=4326;POLYGON ((24.2331750694851... {'type': 'Polygon', 'coordinates': [[[24.23317... POLYGON ((24.23318 35.11103, 24.39833 35.10722...
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
95 application/octet-stream 91f02128-a20e-42ab-8085-f6f5124357e8 S2A_MSIL2A_20240624T093041_N0510_R136_T34SCJ_2... application/octet-stream 1054723531 2024-06-24T14:05:02.000000Z 2024-06-24T14:18:12.391549Z 2024-11-10T00:44:02.507249Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/24/S2A_MSIL... [{'Value': 'fc27bdef54452239ed20d7b9a3380b98',... {'Start': '2024-06-24T09:30:41.024000Z', 'End'... geography'SRID=4326;POLYGON ((18.6663671694377... {'type': 'Polygon', 'coordinates': [[[18.66636... POLYGON ((18.66637 39.72681, 18.6989 38.73821,...
96 application/octet-stream f5529255-4b33-40e2-a7bb-17ca694ca79b S2A_MSIL2A_20240624T093041_N0510_R136_T34TCK_2... application/octet-stream 1104061118 2024-06-24T14:05:03.000000Z 2024-06-24T14:18:30.427790Z 2024-11-10T00:43:01.565370Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/24/S2A_MSIL... [{'Value': '3fc02d552468d8e4fc52e2e9eca79cf8',... {'Start': '2024-06-24T09:30:41.024000Z', 'End'... geography'SRID=4326;POLYGON ((18.6353192129726... {'type': 'Polygon', 'coordinates': [[[18.63531... POLYGON ((18.63532 40.62664, 18.66935 39.63822...
97 application/octet-stream d4f54da9-df46-46b8-ba47-25f60a80d0a9 S2A_MSIL2A_20240608T090601_N0510_R050_T34TGK_2... application/octet-stream 972343913 2024-06-08T13:30:26.000000Z 2024-06-08T13:42:28.353262Z 2024-11-11T10:22:07.345864Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/08/S2A_MSIL... [{'Value': '24341f314e2e35dc0c29a15ead15dd6a',... {'Start': '2024-06-08T09:06:01.024000Z', 'End'... geography'SRID=4326;POLYGON ((23.3642082337128... {'type': 'Polygon', 'coordinates': [[[23.36420... POLYGON ((23.36421 40.62665, 23.33019 39.63823...
98 application/octet-stream 9f86254a-5d99-453f-a4a3-aea18aaf5c36 S2A_MSIL2A_20240608T090601_N0510_R050_T34SFH_2... application/octet-stream 759911864 2024-06-08T13:31:04.000000Z 2024-06-08T13:41:08.985614Z 2024-11-11T10:22:36.477704Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/08/S2A_MSIL... [{'Value': '959ce36bfb6864a8d61d2b6c0334224c',... {'Start': '2024-06-08T09:06:01.024000Z', 'End'... geography'SRID=4326;POLYGON ((22.4824460421271... {'type': 'Polygon', 'coordinates': [[[22.48244... POLYGON ((22.48245 37.84879, 23.3841 37.8353, ...
99 application/octet-stream 1347fcde-5c9e-4aea-aad7-e8d008ba983e S2A_MSIL2A_20240608T090601_N0510_R050_T35SLB_2... application/octet-stream 933844693 2024-06-08T13:30:56.000000Z 2024-06-08T13:40:11.253251Z 2024-11-11T10:24:17.546703Z True 9999-12-31T23:59:59.999999Z /eodata/Sentinel-2/MSI/L2A/2024/06/08/S2A_MSIL... [{'Value': '115aa839a49809ea0b72d09fb2b0f6e6',... {'Start': '2024-06-08T09:06:01.024000Z', 'End'... geography'SRID=4326;POLYGON ((25.9586605029682... {'type': 'Polygon', 'coordinates': [[[25.95866... POLYGON ((25.95866 37.9429, 24.72447 37.92558,...

100 rows × 16 columns

Visualize products footprints using folium#

map1 = folium.Map()
gdf.explore(
    "Id",
    categorical=True,
    tooltip=[
        "Id",
        "Name",
        "OriginDate",
    ],
    popup=True,
    style_kwds=dict(fillOpacity=0.1, width=2),
    name="ODATA",
    m=map1,
)

map1.fit_bounds(bounds=convert_bounds(gdf.unary_union.bounds))
map1

/var/folders/j3/513qxyhx4l30byl48tz1k1jr0000gn/T/ipykernel_25311/4062689537.py:16: DeprecationWarning: The 'unary_union' attribute is deprecated, use the 'union_all()' method instead.
  map1.fit_bounds(bounds=convert_bounds(gdf.unary_union.bounds))
Make this Notebook Trusted to load map: File -> Trust Notebook

Generate access token to be used to download data#

access_token = get_access_token(USERNAME, PASSWORD)

headers = {"Authorization": f"Bearer {access_token}"}
download_folder = "../data/downloaded-products"
extracted_folder = "../data/extracted-products"
os.makedirs(download_folder, exist_ok=True)

Download data from CDSE ODATA API and extract it#

for _, product in products_df.iterrows():
    product_id = product["Id"]
    product_title = product["Name"].replace(" ", "_")
    shortened_name = "_".join(product_title.split("_")[:6])

    download_url = f"https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value"

    # Start the download session
    session = requests.Session()
    session.headers.update(headers)
    response = session.get(download_url, stream=True)

    # Save the product as a ZIP file

    zip_file_path = os.path.join(download_folder, f"{shortened_name}.zip")

    if response.status_code == 200:
        with open(zip_file_path, "wb") as file:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    file.write(chunk)
        print(f"Downloaded: {zip_file_path}")


        # Extract the ZIP file
        try:
            with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
                extraction_path = os.path.join(extracted_folder, shortened_name)
                os.makedirs(extraction_path, exist_ok=True)
                zip_ref.extractall(extraction_path)
            print(f"Extracted to: {extraction_path}")
        except zipfile.BadZipFile:
            print(f"Error: {zip_file_path} is not a valid ZIP file.")

    else:
        print(f"Failed to download product {product_title}. HTTP status: {response.status_code}")

    break

Downloaded: ../data/downloaded-products/S2A_MSIL2A_20240607T094041_N0510_R036_T34TCL.zip
Extracted to: ../data/extracted-products/S2A_MSIL2A_20240607T094041_N0510_R036_T34TCL
Contents