from glob import glob
import xarray as xr
import cftime
import nc_time_axis
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
pd.options.display.max_rows = 200
import intake, intake_esm
! pip install cmip6_preprocessing
from cmip6_preprocessing.preprocessing import (correct_units,rename_cmip6)
def chunk_time(ds):
if 'time' in ds.dims:
ds = ds.chunk({'time':1})
return ds
# Necessary for creating a common time axis for all models
# We want to create a common time axis so there will be no gaps when plotting the results
def fix_time(ds):
""" force calendar to noleap"""
import xarray as xr
if "time" not in ds.dims:
return ds
if ("calendar" not in ds["time"].attrs):
ds["time"].attrs.update({"calendar": "noleap"})
if ds["time"].attrs["calendar"] not in ["noleap", "NOLEAP", "365_day"]:
ds["time"].attrs.update({"calendar": "noleap"})
ds = xr.decode_cf(ds)
return ds
# Pass this function for preprocessing thetao data
def pp_thetao(ds):
ds = rename_cmip6(ds)
ds = fix_time(ds)
ds = correct_units(ds)
return ds
# Pass this function for preprocessing volcello and areacello data
def pp_volcello(ds):
ds = rename_cmip6(ds)
ds = chunk_time(ds)
ds = fix_time(ds)
ds = correct_units(ds)
return ds
# Use this function to reconstruct areacello
def compute_area_regular_grid(ds, Rearth=6378e3):
""" compute the cells area on a regular grid """
rfac = 2 * np.pi * Rearth / 360
dx1d = rfac * 1
dy1d = rfac * 1
dx2d, dy2d = np.meshgrid(dx1d, dy1d)
_, lat2d = np.meshgrid(ds["x"].values, ds["y"].values)
dx = dx2d * np.cos(2 * np.pi * lat2d / 360)
dy = dy2d
area = dx * dy
return xr.DataArray(area, dims=('y', 'x'))
col_url = "https://raw.githubusercontent.com/aradhakrishnanGFDL/gfdl-aws-analysis/community/esm-collection-spec-examples/esgf-world.json"
col = intake.open_esm_datastore(col_url)
esmcol_data = col.esmcol_data
col
gfdltest catalog with 9266 dataset(s) from 956306 asset(s):
unique | |
---|---|
project | 1 |
institute | 38 |
model | 106 |
experiment_id | 217 |
frequency | 2 |
modeling_realm | 3 |
mip_table | 37 |
ensemble_member | 2725 |
grid_label | 13 |
variable | 738 |
temporal subset | 25151 |
version | 510 |
path | 956306 |
#def latest_version(ds):
# """filters latest DRS versions of datasets only"""
# ds=ds.df.sort_values(['version'],ascending=False).groupby(['temporal subset','model','mip_table',
# 'institute','variable','ensemble_member',
# 'grid_label','experiment_id'],as_index=False)#.head(1)
# return ds
def latest_version(cat):
"""
input
cat: esmdatastore
output
esmdatastore with latest DRS versions
"""
latest_cat = cat.df.sort_values(by=['version','path']).drop_duplicates(['temporal subset','model','mip_table',
'institute','variable','ensemble_member',
'grid_label','experiment_id'],keep='last')
return latest_cat
# gn_models = ['ACCESS-ESM1-5','BCC-CSM2-MR','BCC-ESM1','CAMS-CSM1-0','EC-Earth3',
# 'EC-Earth3-Veg','FIO-ESM-2-0','GISS-E2-1-G','GISS-E2-1-G-CC','NESM3','ACCESS-CM2','CIESM','SAM0-UNICON','MPI-ESM1-2-HR',
# 'CanESM5','FGOALS-f3-L','IPSL-CM6A-LR','MIROC6','MPI-ESM-1-2-HAM','MPI-ESM1-2-LR']
gn_models = ['CESM2', 'CESM2-FV2', 'CESM2-WACCM-FV2','IPSL-CM6A-LR','MPI-ESM1-2-HR','MPI-ESM1-2-LR','MIROC6','CanESM5',
'MPI-ESM-1-2-HAM','MRI-ESM2-0','SAM0-UNICON']
cat_T_gn = col.search(experiment_id=['historical'],
mip_table='Omon',
ensemble_member=["r1i1p1f1"],
model=gn_models,
grid_label=['gn'],
variable=["thetao"])
cat_VOmon_gn = col.search(experiment_id=['historical'],
mip_table=['Omon'],
ensemble_member="r1i1p1f1",
model=gn_models,
grid_label='gn',
variable=["volcello"])
cat_VOfx_gn = col.search(experiment_id=['historical'],
mip_table=['Ofx'],
ensemble_member="r1i1p1f1",
model=gn_models,
grid_label='gn',
variable=["volcello"])
cat_A_gn = col.search(experiment_id=['historical'],
mip_table=['Omon','Ofx'],
ensemble_member="r1i1p1f1",
model=gn_models,
grid_label='gn',
variable=['areacello'])
/srv/conda/envs/notebook/lib/python3.8/site-packages/intake_esm/search.py:102: UserWarning: Query returned zero results. warn(message)
cat_VOfx_gn.df
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | CMIP6 | MPI-M | MPI-ESM1-2-HR | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190710 | s3://esgf-world/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR... |
1 | CMIP6 | NCAR | CESM2-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-FV2/hist... |
2 | CMIP6 | NCAR | CESM2-WACCM-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-WACCM-FV... |
3 | CMIP6 | NCAR | CESM2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190308 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2/historic... |
4 | CMIP6 | SNU | SAM0-UNICON | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190323 | s3://esgf-world/CMIP6/CMIP/SNU/SAM0-UNICON/his... |
cat_T_gn_latest = latest_version(cat_T_gn)
cat_VOmon_gn_latest = latest_version(cat_VOmon_gn)
cat_VOfx_gn_latest = latest_version(cat_VOfx_gn)
cat_A_gn_latest = latest_version(cat_A_gn)
cat_VOfx_gn_latest.head()
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
3 | CMIP6 | NCAR | CESM2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190308 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2/historic... |
4 | CMIP6 | SNU | SAM0-UNICON | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190323 | s3://esgf-world/CMIP6/CMIP/SNU/SAM0-UNICON/his... |
0 | CMIP6 | MPI-M | MPI-ESM1-2-HR | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190710 | s3://esgf-world/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR... |
1 | CMIP6 | NCAR | CESM2-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-FV2/hist... |
2 | CMIP6 | NCAR | CESM2-WACCM-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-WACCM-FV... |
cat_VOfx_gn.df
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | CMIP6 | MPI-M | MPI-ESM1-2-HR | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190710 | s3://esgf-world/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR... |
1 | CMIP6 | NCAR | CESM2-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-FV2/hist... |
2 | CMIP6 | NCAR | CESM2-WACCM-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-WACCM-FV... |
3 | CMIP6 | NCAR | CESM2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190308 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2/historic... |
4 | CMIP6 | SNU | SAM0-UNICON | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190323 | s3://esgf-world/CMIP6/CMIP/SNU/SAM0-UNICON/his... |
cat_T_gn = intake.open_esm_datastore(cat_T_gn_latest,esmcol_data=esmcol_data)
cat_VOmon_gn = intake.open_esm_datastore(cat_VOmon_gn_latest,esmcol_data=esmcol_data)
cat_VOfx_gn = intake.open_esm_datastore(cat_VOfx_gn_latest,esmcol_data=esmcol_data)
cat_A_gn = intake.open_esm_datastore(cat_A_gn_latest,esmcol_data=esmcol_data)
cat_VOfx_gn_latest
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
3 | CMIP6 | NCAR | CESM2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190308 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2/historic... |
4 | CMIP6 | SNU | SAM0-UNICON | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190323 | s3://esgf-world/CMIP6/CMIP/SNU/SAM0-UNICON/his... |
0 | CMIP6 | MPI-M | MPI-ESM1-2-HR | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20190710 | s3://esgf-world/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR... |
1 | CMIP6 | NCAR | CESM2-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-FV2/hist... |
2 | CMIP6 | NCAR | CESM2-WACCM-FV2 | historical | NaN | NaN | Ofx | r1i1p1f1 | gn | volcello | NaN | v20191120 | s3://esgf-world/CMIP6/CMIP/NCAR/CESM2-WACCM-FV... |
cat_T_gn.df[cat_T_gn.df['model']=='CanESM5'].groupby(['version']).nunique() #one distinct version only, latest one.
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
version | ||||||||||||
v20190306 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 17 |
v20190429 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 17 |
cat_T_gn_latest[cat_T_gn_latest['model']=='CanESM5'].groupby(['version']).nunique() #one distinct version only, latest one.
project | institute | model | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
version | ||||||||||||
v20190429 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 17 |
cat_T_gn.df.groupby(['model']).nunique()
project | institute | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
model | ||||||||||||
CESM2 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 |
CESM2-FV2 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 4 | 1 | 4 |
CESM2-WACCM-FV2 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 4 | 1 | 4 |
CanESM5 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 2 | 34 |
IPSL-CM6A-LR | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 2 | 1 | 2 |
MIROC6 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 1 | 17 |
MPI-ESM-1-2-HAM | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 9 | 1 | 9 |
MPI-ESM1-2-HR | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 33 | 1 | 33 |
MPI-ESM1-2-LR | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 9 | 1 | 9 |
MRI-ESM2-0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 3 | 1 | 3 |
SAM0-UNICON | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 17 | 1 | 17 |
cat_A_gn.df.groupby(['model']).nunique()
project | institute | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
model | ||||||||||||
CESM2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CESM2-FV2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CESM2-WACCM-FV2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CanESM5 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
IPSL-CM6A-LR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MIROC6 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM-1-2-HAM | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM1-2-HR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM1-2-LR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MRI-ESM2-0 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
SAM0-UNICON | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
cat_A_gn_latest.groupby(['model']).nunique() #.nunique()
project | institute | experiment_id | frequency | modeling_realm | mip_table | ensemble_member | grid_label | variable | temporal subset | version | path | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
model | ||||||||||||
CESM2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CESM2-FV2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CESM2-WACCM-FV2 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
CanESM5 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
IPSL-CM6A-LR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MIROC6 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM-1-2-HAM | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM1-2-HR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MPI-ESM1-2-LR | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
MRI-ESM2-0 | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |
SAM0-UNICON | 1 | 1 | 1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 1 | 1 |