Use HSDS (data chunks in S3 objects) to access Model Output

In [1]:
%matplotlib inline
import h5pyd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
In [2]:
h5pyd.getServerInfo()
Out[2]:
{'about': 'HSDS is a webservice for HSDS data',
 'endpoint': 'http://149.165.157.109:5101',
 'greeting': 'Welcome to HSDS!',
 'hsds_version': '0.1',
 'name': 'HSDS on JetStream',
 'password': '***************',
 'start_time': 1512254660,
 'state': 'READY',
 'username': 'rsignell'}
In [3]:
# Open the wind data "file"
# server endpoint, username, password is found via a config file
#f = h5pyd.File("/home/rsignell/Sandy_ocean_his_nc4.nc", 'r')
#f = h5pyd.File("/home/rsignell/sandy.nc", 'r')
f = h5pyd.File("/home/rsignell/sandy2.nc", 'r')
In [4]:
[print(k) for k in f.attrs.items()];
('CPP_options', b'SANDY, ANA_BSFLUX, ANA_BTFLUX, ANA_FSOBC, ANA_M2OBC, ANA_SRFLUX, ANA_SSFLUX, ASSUMED_SHAPE, ATM_PRESS, ATM2OCN_FLUXE AVERAGES, COARE_TAYLOR_YELLAND, CURVGRID, DJ_GRADPS, DOUBLE_PRECISION, GLS_MIXING, KANTHA_CLAYSON, MASKING, MCT_LIB, MIX_GEO_TS, MIX_S_UV, MPI, NESTING, NONLINEAR, NONLIN_EOS, N2S2_HORAVG, !ONE_WAY, POWER_LAW, PROFILE, K_GSCHEME, RAMP_TIDES, RI_SPLINES, !RST_SINGLE, SALINITY, SOLAR_SOURCE, SOLVE3D, SPLINES_VDIFF, SPLINES_VVISC, SSH_TIDES, SWAN_COUPLING, TS_U3HADVECTION, TS_C4VADVECTION, TS_DIF2, UV_ADV, UV_COR, UV_U3HADVECTION, UV_C4VADVECTION, UV_LOGDRAG, UV_TIDES, UV_VIS2, VAR_RHO_2D, WAVES_OCEAN, WRF_COUPLING')
('Conventions', b'CF-1.4, SGRID-0.3')
('NLM_LBC', b'\nEDGE:  WEST   SOUTH  EAST   NORTH  \nzeta:  Clo    Cha    Cha    Cha    \nubar:  Clo    Fla    Fla    Fla    \nvbar:  Clo    Fla    Fla    Fla    \nu:     Clo    RadNud RadNud RadNud \nv:     Clo    RadNud RadNud RadNud \ntemp:  Gra    RadNud RadNud Gra    \nsalt:  Gra    RadNud RadNud Gra    \ntke:   Clo    Gra    Gra    Gra')
('_NCProperties', b'version=1|netcdflibversion=4.5.0|hdf5libversion=1.10.1')
('_nc3_strict', 1)
('ana_file', b'ROMS/Functionals/ana_btflux.h, ROMS/Functionals/ana_fsobc.h, ROMS/Functionals/ana_m2obc.h, ROMS/Functionals/ana_stflux.h')
('avg_file', b'Sandy_ocean_avg.nc')
('bry_file_01', b'Projects/Sandy/Sandy_bdy.nc')
('code_dir', b'/peach/data2/tkalra/COAWST_regress_peach/COAWST_src2')
('compiler_command', b'/share/apps/openmpi/bin/mpif90')
('compiler_flags', b' -fastsse -Mipa=fast -tp k8-64 -Mfree')
('compiler_system', b'pgi')
('cpu', b'x86_64')
('file', b'Sandy_ocean_his.nc')
('format', b'netCDF-3 64bit offset file')
('frc_file_01', b'Projects/Sandy/roms_namnarr_Sandy2012.nc')
('frc_file_02', b'Projects/Sandy/tide_forc_Sandy.nc')
('grd_file', b'Projects/Sandy/Sandy_roms_grid.nc')
('header_dir', b'/peach/data2/tkalra/COAWST_regress_peach/COAWST_src2/Projects/Sandy')
('header_file', b'sandy.h')
('his_file', b'Sandy_ocean_his.nc')
('history', b'ROMS/TOMS, Version 3.7, Thursday - February 23, 2017 -  7:31:24 AM')
('ini_file', b'Projects/Sandy/Sandy_ini.nc')
('os', b'Linux')
('rst_file', b'Sandy_ocean_rst.nc')
('svn_rev', b'1138M')
('svn_url', b'https:://myroms.org/svn/src')
('tiling', b'004x002')
('title', b'Hurricane Sandy')
('type', b'ROMS/TOMS history file')
('var_info', b'ROMS/External/varinfo.dat')
In [5]:
list(f)  # list the datasets in the file
Out[5]:
['AKv',
 'Akk_bak',
 'Akp_bak',
 'Akt_bak',
 'Akv_bak',
 'Charnok_alpha',
 'CrgBan_cw',
 'Cs_r',
 'Cs_w',
 'Dwave',
 'EminusP',
 'FSobc_in',
 'FSobc_out',
 'Falpha',
 'Fbeta',
 'Fgamma',
 'Hwave',
 'Lm2CLM',
 'Lm3CLM',
 'LnudgeM2CLM',
 'LnudgeM3CLM',
 'LnudgeTCLM',
 'LsshCLM',
 'LtracerCLM',
 'LtracerSponge',
 'LtracerSrc',
 'LuvSponge',
 'LuvSrc',
 'LwSrc',
 'Lwave',
 'Lwavep',
 'M2nudg',
 'M2obc_in',
 'M2obc_out',
 'M3nudg',
 'M3obc_in',
 'M3obc_out',
 'N',
 'Pair',
 'Pwave_bot',
 'Pwave_top',
 'Tcline',
 'Tnudg',
 'Tnudg_SSS',
 'Tobc_in',
 'Tobc_out',
 'Uwave_rms',
 'Uwind',
 'Uwind_eastward',
 'Vstretching',
 'Vtransform',
 'Vwind',
 'Vwind_northward',
 'Znudg',
 'Zob',
 'Zos',
 'Zos_hsig_alpha',
 'angle',
 'boundary',
 'bustr',
 'bvstr',
 'dstart',
 'dt',
 'dtfast',
 'el',
 'eta_psi',
 'eta_rho',
 'eta_u',
 'eta_v',
 'evaporation',
 'f',
 'gamma2',
 'gls',
 'gls_Kmin',
 'gls_Pmin',
 'gls_c1',
 'gls_c2',
 'gls_c3m',
 'gls_c3p',
 'gls_cmu0',
 'gls_m',
 'gls_n',
 'gls_p',
 'gls_sigk',
 'gls_sigp',
 'grid',
 'h',
 'hc',
 'lat_psi',
 'lat_rho',
 'lat_u',
 'lat_v',
 'latent',
 'lon_psi',
 'lon_rho',
 'lon_u',
 'lon_v',
 'lwrad',
 'mask_psi',
 'mask_rho',
 'mask_u',
 'mask_v',
 'nAVG',
 'nHIS',
 'nRST',
 'ndefAVG',
 'ndefHIS',
 'ndtfast',
 'nl_tnu2',
 'nl_visc2',
 'ntimes',
 'ntsAVG',
 'ocean_time',
 'omega',
 'pm',
 'pn',
 'rain',
 'rdrg',
 'rdrg2',
 'rho0',
 's_rho',
 's_w',
 'salt',
 'sensible',
 'shflux',
 'spherical',
 'ssflux',
 'sustr',
 'svstr',
 'swrad',
 'sz_alpha',
 'temp',
 'theta_b',
 'theta_s',
 'tke',
 'tracer',
 'u',
 'ubar',
 'v',
 'vbar',
 'w',
 'xi_psi',
 'xi_rho',
 'xi_u',
 'xi_v',
 'xl',
 'zeta']
In [6]:
dset = f['Hwave']
In [7]:
dset.id.id  # if this is an int, then you are using h5py!
Out[7]:
'd-66a40690-d860-11e7-a439-0242ac11000e'
In [8]:
dset.shape
Out[8]:
(97, 64, 84)
In [9]:
plt.plot(dset[:,30,40])
Out[9]:
[<matplotlib.lines.Line2D at 0x7fe59403b5c0>]
In [10]:
dset.chunks
Out[10]:
[64, 64, 84]
In [11]:
%time data =  dset[55,:,:]
CPU times: user 8 ms, sys: 0 ns, total: 8 ms
Wall time: 15.7 ms
In [12]:
data = np.ma.masked_equal(data, dset.fillvalue)
In [13]:
plt.imshow(data, origin="lower")
Out[13]:
<matplotlib.image.AxesImage at 0x7fe588209eb8>
In [14]:
dset.fillvalue
Out[14]:
0.0
In [15]:
data = np.ma.masked_equal(data, 1.e37)
In [16]:
plt.imshow(data, origin="lower")
Out[16]:
<matplotlib.image.AxesImage at 0x7fe5881876d8>