%matplotlib inline
from dask.distributed import Client, progress, LocalCluster
from dask_kubernetes import KubeCluster
import xarray as xr
import gcsfs
cluster = KubeCluster()
cluster.scale(10);
cluster
Failed to display Jupyter Widget of type VBox
.
If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean that the widgets JavaScript is still loading. If this message persists, it likely means that the widgets JavaScript library is either not installed or not enabled. See the Jupyter Widgets Documentation for setup instructions.
If you're reading this message in another frontend (for example, a static rendering on GitHub or NBViewer), it may mean that your frontend doesn't currently support widgets.
client = Client(cluster)
client
Client
|
Cluster
|
fs = gcsfs.GCSFileSystem(project='pangeo-181919')
bucket = 'pangeo-data/rsignell/nwm/test_week5c'
gcsmap = gcsfs.mapping.GCSMap(bucket, gcs=fs, check=False, create=False)
ds = xr.open_zarr(gcsmap)
ds
<xarray.Dataset> Dimensions: (time: 168, x: 4608, y: 3840) Coordinates: * time (time) datetime64[ns] 2018-04-01T01:00:00 2018-04-01T02:00:00 ... * x (x) float64 -2.304e+06 -2.303e+06 -2.302e+06 -2.301e+06 ... * y (y) float64 -1.92e+06 -1.919e+06 -1.918e+06 -1.917e+06 ... Data variables: LWDOWN (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> PSFC (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> Q2D (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> RAINRATE (time, y, x) float32 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> SWDOWN (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> T2D (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> U2D (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> V2D (time, y, x) float64 dask.array<shape=(168, 3840, 4608), chunksize=(168, 384, 288)> Attributes: model_initialization_time: 2018-04-01_00:00:00 model_output_valid_time: 2018-04-01_01:00:00
var='RAINRATE'
print('start:',ds[var][0].time.values)
print(' stop:',ds[var][-1].time.values)
start: 2018-04-01T01:00:00.000000000 stop: 2018-04-08T00:00:00.000000000
ds.nbytes/1.e9
178.362846528
ds[var].nbytes/1.e9
11.89085184
%%time
mean_temp = ds[var][:,:,:].mean(dim='time').compute()
CPU times: user 2.66 s, sys: 469 ms, total: 3.12 s Wall time: 11.1 s
isub = 4
mean_temp[::isub,::isub].plot.imshow(figsize=(8,6));
%%time
ds1d = ds[var][:,2000,2000]
ds1d.plot()
CPU times: user 100 ms, sys: 1 ms, total: 101 ms Wall time: 786 ms