import fsspec
import fsspec.implementations.reference
import zarr
import xarray as xr
from pathlib import Path
from rechunker import rechunk
import rechunker
rechunker.__version__
'0.5.0'
import zarr
zarr.__version__
'2.11.1'
#client.close(); cluster.close()
Use a custom helper function ebd.start_dask_cluster
to set options on this cluster. We don't have to use this helper, it just cuts down on lines of code in notebooks.
from dask.distributed import LocalCluster, Client
cluster = LocalCluster(n_workers=40, threads_per_worker=1)
client = Client(cluster)
client
2023-04-17 12:00:36,249 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-94yum0m1', purging 2023-04-17 12:00:36,279 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-gqk8ud6j', purging 2023-04-17 12:00:36,293 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-so9yygnc', purging 2023-04-17 12:00:36,316 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-lja04i9q', purging 2023-04-17 12:00:36,329 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-s9ep7ejd', purging 2023-04-17 12:00:36,364 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-r1dytxx2', purging 2023-04-17 12:00:36,389 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-p5y46miz', purging 2023-04-17 12:00:36,417 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-khh32umt', purging 2023-04-17 12:00:36,431 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-74wiprq7', purging 2023-04-17 12:00:36,459 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-ao4nz4bb', purging 2023-04-17 12:00:36,488 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-613ceoxb', purging 2023-04-17 12:00:36,499 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-bzubg9u5', purging 2023-04-17 12:00:36,519 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-un9o3oup', purging 2023-04-17 12:00:36,544 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-m7qz8kb7', purging 2023-04-17 12:00:36,566 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-1urleedt', purging 2023-04-17 12:00:36,609 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-vzjbga3u', purging 2023-04-17 12:00:36,622 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-lebenvpy', purging 2023-04-17 12:00:36,641 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-n9jah_b1', purging 2023-04-17 12:00:36,652 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-z2d3kjz1', purging 2023-04-17 12:00:36,669 - distributed.diskutils - INFO - Found stale lock file and directory '/home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-s1a3_up4', purging
Client-608f045d-dd41-11ed-ac2b-00010100060b
Connection method: Cluster object | Cluster type: distributed.LocalCluster |
Dashboard: /proxy/8787/status |
6e64fa72
Dashboard: /proxy/8787/status | Workers: 40 |
Total threads: 40 | Total memory: 187.58 GiB |
Status: running | Using processes: True |
Scheduler-1f1530bb-beb0-4ac6-abc8-d7892dc1af2f
Comm: tcp://127.0.0.1:39595 | Workers: 40 |
Dashboard: /proxy/8787/status | Total threads: 40 |
Started: Just now | Total memory: 187.58 GiB |
Comm: tcp://127.0.0.1:39429 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:37701 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-6breli8g |
Comm: tcp://127.0.0.1:38977 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:41887 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-khnu98bg |
Comm: tcp://127.0.0.1:45311 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39641 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-dgpzjx8u |
Comm: tcp://127.0.0.1:34367 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:46523 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-r9hv5il6 |
Comm: tcp://127.0.0.1:38191 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:40901 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-kieip7os |
Comm: tcp://127.0.0.1:35595 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:33303 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-kn4x7odt |
Comm: tcp://127.0.0.1:44037 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:36131 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-kynthw4w |
Comm: tcp://127.0.0.1:40149 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39499 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-6gw_ymaw |
Comm: tcp://127.0.0.1:35861 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39685 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-imjk7r27 |
Comm: tcp://127.0.0.1:36575 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:32969 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-sd3f_yp4 |
Comm: tcp://127.0.0.1:40751 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39051 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-mdvb9tfs |
Comm: tcp://127.0.0.1:39269 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39721 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-yw9vopdc |
Comm: tcp://127.0.0.1:38671 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:41627 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-_jy8v7ka |
Comm: tcp://127.0.0.1:39969 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:42013 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-9lqe3o95 |
Comm: tcp://127.0.0.1:46021 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:37477 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-_rywm7k7 |
Comm: tcp://127.0.0.1:36479 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:36347 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-wlft9hqc |
Comm: tcp://127.0.0.1:35425 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:38377 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-5c92uqcj |
Comm: tcp://127.0.0.1:36183 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:46389 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-y5s3h2hf |
Comm: tcp://127.0.0.1:34695 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:38971 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-mi_12icc |
Comm: tcp://127.0.0.1:46651 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39469 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-ag4udpw6 |
Comm: tcp://127.0.0.1:33705 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:39139 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-ga4uudc6 |
Comm: tcp://127.0.0.1:38237 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:45863 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-8_dbflaa |
Comm: tcp://127.0.0.1:45951 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:37985 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-vm6p22re |
Comm: tcp://127.0.0.1:34231 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:46737 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-k0cy4tc1 |
Comm: tcp://127.0.0.1:38309 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:37691 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-n_v_9hau |
Comm: tcp://127.0.0.1:44057 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:35579 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-pq_7wuvb |
Comm: tcp://127.0.0.1:41263 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:43327 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-0_qbos9b |
Comm: tcp://127.0.0.1:46697 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:46797 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-81uv5evv |
Comm: tcp://127.0.0.1:37171 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:35565 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-s5_0j5ql |
Comm: tcp://127.0.0.1:40643 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:35781 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-xlb8swuo |
Comm: tcp://127.0.0.1:33873 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:33755 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-w1yngzfx |
Comm: tcp://127.0.0.1:40721 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:40249 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-q16aeamx |
Comm: tcp://127.0.0.1:35343 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:34345 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-idacy1kx |
Comm: tcp://127.0.0.1:39471 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:41079 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-5uxfhfnr |
Comm: tcp://127.0.0.1:38827 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:37301 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-7ovms1sn |
Comm: tcp://127.0.0.1:43817 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:36443 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-hpl3219f |
Comm: tcp://127.0.0.1:45747 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:38447 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-xl7gh6xw |
Comm: tcp://127.0.0.1:35157 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:38739 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-zp3yrjzg |
Comm: tcp://127.0.0.1:42667 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:41109 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-bgfbavty |
Comm: tcp://127.0.0.1:35971 | Total threads: 1 |
Dashboard: /proxy/8787/status | Memory: 4.69 GiB |
Nanny: tcp://127.0.0.1:40925 | |
Local directory: /home/rsignell/EarthMap/Projects/repos/esip-qhub-notebooks/sudhir/local/dask-worker-space/worker-i66uw6q3 |
combined_json = '/caldera/hytest_scratch/scratch/rsignell/sudhir.json'
%%time
fs = fsspec.filesystem("reference", fo=combined_json, ref_storage_args={'skip_instance_cache':True})
m = fs.get_mapper("")
ds = xr.open_dataset(m, engine="zarr", mask_and_scale=False, decode_cf=False, decode_times=False,
backend_kwargs={'consolidated':False}, chunks={})
CPU times: user 652 ms, sys: 280 ms, total: 932 ms Wall time: 3.32 s
/home/rsignell/miniconda3/envs/pangeo/lib/python3.9/site-packages/xarray/backends/plugins.py:71: RuntimeWarning: Engine 'rasterio' loading failed: libLerc.so.4: cannot open shared object file: No such file or directory warnings.warn(f"Engine {name!r} loading failed:\n{ex}", RuntimeWarning)
ds
<xarray.Dataset> Dimensions: (time: 387, latitude: 4201, longitude: 8401) Coordinates: * latitude (latitude) float64 20.0 20.01 20.02 ... 54.99 55.0 * longitude (longitude) float64 -130.0 -130.0 ... -60.01 -60.0 * time (time) float64 9.494e+08 9.494e+08 ... 9.508e+08 Data variables: APCP_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> DLWRF_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> DSWRF_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> PRES_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> SPFH_2maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> TMP_2maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> UGRD_10maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> VGRD_10maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray>
ds = ds.isel(time=slice(0,144))
ds
<xarray.Dataset> Dimensions: (time: 144, latitude: 4201, longitude: 8401) Coordinates: * latitude (latitude) float64 20.0 20.01 20.02 ... 54.99 55.0 * longitude (longitude) float64 -130.0 -130.0 ... -60.01 -60.0 * time (time) float64 9.494e+08 9.494e+08 ... 9.499e+08 Data variables: APCP_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> DLWRF_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> DSWRF_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> PRES_surface (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> SPFH_2maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> TMP_2maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> UGRD_10maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray> VGRD_10maboveground (time, latitude, longitude) int16 dask.array<chunksize=(1, 1024, 2048), meta=np.ndarray>
ds.nbytes/1e9
81.314254672
fs_write = fsspec.filesystem('file')
temp_name = '/caldera/hytest_scratch/scratch/rsignell/AORC/rechunk/test01.tmp'
target_name = '/caldera/hytest_scratch/scratch/rsignell/AORC/rechunk/test01.zarr'
temp_store = fs_write.get_mapper(temp_name)
target_store = fs_write.get_mapper(target_name)
try:
fs_write.rm(temp_name, recursive=True)
except:
pass
try:
fs_write.rm(target_name, recursive=True)
except:
pass
nt = len(ds.time)
nt = 365*24*30
a = int(nt)/144
a
1825.0
b = (len(ds.longitude) * len(ds.latitude))/((2*200)*(2*200))
b
220.57875625
a/b
8.273688867533453
2.8/4
0.7
#client.close()
#from dask.distributed import Client
#client = Client(threads_per_worker=1)
#client.amm.start()
import zarr.storage
from numcodecs import Zstd
zarr.storage.default_compressor = Zstd(level=9)
(200*200*144)*2/1e6
11.52
190/40*.7
3.3249999999999997
chunks={'latitude':256, 'longitude':256, 'time':144}
verbose=True
mem = '2.8GB'
group_chunks = {}
# newer tuple version that also takes into account when specified chunks are larger than the array
for var in ds.variables:
# pick appropriate chunks from above, and default to full length chunks for dimensions that are not in `chunks` above.
group_chunks[var] = []
for di in ds[var].dims:
if di in chunks.keys():
if chunks[di] > len(ds[di]):
group_chunks[var].append(len(ds[di]))
else:
group_chunks[var].append(chunks[di])
else:
group_chunks[var].append(len(ds[di]))
group_chunks[var] = tuple(group_chunks[var])
if verbose:
print(f"Rechunking to: {group_chunks}")
print(f"mem:{mem}")
Rechunking to: {'APCP_surface': (144, 256, 256), 'DLWRF_surface': (144, 256, 256), 'DSWRF_surface': (144, 256, 256), 'PRES_surface': (144, 256, 256), 'SPFH_2maboveground': (144, 256, 256), 'TMP_2maboveground': (144, 256, 256), 'UGRD_10maboveground': (144, 256, 256), 'VGRD_10maboveground': (144, 256, 256), 'latitude': (256,), 'longitude': (256,), 'time': (144,)} mem:2.8GB
from dask.distributed import performance_report
ds.time.attrs['standard_name'] = 'time'
del ds.time.attrs['missing_value']
ds.longitude.attrs['standard_name'] = 'longitude'
del ds.longitude.attrs['missing_value']
ds.latitude.attrs['standard_name'] = 'latitude'
del ds.latitude.attrs['missing_value']
%%time
rechunked = rechunker.rechunk(ds, target_chunks=group_chunks, max_mem=mem,
target_store=target_store, temp_store=temp_store)
CPU times: user 1.07 s, sys: 380 ms, total: 1.45 s Wall time: 5.44 s
%%time
with performance_report(filename="dask-report-whole.html"):
rechunked.execute(retries=10)
CPU times: user 1min 45s, sys: 14.8 s, total: 2min Wall time: 3min 35s
zarr.convenience.consolidate_metadata(target_store)
<zarr.hierarchy.Group '/'>
ds2 = xr.open_dataset(target_store, engine='zarr', chunks={})
ds2.nbytes/1e9 #uncompressed data size in GB
ds2
ds2.APCP_surface
import hvplot.xarray
%%time
da = ds.APCP_surface.sel(longitude=-115.18, latitude=46.65, method='nearest').load()
da.hvplot(x='time', grid=True)
cluster.close()
client.close()
client