%reload_ext autoreload
%autoreload 2
import os
import sys
import shutil
import pprint
import pymongo
import json
import pandas as pd
import climatools.lblnew.setup_bestfit as setup_bestfit
from climatools.lblnew.bestfit_params import *
from climatools.lblnew.pipeline import *
import climatools.lblnew.runrecord_bestfit as runrecord
from climatools.atm import *
from climatools.cliradlw.utils import *
MOLECULE = 'h2o'
atm = AtmComposition.cliradlw_nongreys()
BANDMOLS = [(mapband_new2old()[band], 'h2o') for band, absorbers in atm.gasinbands.items() if 'h2o' in absorbers]
ATMPROS = [('saw_trpwlayer', 257), ('trp_sawwlayer', 300)]
PARAMS = []
for band, absorber in BANDMOLS[:1]:
for atmpro, tsfc in ATMPROS:
param = kdist_params(molecule=absorber, band=band)
param['commitnumber'] = '9dc25c5'
param['atmpro'], param['tsfc'] = atmpro, tsfc
param['option_compute_ktable'] = 0
param['option_compute_btable'] = 0
PARAMS.append(param)
pipeline_fortran_bsub(params=PARAMS[1:], setup=setup_bestfit)
/chia_cluster/home/jackyu/radiation/crd/LW/examples/separate_g_groups/study__lblnew_g1_threshold/h2o/conc_None/band01_wn_20_340/nv_1000/dv_0.001/ng_12/g_ascending_k_descending/refPTs_P_1_T_250__P_30_T_250__P_300_T_250/ng_refs_3__3__6/ng_adju_0__0__0/getabsth_auto__auto__auto/absth_dlogN_uniform__dlogN_uniform__dlogN_uniform/klin_none/atmpro_trp_sawwlayer/wgt_k_1/wgt_0.2_0.2_0.2__0.2_0.2_0.2__0.2_0.2_0.65_0.65_0.65_0.65/wgt_flux_1/w_diffuse_1.66_1.66_1.66__1.66_1.66_1.66__1.66_1.66_1.5_1.5_1.5_1.5/option_compute_ktable_0/option_compute_btable_0/crd_9dc25c5
!bjobs
JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME 441568 jackyu RUN serial chialn00 chiacn099 *new_h2o_1 Jul 8 12:34 441567 jackyu RUN serial chialn00 chiacn105 *new_h2o_1 Jul 8 12:33
client = pymongo.MongoClient('localhost', 27017)
def make_doc(param=None):
'''
Makes a Mongodb document for an lblnew-bestfit run.
Parameters
----------
param: dict
lblnew-bestfit input parameters.
doc: dict
lblnew-bestfit input (param) and output (flux and cooling rate)
'''
dir_fortran = get_dir_case(param=param, setup=setup_bestfit)
fnames = [n for n in os.listdir(dir_fortran) if n in
('output_wfluxg.dat', 'output_wcoolrg.dat',
'output_fluxg.dat', 'output_coolrg.dat',
'output_flux.dat', 'output_coolr.dat',
'abscom.dat', 'dgdgs.dat',
'kg_lin.dat', 'kg_nonlin.dat')]
s_outputs = []
for fname in fnames:
with open(os.path.join(dir_fortran, fname),
mode='r', encoding='utf-8') as f:
s = f.read()
s_outputs.append(s)
doc = {}
doc['param'] = param
for fname, s in zip(fnames, s_outputs):
doc[fname.split('.')[0]] = s
return doc
def make_query(param=None):
q = {}
for n, v in param.items():
if n == 'molecule' and type(v) == dict:
for mol, conc in v.items():
q[f'param.molecule.{mol}'] = conc
else:
q[f'param.{n}'] = v
return q
docs = (make_doc(param=param) for param in PARAMS)
client.lblnew.bestfit_lw.count()
151
client.lblnew.bestfit_lw.insert_many(docs)
<pymongo.results.InsertManyResult at 0x2b06c4843bd0>
#for param in params:
# client.lblnew.bestfit_lw.delete_one(make_query(param))
from climatools.atm.absorbers import *
from climatools.cliradlw.utils import *
ATMPROS = [('mls', 294), ('saw', 257), ('trp', 300)]
def run_exists(param=None):
q = make_query(param)
cnt = client.lblnew.bestfit_lw.count(q)
return True if cnt else False
def notebook_exists(param=None):
d = get_analysis_dir(param=param, setup=setup_bestfit)
try:
return 'results.ipynb' in os.listdir(d)
except FileNotFoundError:
return False
def get_params(band=1):
bandmols = [(b, mol) for b, molconc in nongreys_byband().items() if b == band
for mol in molconc.keys()]
bandmols = [(mapband_new2old()[band], mol) for band, mol in bandmols]
params = []
for band, mol in bandmols:
p = kdist_params(molecule=mol, band=band)
for k in ('option_compute_ktable', 'option_compute_btable', 'commitnumber'):
if k in p: del p[k]
for atmpro, tsfc in ATMPROS:
p['atmpro'], p['tsfc'] = atmpro, tsfc
q = make_query(p)
r = client.lblnew.bestfit_lw.find_one(q, projection=['param'])
if not r:
print('Following lblnew run not available:')
pprint.pprint(p)
continue
params.append(r.get('param'))
params = [p for p in params if run_exists(p)]
return params
# This deletes some excel files in the analysis directories for params.
#for p in params:
# print('band=', p['band'], 'molecule=', p['molecule'], 'atmpro=', p['atmpro'])
# path = get_analysis_dir(p, setup=setup_bestfit)
# print(os.listdir(path))
# xfiles = [f for f in os.listdir(path) if f in ('lblnew-bestfit_flux.xlsx', 'lblnew-bestfit_cool.xlsx')]
# xfiles = [os.path.join(path, f) for f in xfiles]
# [os.remove(f) for f in xfiles]
# print(os.listdir(path))
# print()
params = get_params(band=11)
gprocs = pipeline_ipynb2git(params=PARAMS, setup=setup_bestfit)
h2o 1 9dc25c5 h2o 2 9dc25c5 co2 3a 9dc25c5 co2 3b 9dc25c5 h2o 1 9dc25c5 h2o 2 9dc25c5 co2 3a 9dc25c5 co2 3b 9dc25c5
for p in PARAMS:
path = get_analysis_dir(param=p, setup=setup_bestfit)
%ls
!git add {path}/lblnew-bestfit_fluxcool.xlsx
lblnew-bestfit_fluxcool.xlsx param.py results.ipynb lblnew-bestfit_fluxcool.xlsx param.py results.ipynb lblnew-bestfit_fluxcool.xlsx param.py results.ipynb lblnew-bestfit_fluxcool.xlsx param.py results.ipynb
for param in PARAMS:
print(param['molecule'], 'band', param['band'], param['atmpro'])
print(nbviewer_url(param=param, setup=setup_bestfit))
print()
h2o band 1 mls http://nbviewer.jupyter.org/github/qap/analysis_-_new_kdist_param/blob/master/lblnew/h2o/conc_None/band01_wn_20_340/nv_10000/dv_0.0001/ng_12/g_ascending_k_descending/refPTs_P_1_T_250__P_30_T_250__P_300_T_250/ng_refs_3__3__6/ng_adju_0__0__0/getabsth_auto__auto__auto/absth_dlogN_uniform__dlogN_uniform__dlogN_uniform/klin_none/atmpro_mls/wgt_k_1/wgt_0.2_0.2_0.2__0.2_0.2_0.2__0.2_0.2_0.65_0.65_0.65_0.65/wgt_flux_1/w_diffuse_1.66_1.66_1.66__1.66_1.66_1.66__1.66_1.66_1.5_1.5_1.5_1.5/option_compute_ktable_0/option_compute_btable_0/crd_9dc25c5/results.ipynb h2o band 2 mls http://nbviewer.jupyter.org/github/qap/analysis_-_new_kdist_param/blob/master/lblnew/h2o/conc_None/band02_wn_340_540/nv_10000/dv_0.0001/ng_11/g_ascending_k_descending/refPTs_P_1_T_250__P_10_T_250__P_500_T_250/ng_refs_2__3__6/ng_adju_-2__-2__0/getabsth_auto__auto__auto/absth_dlogN_uniform__dlogN_uniform__dlogN_uniform/klin_none/atmpro_mls/wgt_k_1/wgt_0.6_0.6__0.6_0.6_0.6__0.6_0.6_0.6_0.6_0.6_0.6/wgt_flux_1/w_diffuse_1.66_1.66__1.8_1.8_1.8__1.8_1.66_1.45_1.45_1.45_1.45/option_compute_ktable_0/option_compute_btable_0/crd_9dc25c5/results.ipynb co2 band 3a mls http://nbviewer.jupyter.org/github/qap/analysis_-_new_kdist_param/blob/master/lblnew/co2/conc_0.0004/band03a_wn_540_620/nv_10000/dv_0.0001/ng_9/g_ascending_k_descending/refPTs_P_1_T_250__P_10_T_250__P_500_T_250/ng_refs_3__2__4/ng_adju_0__0__0/getabsth_auto__auto__auto/absth_dlogN_uniform__dlogN_uniform__dlogN_uniform/klin_6.375563e-24/atmpro_mls/wgt_k_1/wgt_0.7_0.3_0.7__0.7_0.6__0.4_0.5_0.8_0.95/wgt_flux_1/w_diffuse_1.6_1.6_1.7__1.75_1.75__1.55_1.55_1.6_1.85/option_compute_ktable_0/option_compute_btable_0/crd_9dc25c5/results.ipynb co2 band 3b mls http://nbviewer.jupyter.org/github/qap/analysis_-_new_kdist_param/blob/master/lblnew/co2/conc_0.0004/band03b_wn_620_720/nv_10000/dv_0.0001/ng_9/g_ascending_k_descending/refPTs_P_1_T_250__P_10_T_250/ng_refs_5__4/ng_adju_0__0/getabsth_auto__auto/absth_dlogN_uniform__dlogN_uniform/klin_none/atmpro_mls/wgt_k_1/wgt_0_0.5_0.4_0.6_0.65__0.7_0.7_0.55_0.6/wgt_flux_1/w_diffuse_1.66_1.66_1.66_1.66_1.66__1.66_1.66_1.66_1.66/option_compute_ktable_0/option_compute_btable_0/crd_9dc25c5/results.ipynb