import pandas as pd
import numpy as np
from tqdm import tqdm
#sra_dump_pickle_dir='/cellar/users/btsui/Data/SRA/DUMP/sra_dump.pickle'
uploadDir='/cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/'
/cellar/users/btsui/anaconda3/envs/deep_nlp_cpu/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88 return f(*args, **kwds)
!ls /cellar/users/btsui/Data/SRA/DUMP/
FULL_SRA_meta file_meta.txt NCBI_SRA_Metadata_Full_20180702.tar.gz merged_variant_aligning_statistics.tsv SRA_Accessions.tab meta.txt SRA_Run_Members.tab sra_dump.csv.gz allSRS.pickle.gz sra_dump.fastqc.bowtie_algn.pickle allSRX.pickle.gz sra_dump.pickle
#pd.read_pickle('/cellar/users/btsui/Data/SRA/DUMP/sra_dump.fastqc.bowtie_algn.pickle')
!ls -laht /cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/ | head
total 54G -rw-r--r-- 1 btsui users 2.3G Aug 9 21:15 5100000.pickle.gz -rw-r--r-- 1 btsui users 1.4G Aug 9 20:43 5400000.pickle.gz -rw-r--r-- 1 btsui users 1.3G Aug 9 20:34 5300000.pickle.gz -rw-r--r-- 1 btsui users 1.2G Aug 9 20:28 5000000.pickle.gz -rw-r--r-- 1 btsui users 1.2G Aug 9 20:26 5200000.pickle.gz -rw-r--r-- 1 btsui users 1.1G Aug 9 20:25 5800000.pickle.gz -rw-r--r-- 1 btsui users 1.4G Aug 9 20:21 5900000.pickle.gz -rw-r--r-- 1 btsui users 770M Aug 9 20:20 6200000.pickle.gz -rw-r--r-- 1 btsui users 746M Aug 9 20:11 5600000.pickle.gz
import os
#syn15624400
synaspeFolderId='syn15624400'
inFnames=os.listdir(uploadDir)
#pd.read_pickle('/cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/')
#4 #python 3.5
!synapse login
Welcome, Brian Yik Tak Tsui! Logged in as: btsui (3329338)
cmdFmt="synapse add {fDir} --parentId={synaspeFolderId} --annotations 'check https://github.com/brianyiktaktsui/Skymap for more info'"
for inFname in tqdm(os.listdir(uploadDir)):
cmd=cmdFmt.format(fDir=uploadDir+inFname,synaspeFolderId=synaspeFolderId)
out= (os.system(cmd))
3%|▎ | 2/61 [02:06<1:02:11, 63.25s/it]
#'synapse login -u me@example.com -p secret --rememberMe'
outDir
'/cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/0.pickle'
!ls /cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/
0.pickle.gz
#!ls /cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/
#!rm /cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/*
#!ls -lah /cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/0.pickle
import h5py
#!ls /tmp/
f = h5py.File("/tmp/100000.hdf5",)
#tmp=f['/100000/']
f.keys()
KeysView(<HDF5 file "100000.hdf5" (mode r+)>)
list(tmp['/chunk/'])
['axis0', 'axis1_label0', 'axis1_label1', 'axis1_label2', 'axis1_label3', 'axis1_label4', 'axis1_level0', 'axis1_level1', 'axis1_level2', 'axis1_level3', 'axis1_level4', 'block0_items', 'block0_values']
tmpDf=pd.read_pickle('/cellar/users/btsui/Data/merged/snp/hg38/mergedBySRR/5600000.pickle.gz')
tmpDf.to_pickle('/tmp/test.pickle',protocol=3)
import pandas as pd
pd.read_pickle('/tmp/test.pickle')
#0.18.1