import pandas as pd
import os
import boto3
import botocore
import paramiko
mergedMatricesDir='/nrnb/users/btsui/Data/all_seq/rnaseq_merged/'
inFnameS=pd.Series(os.listdir(mergedMatricesDir))
include_m1=(inFnameS.str.contains('gene_symbol')&(~inFnameS.str.contains('.gz')))
include_m2=inFnameS.isin(['merged_kallisto_run_info.pickle'])
transferFnames=inFnameS[include_m1|include_m2]
key = paramiko.RSAKey.from_private_key_file('/cellar/users/btsui/.ssh/jupyter_hub.pem')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
instance_ip='ec2-34-215-138-8.us-west-2.compute.amazonaws.com'
mountEfs='sudo mount -t efs fs-e1636448:/ efs'
#try:
# Here 'ubuntu' is user name and 'instance_ip' is public IP of EC2
client.connect(hostname=instance_ip, username="ec2-user", pkey=key)
# Execute a command(cmd) after connecting/ssh to an instance
cmd='mkdir ~/efs'
stdin, stdout, stderr = client.exec_command(cmd)
cmd='sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 fs-e1636448.efs.us-west-2.amazonaws.com:/ ~/efs'
stdin, stdout, stderr = client.exec_command(cmd)
print(stdout.read(),stderr.read())
b'' b''
remoteDir='~/efs/all_seq/rnaseq_merged/'
cmd='mkdir '+remoteDir
stdin, stdout, stderr = client.exec_command(cmd)
print (stdout.read())
print (stderr.read())
b'' b'mkdir: cannot create directory \xe2\x80\x98/home/ec2-user/efs/all_seq/rnaseq_merged/\xe2\x80\x99: File exists\n'
addHostVerificationCmd="ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/jupyter_hub.pem ec2-user@{instance_ip} ls".format(instance_ip=instance_ip)
os.system(addHostVerificationCmd)
0
myRsyncDirFormat='rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" {localFDir} ec2-user@{instance_ip}:{remoteDir}'
for localFname in transferFnames.values:
localFDir= (mergedMatricesDir+localFname)
rsyncCmd=myRsyncDirFormat.format(localFDir=localFDir,instance_ip=instance_ip,remoteDir=remoteDir+'.')
print (rsyncCmd)
print (os.system(rsyncCmd))
rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Danio_rerio.gene_symbol.tpm.columns.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Danio_rerio.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.est_counts.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.tpm.columns.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/merged_kallisto_run_info.pickle ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/. 0 rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.est_counts.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.
#!rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt ec2-user@ec2-52-13-69-172.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.
#!rsync -Pvu -e "ssh -i $HOME/.ssh/jupyter_hub.pem" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt \
#ec2-user@ec2-52-24-55-248.us-west-2.compute.amazonaws.com:~/.
#!ls $HOME/.ssh/
authorized_keys id_dsa.pub id_rsa.jupyterhub.pub known_hosts~ aws_upload.pem id_rsa id_rsa.pub id_dsa id_rsa.jupyterhub known_hosts
import seaborn
#!cp ./upload_AWS.ipynb ./../../Update_SRA_meta_data/.