#!/usr/bin/env python
# coding: utf-8

# # ISB-CGC Community Notebooks
# Check out more notebooks at our [Community Notebooks Repository](https://github.com/isb-cgc/Community-Notebooks)!
# 
# ```
# Title:   How to work with cloud storage
# Author:  David L Gibbs
# Created: 2019-07-17
# Purpose: Demonstrate how to move files --in and out of-- GCS.
# ```
# ***
# 

# ## **How to work with cloud storage.**

# ### **In this notebook, we demonstrate how to work with files stored in GCS buckets.**
# 

# Documentation at:  https://cloud.google.com/storage/docs/listing-objects

# ### Let's authenticate ourselves

# In[1]:


# with gcloud, we can authenticate ourselves

get_ipython().system('gcloud auth login')


# In[2]:


# and we can select our project

get_ipython().system('gcloud config set project PROJECT_ID')


# ### Using 'bangs'.

# Using a 'bang', or the exclamaition point (!), we can run command line commands.
# This includes file operations like 'ls', 'mv', and 'cp'.
# We can also use Google's tools including 'gsutil'.

# In[3]:


# here we get a list of files stored *locally*
get_ipython().system('ls -lha')


# In[ ]:





# In[4]:


# Now we use gsutil to list files in our bucket.
get_ipython().system('gsutil ls gs://bam_bucket_1/')


# In[ ]:





# In[5]:


# then we can copy to our local env.
get_ipython().system('gsutil cp gs://bam_bucket_1/test.bam test_dl.bam')


# In[ ]:





# In[6]:


# and it made it?
get_ipython().system('ls -lha | grep test')


# In[ ]:





# In[7]:


# then we can copy it back to our bucket
get_ipython().system('mv test_dl.bam renamed_test.bam')
get_ipython().system('gsutil cp renamed_test.bam gs://bam_bucket_1/renamed_test.bam')


# In[ ]:





# ### Using 'pythons'.

# In[8]:


# to install and import the library
#!pip3 install --upgrade --user google-cloud-storage


# In[12]:


get_ipython().system('gcloud auth application-default login')
#Your browser has been opened to visit:
#
#    https://accounts.google.com/o/oauth2/auth?redirect_uri=.....
#
#Credentials saved to file: [/home/davidgibbs/.config/gcloud/application_default_credentials.json]


# In[ ]:





# In[18]:


get_ipython().system('export GOOGLE_APPLICATION_CREDENTIALS="/home/davidgibbs/.config/gcloud/application_default_credentials.json"')
import google.auth
import google.cloud.storage as storage
credentials, project = google.auth.default()


# In[19]:


# connection to our project
storage_client = storage.Client()


# In[20]:


for b in storage_client.list_buckets():
    print(b)


# In[22]:


# here we'll create a bucket
bucket = storage_client.create_bucket('wild_new_bucket_2000')
print('Bucket {} created'.format(bucket.name))


# In[27]:


# and then move a file to it
bucket = storage_client.get_bucket('wild_new_bucket_2000')
blob = bucket.blob('test_dl_upload.bam')
blob.upload_from_filename('renamed_test.bam')


# In[29]:


# and check that it made it 
blobs = bucket.list_blobs()
for blob in blobs:
    print(blob.name)


# In[ ]:


# end of notebook