#!/usr/bin/env python
# coding: utf-8

# In[1]:


from IPython.parallel import Client


# In[2]:


c = Client()


# In[3]:


len(c.ids)


# Print all 128 nodes hostnames

# In[4]:


get_ipython().run_line_magic('px', 'import socket; print(socket.gethostname())')


# In[5]:


view = c.load_balanced_view()


# In[6]:


def process_file(filename):
    import time
    time.sleep(10)


# In[7]:


from glob import glob


# In[8]:


filenames = sorted(glob("data/*.csv"))


# In[9]:


filenames[:5]


# In[10]:


len(filenames)


# `view.map` gets 2 arguments: a function that accepts 1 argument and a list of arguments.
# 
# The function will be sent to the available engines in order to process all the arguments in the list. Jupyter will take care of load balancing the resources available

# In[11]:


get_ipython().run_line_magic('time', 'optional_return_values = view.map(process_file, filenames, block=True)')


# In[ ]: