In [1]:
from IPython.parallel import Client
In [2]:
c = Client()
In [3]:
len(c.ids)
Out[3]:
128

Print all 128 nodes hostnames

In [4]:
%px import socket; print(socket.gethostname())
[stdout:0] gcn-4-51.sdsc.edu
[stdout:1] gcn-4-51.sdsc.edu
[stdout:2] gcn-4-51.sdsc.edu
[stdout:3] gcn-4-57.sdsc.edu
[stdout:4] gcn-4-57.sdsc.edu
[stdout:5] gcn-4-57.sdsc.edu
[stdout:6] gcn-4-57.sdsc.edu
[stdout:7] gcn-4-65.sdsc.edu
[stdout:8] gcn-4-65.sdsc.edu
[stdout:9] gcn-4-65.sdsc.edu
[stdout:10] gcn-4-55.sdsc.edu
[stdout:11] gcn-4-55.sdsc.edu
[stdout:12] gcn-4-55.sdsc.edu
[stdout:13] gcn-4-55.sdsc.edu
[stdout:14] gcn-4-55.sdsc.edu
[stdout:15] gcn-4-55.sdsc.edu
[stdout:16] gcn-4-55.sdsc.edu
[stdout:17] gcn-4-55.sdsc.edu
[stdout:18] gcn-4-64.sdsc.edu
[stdout:19] gcn-4-64.sdsc.edu
[stdout:20] gcn-4-64.sdsc.edu
[stdout:21] gcn-4-64.sdsc.edu
[stdout:22] gcn-4-64.sdsc.edu
[stdout:23] gcn-4-64.sdsc.edu
[stdout:24] gcn-4-64.sdsc.edu
[stdout:25] gcn-4-64.sdsc.edu
[stdout:26] gcn-4-58.sdsc.edu
[stdout:27] gcn-4-58.sdsc.edu
[stdout:28] gcn-4-58.sdsc.edu
[stdout:29] gcn-4-58.sdsc.edu
[stdout:30] gcn-4-58.sdsc.edu
[stdout:31] gcn-4-58.sdsc.edu
[stdout:32] gcn-4-58.sdsc.edu
[stdout:33] gcn-4-58.sdsc.edu
[stdout:34] gcn-4-58.sdsc.edu
[stdout:35] gcn-4-58.sdsc.edu
[stdout:36] gcn-4-58.sdsc.edu
[stdout:37] gcn-4-58.sdsc.edu
[stdout:38] gcn-4-58.sdsc.edu
[stdout:39] gcn-4-58.sdsc.edu
[stdout:40] gcn-4-58.sdsc.edu
[stdout:41] gcn-4-58.sdsc.edu
[stdout:42] gcn-4-53.sdsc.edu
[stdout:43] gcn-4-53.sdsc.edu
[stdout:44] gcn-4-53.sdsc.edu
[stdout:45] gcn-4-54.sdsc.edu
[stdout:46] gcn-4-54.sdsc.edu
[stdout:47] gcn-4-54.sdsc.edu
[stdout:48] gcn-4-51.sdsc.edu
[stdout:49] gcn-4-51.sdsc.edu
[stdout:50] gcn-4-51.sdsc.edu
[stdout:51] gcn-4-51.sdsc.edu
[stdout:52] gcn-4-51.sdsc.edu
[stdout:53] gcn-4-51.sdsc.edu
[stdout:54] gcn-4-51.sdsc.edu
[stdout:55] gcn-4-51.sdsc.edu
[stdout:56] gcn-4-51.sdsc.edu
[stdout:57] gcn-4-51.sdsc.edu
[stdout:58] gcn-4-51.sdsc.edu
[stdout:59] gcn-4-51.sdsc.edu
[stdout:60] gcn-4-51.sdsc.edu
[stdout:61] gcn-4-57.sdsc.edu
[stdout:62] gcn-4-57.sdsc.edu
[stdout:63] gcn-4-57.sdsc.edu
[stdout:64] gcn-4-57.sdsc.edu
[stdout:65] gcn-4-57.sdsc.edu
[stdout:66] gcn-4-57.sdsc.edu
[stdout:67] gcn-4-57.sdsc.edu
[stdout:68] gcn-4-57.sdsc.edu
[stdout:69] gcn-4-57.sdsc.edu
[stdout:70] gcn-4-57.sdsc.edu
[stdout:71] gcn-4-57.sdsc.edu
[stdout:72] gcn-4-57.sdsc.edu
[stdout:73] gcn-4-65.sdsc.edu
[stdout:74] gcn-4-65.sdsc.edu
[stdout:75] gcn-4-65.sdsc.edu
[stdout:76] gcn-4-65.sdsc.edu
[stdout:77] gcn-4-65.sdsc.edu
[stdout:78] gcn-4-65.sdsc.edu
[stdout:79] gcn-4-65.sdsc.edu
[stdout:80] gcn-4-65.sdsc.edu
[stdout:81] gcn-4-65.sdsc.edu
[stdout:82] gcn-4-65.sdsc.edu
[stdout:83] gcn-4-65.sdsc.edu
[stdout:84] gcn-4-65.sdsc.edu
[stdout:85] gcn-4-65.sdsc.edu
[stdout:86] gcn-4-55.sdsc.edu
[stdout:87] gcn-4-55.sdsc.edu
[stdout:88] gcn-4-55.sdsc.edu
[stdout:89] gcn-4-55.sdsc.edu
[stdout:90] gcn-4-55.sdsc.edu
[stdout:91] gcn-4-55.sdsc.edu
[stdout:92] gcn-4-55.sdsc.edu
[stdout:93] gcn-4-55.sdsc.edu
[stdout:94] gcn-4-64.sdsc.edu
[stdout:95] gcn-4-64.sdsc.edu
[stdout:96] gcn-4-64.sdsc.edu
[stdout:97] gcn-4-64.sdsc.edu
[stdout:98] gcn-4-64.sdsc.edu
[stdout:99] gcn-4-64.sdsc.edu
[stdout:100] gcn-4-64.sdsc.edu
[stdout:101] gcn-4-64.sdsc.edu
[stdout:102] gcn-4-53.sdsc.edu
[stdout:103] gcn-4-53.sdsc.edu
[stdout:104] gcn-4-53.sdsc.edu
[stdout:105] gcn-4-53.sdsc.edu
[stdout:106] gcn-4-53.sdsc.edu
[stdout:107] gcn-4-53.sdsc.edu
[stdout:108] gcn-4-53.sdsc.edu
[stdout:109] gcn-4-53.sdsc.edu
[stdout:110] gcn-4-53.sdsc.edu
[stdout:111] gcn-4-53.sdsc.edu
[stdout:112] gcn-4-53.sdsc.edu
[stdout:113] gcn-4-53.sdsc.edu
[stdout:114] gcn-4-53.sdsc.edu
[stdout:115] gcn-4-54.sdsc.edu
[stdout:116] gcn-4-54.sdsc.edu
[stdout:117] gcn-4-54.sdsc.edu
[stdout:118] gcn-4-54.sdsc.edu
[stdout:119] gcn-4-54.sdsc.edu
[stdout:120] gcn-4-54.sdsc.edu
[stdout:121] gcn-4-54.sdsc.edu
[stdout:122] gcn-4-54.sdsc.edu
[stdout:123] gcn-4-54.sdsc.edu
[stdout:124] gcn-4-54.sdsc.edu
[stdout:125] gcn-4-54.sdsc.edu
[stdout:126] gcn-4-54.sdsc.edu
[stdout:127] gcn-4-54.sdsc.edu

In [5]:
view = c.load_balanced_view()
In [6]:
def process_file(filename):
    import time
    time.sleep(10)
In [7]:
from glob import glob
In [8]:
filenames = sorted(glob("data/*.csv"))
In [9]:
filenames[:5]
Out[9]:
['data/data_0000.csv',
 'data/data_0001.csv',
 'data/data_0002.csv',
 'data/data_0003.csv',
 'data/data_0004.csv']
In [10]:
len(filenames)
Out[10]:
1000

view.map gets 2 arguments: a function that accepts 1 argument and a list of arguments.

The function will be sent to the available engines in order to process all the arguments in the list. Jupyter will take care of load balancing the resources available

In [11]:
%time optional_return_values = view.map(process_file, filenames, block=True)
CPU times: user 28.9 s, sys: 2.51 s, total: 31.4 s
Wall time: 1min 20s
In [ ]: