from IPython.parallel import Client
c = Client()
len(c.ids)
128
Print all 128 nodes hostnames
%px import socket; print(socket.gethostname())
[stdout:0] gcn-4-51.sdsc.edu [stdout:1] gcn-4-51.sdsc.edu [stdout:2] gcn-4-51.sdsc.edu [stdout:3] gcn-4-57.sdsc.edu [stdout:4] gcn-4-57.sdsc.edu [stdout:5] gcn-4-57.sdsc.edu [stdout:6] gcn-4-57.sdsc.edu [stdout:7] gcn-4-65.sdsc.edu [stdout:8] gcn-4-65.sdsc.edu [stdout:9] gcn-4-65.sdsc.edu [stdout:10] gcn-4-55.sdsc.edu [stdout:11] gcn-4-55.sdsc.edu [stdout:12] gcn-4-55.sdsc.edu [stdout:13] gcn-4-55.sdsc.edu [stdout:14] gcn-4-55.sdsc.edu [stdout:15] gcn-4-55.sdsc.edu [stdout:16] gcn-4-55.sdsc.edu [stdout:17] gcn-4-55.sdsc.edu [stdout:18] gcn-4-64.sdsc.edu [stdout:19] gcn-4-64.sdsc.edu [stdout:20] gcn-4-64.sdsc.edu [stdout:21] gcn-4-64.sdsc.edu [stdout:22] gcn-4-64.sdsc.edu [stdout:23] gcn-4-64.sdsc.edu [stdout:24] gcn-4-64.sdsc.edu [stdout:25] gcn-4-64.sdsc.edu [stdout:26] gcn-4-58.sdsc.edu [stdout:27] gcn-4-58.sdsc.edu [stdout:28] gcn-4-58.sdsc.edu [stdout:29] gcn-4-58.sdsc.edu [stdout:30] gcn-4-58.sdsc.edu [stdout:31] gcn-4-58.sdsc.edu [stdout:32] gcn-4-58.sdsc.edu [stdout:33] gcn-4-58.sdsc.edu [stdout:34] gcn-4-58.sdsc.edu [stdout:35] gcn-4-58.sdsc.edu [stdout:36] gcn-4-58.sdsc.edu [stdout:37] gcn-4-58.sdsc.edu [stdout:38] gcn-4-58.sdsc.edu [stdout:39] gcn-4-58.sdsc.edu [stdout:40] gcn-4-58.sdsc.edu [stdout:41] gcn-4-58.sdsc.edu [stdout:42] gcn-4-53.sdsc.edu [stdout:43] gcn-4-53.sdsc.edu [stdout:44] gcn-4-53.sdsc.edu [stdout:45] gcn-4-54.sdsc.edu [stdout:46] gcn-4-54.sdsc.edu [stdout:47] gcn-4-54.sdsc.edu [stdout:48] gcn-4-51.sdsc.edu [stdout:49] gcn-4-51.sdsc.edu [stdout:50] gcn-4-51.sdsc.edu [stdout:51] gcn-4-51.sdsc.edu [stdout:52] gcn-4-51.sdsc.edu [stdout:53] gcn-4-51.sdsc.edu [stdout:54] gcn-4-51.sdsc.edu [stdout:55] gcn-4-51.sdsc.edu [stdout:56] gcn-4-51.sdsc.edu [stdout:57] gcn-4-51.sdsc.edu [stdout:58] gcn-4-51.sdsc.edu [stdout:59] gcn-4-51.sdsc.edu [stdout:60] gcn-4-51.sdsc.edu [stdout:61] gcn-4-57.sdsc.edu [stdout:62] gcn-4-57.sdsc.edu [stdout:63] gcn-4-57.sdsc.edu [stdout:64] gcn-4-57.sdsc.edu [stdout:65] gcn-4-57.sdsc.edu [stdout:66] gcn-4-57.sdsc.edu [stdout:67] gcn-4-57.sdsc.edu [stdout:68] gcn-4-57.sdsc.edu [stdout:69] gcn-4-57.sdsc.edu [stdout:70] gcn-4-57.sdsc.edu [stdout:71] gcn-4-57.sdsc.edu [stdout:72] gcn-4-57.sdsc.edu [stdout:73] gcn-4-65.sdsc.edu [stdout:74] gcn-4-65.sdsc.edu [stdout:75] gcn-4-65.sdsc.edu [stdout:76] gcn-4-65.sdsc.edu [stdout:77] gcn-4-65.sdsc.edu [stdout:78] gcn-4-65.sdsc.edu [stdout:79] gcn-4-65.sdsc.edu [stdout:80] gcn-4-65.sdsc.edu [stdout:81] gcn-4-65.sdsc.edu [stdout:82] gcn-4-65.sdsc.edu [stdout:83] gcn-4-65.sdsc.edu [stdout:84] gcn-4-65.sdsc.edu [stdout:85] gcn-4-65.sdsc.edu [stdout:86] gcn-4-55.sdsc.edu [stdout:87] gcn-4-55.sdsc.edu [stdout:88] gcn-4-55.sdsc.edu [stdout:89] gcn-4-55.sdsc.edu [stdout:90] gcn-4-55.sdsc.edu [stdout:91] gcn-4-55.sdsc.edu [stdout:92] gcn-4-55.sdsc.edu [stdout:93] gcn-4-55.sdsc.edu [stdout:94] gcn-4-64.sdsc.edu [stdout:95] gcn-4-64.sdsc.edu [stdout:96] gcn-4-64.sdsc.edu [stdout:97] gcn-4-64.sdsc.edu [stdout:98] gcn-4-64.sdsc.edu [stdout:99] gcn-4-64.sdsc.edu [stdout:100] gcn-4-64.sdsc.edu [stdout:101] gcn-4-64.sdsc.edu [stdout:102] gcn-4-53.sdsc.edu [stdout:103] gcn-4-53.sdsc.edu [stdout:104] gcn-4-53.sdsc.edu [stdout:105] gcn-4-53.sdsc.edu [stdout:106] gcn-4-53.sdsc.edu [stdout:107] gcn-4-53.sdsc.edu [stdout:108] gcn-4-53.sdsc.edu [stdout:109] gcn-4-53.sdsc.edu [stdout:110] gcn-4-53.sdsc.edu [stdout:111] gcn-4-53.sdsc.edu [stdout:112] gcn-4-53.sdsc.edu [stdout:113] gcn-4-53.sdsc.edu [stdout:114] gcn-4-53.sdsc.edu [stdout:115] gcn-4-54.sdsc.edu [stdout:116] gcn-4-54.sdsc.edu [stdout:117] gcn-4-54.sdsc.edu [stdout:118] gcn-4-54.sdsc.edu [stdout:119] gcn-4-54.sdsc.edu [stdout:120] gcn-4-54.sdsc.edu [stdout:121] gcn-4-54.sdsc.edu [stdout:122] gcn-4-54.sdsc.edu [stdout:123] gcn-4-54.sdsc.edu [stdout:124] gcn-4-54.sdsc.edu [stdout:125] gcn-4-54.sdsc.edu [stdout:126] gcn-4-54.sdsc.edu [stdout:127] gcn-4-54.sdsc.edu
view = c.load_balanced_view()
def process_file(filename):
import time
time.sleep(10)
from glob import glob
filenames = sorted(glob("data/*.csv"))
filenames[:5]
['data/data_0000.csv', 'data/data_0001.csv', 'data/data_0002.csv', 'data/data_0003.csv', 'data/data_0004.csv']
len(filenames)
1000
view.map
gets 2 arguments: a function that accepts 1 argument and a list of arguments.
The function will be sent to the available engines in order to process all the arguments in the list. Jupyter will take care of load balancing the resources available
%time optional_return_values = view.map(process_file, filenames, block=True)
CPU times: user 28.9 s, sys: 2.51 s, total: 31.4 s Wall time: 1min 20s