#!/usr/bin/env python
# coding: utf-8

# # Using Parallel Magics

# IPython has a few magics for working with your engines.
# 
# This assumes you have started an IPython cluster, either with the notebook interface,
# or the `ipcluster/controller/engine` commands.

# In[ ]:


from IPython import parallel
rc = parallel.Client()
dv = rc[:]
rc.ids


# Creating a Client registers the parallel magics `%px`, `%%px`, `%pxresult`, `pxconfig`, and `%autopx`.  
# These magics are initially associated with a DirectView always associated with all currently registered engines.

# Now we can execute code remotely with `%px`:

# In[ ]:


get_ipython().run_line_magic('px', 'a=5')


# In[ ]:


get_ipython().run_line_magic('px', 'print(a)')


# In[ ]:


get_ipython().run_line_magic('px', 'a')


# In[ ]:


with dv.sync_imports():
    import sys


# In[ ]:


get_ipython().run_line_magic('px', 'from __future__ import print_function')
get_ipython().run_line_magic('px', 'print("ERROR", file=sys.stderr)')


# You don't have to wait for results.  The `%pxconfig` magic lets you change the default blocking/targets for the `%px` magics:

# In[ ]:


get_ipython().run_line_magic('pxconfig', '--noblock')


# In[ ]:


get_ipython().run_line_magic('px', 'import time')
get_ipython().run_line_magic('px', 'time.sleep(5)')
get_ipython().run_line_magic('px', 'time.time()')


# But you will notice that this didn't output the result of the last command.
# For this, we have `%pxresult`, which displays the output of the latest request:

# In[ ]:


get_ipython().run_line_magic('pxresult', '')


# Remember, an IPython engine is IPython, so you can do magics remotely as well!

# In[ ]:


get_ipython().run_line_magic('pxconfig', '--block')
get_ipython().run_line_magic('px', '%matplotlib inline')


# In[ ]:


get_ipython().run_cell_magic('px', '', 'import numpy as np\nimport matplotlib.pyplot as plt\n')


# `%%px` can also be used as a cell magic, for submitting whole blocks.
# This one acceps `--block` and `--noblock` flags to specify
# the blocking behavior, though the default is unchanged.
# 

# In[ ]:


dv.scatter('id', dv.targets, flatten=True)
dv['stride'] = len(dv)


# In[ ]:


get_ipython().run_cell_magic('px', '--noblock', 'x = np.linspace(0,np.pi,1000)\nfor n in range(id,12, stride):\n    print(n)\n    plt.plot(x,np.sin(n*x))\nplt.title("Plot %i" % id)\n')


# In[ ]:


get_ipython().run_line_magic('pxresult', '')


# It also lets you choose some amount of the grouping of the outputs with `--group-outputs`:
# 
# The choices are:
# 
# * `engine` - all of an engine's output is collected together
# * `type` - where stdout of each engine is grouped, etc. (the default)
# * `order` - same as `type`, but individual displaypub outputs are interleaved.
#   That is, it will output the first plot from each engine, then the second from each,
#   etc.

# In[ ]:


get_ipython().run_cell_magic('px', '--group-outputs=engine', 'x = np.linspace(0,np.pi,1000)\nfor n in range(id+1,12, stride):\n    print(n)\n    plt.figure()\n    plt.plot(x,np.sin(n*x))\n    plt.title("Plot %i" % n)\n')


# When you specify 'order', then individual display outputs (e.g. plots) will be interleaved.
# 
# `%pxresult` takes the same output-ordering arguments as `%%px`, 
# so you can view the previous result in a variety of different ways with a few sequential calls to `%pxresult`:

# In[ ]:


get_ipython().run_line_magic('pxresult', '--group-outputs=order')


# ## Single-engine views

# When a DirectView has a single target, the output is a bit simpler (no prefixes on stdout/err, etc.):

# In[ ]:


from __future__ import print_function

def generate_output():
    """function for testing output
    
    publishes two outputs of each type, and returns something
    """
    
    import sys,os
    from IPython.display import display, HTML, Math
    
    print("stdout")
    print("stderr", file=sys.stderr)
    
    display(HTML("<b>HTML</b>"))
    
    print("stdout2")
    print("stderr2", file=sys.stderr)
    
    display(Math(r"\alpha=\beta"))
    
    return os.getpid()

dv['generate_output'] = generate_output


# You can also have more than one set of parallel magics registered at a time.
# 
# The `View.activate()` method takes a suffix argument, which is added to `'px'`.

# In[ ]:


e0 = rc[-1]
e0.block = True
e0.activate('0')


# In[ ]:


get_ipython().run_line_magic('px0', 'generate_output()')


# In[ ]:


get_ipython().run_line_magic('px', 'generate_output()')


# As mentioned above, we can redisplay those same results with various grouping:

# In[ ]:


get_ipython().run_line_magic('pxresult', '--group-outputs order')


# In[ ]:


get_ipython().run_line_magic('pxresult', '--group-outputs engine')


# ## Parallel Exceptions

# When you raise exceptions with the parallel exception,
# the CompositeError raised locally will display your remote traceback.

# In[ ]:


get_ipython().run_cell_magic('px', '', "from numpy.random import random\nA = random((100,100,'invalid shape'))\n")


# ## Remote Cell Magics

# Remember, Engines are IPython too, so the cell that is run remotely by %%px can in turn use a cell magic.

# In[ ]:


get_ipython().run_cell_magic('px', '', '%%timeit\nfrom numpy.random import random\nfrom numpy.linalg import norm\nA = random((100,100))\nnorm(A, 2)\n')


# ## Local Execution

# As of IPython 1.0, you can instruct `%%px` to also execute the cell locally.
# This is useful for interactive definitions,
# or if you want to load a data source everywhere,
# not just on the engines.

# In[ ]:


get_ipython().run_cell_magic('px', '--local', 'import os\nthispid = os.getpid()\nprint(thispid)\n')