def get_enquiry_depths(self):
        # Should be called from all processors associated with operator

        enq0 = None
        enq1 = None

        get0 = "self.inlets[0].get_enquiry_depth()"
        get1 = "self.inlets[1].get_enquiry_depth()"

        if self.myid == self.master_proc:

            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
            else:
                enq0 = pypar.receive(self.enquiry_proc[0])

            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
            else:
                enq1 = pypar.receive(self.enquiry_proc[1])

        else:
            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
                pypar.send(enq0, self.master_proc)

            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
                pypar.send(enq1, self.master_proc)

        return [enq0, enq1]

def distributed_generator(iterable):
    """
    Distribute the values from a generator across workers.
    """
    RUN, DIE = range(2)
    P = pp.size()
    if P == 1:
        for el in iterable:
            yield el
    else:
        if pp.rank() == 0:
            it = iter(iterable)
            while True:
                try:
                    first = next(it)
                    for p in range(1, P):
                        pp.send(next(it), p, tag=RUN)
                    yield first
                except StopIteration:
                    for p in range(1, P):
                        pp.send(666, p, tag=DIE)
                    break
        else:
            while True:
                el, status = pp.receive(0, tag=pp.any_tag, return_status=True)
                if status.tag == DIE:
                    break
                yield el

def scatter_dict(whole):
    """
    Broadcast and recieve a dictionary where the values are 1d arrays
    and the arrays are chunked for the workers.
    Only rank 0 needs the whole dictionary.

    :param whole: The dictionary of 1d arrays to subdict.
    :returns: (chunk of dictionary of 1d arrays, indexes of whole array)
    """
    if not STATE.is_parallel:
        array_len = len(whole[whole.keys()[0]])
        return whole, numpy.array(range(0, array_len))
    else:
        import pypar     # pylint: disable=W0404

    if STATE.rank == 0:
        array_len = len(whole[whole.keys()[0]])
        for pro in range(0, STATE.size):
            temp_indexes = numpy.array(range(pro, array_len, STATE.size))
            temp_subdict = {}
            for key in whole.keys():
                temp_subdict[key] = whole[key][temp_indexes]
            if pro is 0:
                indexes = temp_indexes
                subdict = temp_subdict
            else:
                pypar.send(temp_indexes, pro)
                pypar.send(temp_subdict, pro)
    else:
        indexes = pypar.receive(0)
        subdict = pypar.receive(0)
    return subdict, indexes

    def get_global_average_elevation(self):
        # GLOBAL: Master processor gathers elevations from all child processors, and returns average

        # WARNING: requires synchronization, must be called by all procs associated
        # with this inlet

        import pypar
        local_elevation = num.sum(self.get_elevations()*self.get_areas())
        global_area = self.get_global_area()



        global_elevation = local_elevation

        if self.myid == self.master_proc:
            for i in self.procs:
                if i == self.master_proc: continue

                val = pypar.receive(i)
                global_elevation = global_elevation + val
        else:
            pypar.send(local_elevation, self.master_proc)


        if global_area > 0.0:
            return global_elevation/global_area
        else:
            return 0.0

	def balanceArrays(self, arrayFragment):
	
		'''Redistributes the elements in a set of arrays equally across the nodes'''
		
		if Environment.isParallel:
			import pypar
			if self.isRoot():
				completeArray = arrayFragment
				for i in range(1, pypar.size()):
					fragment = pypar.receive(i)
					completeArray.extend(fragment)
				
				#Divide it up
				divisions = self._divideArray(completeArray)
				
				#Send the fragments
				for i in range(1, pypar.size()):
					start, end = divisions[i]
					pypar.send(completeArray[start:end], i)
					
				self.output('[ENV] Rebalanced array divisions %s' % divisions)	
					
				#Assign root fragment	
				start, end = divisions[0]	
				arrayFragment = completeArray[start:end]
			else:
				#Send the fragment
				pypar.send(arrayFragment, 0)
				#Retrieve the array
				arrayFragment = pypar.receive(0)
		else:
			completeArray = arrayFragment
		
		return arrayFragment

def test_mpi_recvsend(target_array, message_range, target_rank, tag_mark):

    send_range = message_range[0]
    recv_range = message_range[1]
    target_array[recv_range] = mpi.receive(target_rank,tag=tag_mark)
  #  print 'I`m', myrank, 'Recv : ', target_rank, 'range : ', recv_range
    mpi.send(target_array[send_range].copy(), target_rank, tag=tag_mark)

	def splitArray(self, array):
	
		'''Splits array between all the processes in the environment.
		
		Each process will be returned a different section of the array to work on'''
	
		if Environment.isParallel:
			import pypar
			#Split the array into sections and return the section for this processor
			divisions = []
			if self.isRoot():
				#Root does the splitting - we send each processor the start and end index
				#NOTE: pypar broadcast won't work even when setting vanilla
				#It always returns message trucated error.
				divisions = self._divideArray(array)	
				for i in range(1,pypar.size()):
					pypar.send(divisions[i], i)
				start = divisions[0][0]
				end = divisions[0][1]
			else:	
				indexes = pypar.receive(0)
				start = indexes[0]
				end = indexes[1]
				
			return array[start:end]
		else:
			return array

 def send(self, *args, **kwargs):
     """
     Wrapper for pypar.send
     """
     if self.is_parallel is True:
         import pypar
         pypar.send(*args, **kwargs)

    def get_enquiry_water_depths(self):

        enq0 = None
        enq1 = None

        get0 = 'self.inlets[0].get_enquiry_water_depth()'
        get1 = 'self.inlets[1].get_enquiry_water_depth()'


        if self.myid == self.master_proc:

            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
            else:
                enq0 = pypar.receive(self.enquiry_proc[0])


            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
            else:
                enq1 = pypar.receive(self.enquiry_proc[1])

        else:
            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
                pypar.send(enq0, self.master_proc)

            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
                pypar.send(enq1, self.master_proc)


        return [enq0, enq1]

def scatter( vec ):
	if root():
		for i in xrange(p.size()-1):
			p.send(vec[i+1],i+1)
		return vec[0]
	else:
		return p.receive(0)

def broadcast( obj ):
	if root():
		for i in xrange(p.size()-1):
			p.send(obj,i+1)
		return obj
	else:
		return p.receive(0)

    def get_enquiry_velocitys(self):

        enq0 = None
        enq1 = None

        get0 = "self.inlets[0].get_enquiry_velocity()"
        get1 = "self.inlets[1].get_enquiry_velocity()"

        if self.myid == self.master_proc:

            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
            else:
                enq0 = pypar.receive(self.enquiry_proc[0])

            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
            else:
                enq1 = pypar.receive(self.enquiry_proc[1])

        else:
            if self.myid == self.enquiry_proc[0]:
                enq0 = eval(get0)
                pypar.send(enq0, self.master_proc)

            if self.myid == self.enquiry_proc[1]:
                enq1 = eval(get1)
                pypar.send(enq1, self.master_proc)

        return [enq0, enq1]

def broadcast_vec( vec, i ):
	myid = p.rank()
	if myid == i:
		for j in xrange(p.size()):
			if j != myid:
				p.send(vec[i],j)
	else:
		vec[i] = p.receive(i)

def gather( obj ):
	if root():
		result = [ None for i in xrange(p.size()) ]
		result[0] = obj
		for i in xrange(p.size()-1):
			result[i+1] = p.receive(i+1)
		return result
	else:
		p.send(obj,0)

    def Calculate(self, varsByCalc, params = None):
        """
        Calculate model predictions for everything in varsByCalc.

        varsByCalc is a dictionary of the form:
            dict[calc name][dep var] = ind var
        
        The return dictionary is of the form:
            dictionary[calc name][dep var][ind var] = result
        """
        if params is not None:
            self.params.update(params)

        results = {}

        calcs_to_do = varsByCalc.keys()
        # Record which calculation each node is doing
        calc_assigned = {}
        while calcs_to_do:
            # The number of calculations to do this round. We want to use
            #  all the processors if possible.
            len_this_block = min(SloppyCell.num_procs, len(calcs_to_do))

            for worker in range(1, len_this_block):
                calc = calcs_to_do.pop()
                calc_assigned[worker] = calc
                logger.debug('Assigning calculation %s to worker %i.'
                             % (calc, worker))
                command = 'Network.calculate(net, vars, params)'
                args = {'net': self.get(calc), 'vars': varsByCalc[calc],
                        'params': self.params}
                pypar.send((command, args), worker)

            # The master does his share here
            calc = calcs_to_do.pop()
            # We use the finally statement because we want to ensure that we
            #  *always* wait for replies from the workers, even if the master
            #  encounters an exception in his evaluation.
            try:
                results[calc] = self.get(calc).calculate(varsByCalc[calc], 
                                                         self.params)
            finally:
                # Collect results from the workers
                for worker in range(1, len_this_block):
                    logger.debug('Receiving result from worker %i.' % worker)
                    results[calc_assigned[worker]] = pypar.receive(worker)
                # If the master encounts an exception, we'll break out of the
                #  function ***here***

            # Check the results we received. If any is a SloppyCellException, 
            #  reraise it.
            for val in results.values():
                if isinstance(val, Utility.SloppyCellException):
                    raise val

        return results

   def process(self):
      # loop through all the rays that go through the window
      for row in xrange(self.Rows):
         for col in xrange(self.WindowCols):

            # loop through all the light sources
            for Light in self.LightSources:
               Energy = 0.0
                  
               # move the ray to the corresponding pixel on the window. dx and dy changes
               MoveUp = self.WindowHeight/2.0 - (row+self.RowStart - 0.5)*\
                        self.WindowHeight/self.WindowRows # starts from top most pixel
               MoveRight = -self.WindowWidth/2.0 + (col + 0.5)*\
                           self.WindowWidth/self.WindowCols # starts from left most pixel
                  
               # define the ray and rotate (in look direction)
               # then translate (window distance)
               Vec = np.array([self.WindowDistance, -MoveRight, MoveUp])
               Vec = self.__RotVec(Vec, self.__Lon, self.__Lat, self.Yaw)
               Photon = Ray.Ray(self.LookPos,Vec,1.0)
               
               Cont = True
               
		         # get energy after first contact
               Res = self.__getEn(Photon, Light)
               Energy += Res[0]
               UnScatRay = Res[1]
               Cont = Res[2]
               
               #pypar.barrier();
               
               for k in range(1):
                  # get energy after N contacts
                  if Cont:
                     Res = self.__getEn(UnScatRay, Light)
                     Energy += Res[0]
                     UnScatRay = Res[1]
                     Cont = Res[2]
                  
               self.Window[row, col] += Energy # set the energy to the corresponding window
  
      # stitch together the images from each process
      if self.myid != 0:
         pypar.send(self.Window, 0)
      else:
         self.Visual = np.zeros(shape = (self.WindowRows, self.WindowCols))
         RowEnd = self.WindowRows/self.numproc
         self.Visual[:RowEnd, :] = self.Window.copy()
         for Pr in range(1,self.numproc):            
            RowStart = self.WindowRows/self.numproc * Pr
            RowEnd = self.WindowRows/self.numproc * (Pr+1)
            self.Visual[RowStart:RowEnd,:] = pypar.receive(Pr)
      
         print "Elapsed time for process %d is %f" % (self.myid, time.time() - self.start)

    def grad_func(x0, *grad_args):
        sys.stdout.flush()
        x0 = numpy.asarray(x0)

        # Generate a list of parameter sets to evaluate the function at
        x_todo = [x0]
        for ii in range(len(x0)):
            eps = numpy.zeros(len(x0), numpy.float_)
            eps[ii] = epsilon
            x_todo.append(x0 + eps)

        # Break that list apart into work for each node
        x_by_node = []
        for node_ii in range(num_procs):
            x_by_node.append(x_todo[node_ii::num_procs])

        # Master sends orders to all other nodes.
        command = 'eval_func_over_list(func_str, x_todo, *grad_args)'
        for node_ii in range(1, num_procs):
            x_this_node = x_by_node[node_ii]
            arguments = {'func_str': func_str,
                         'x_todo': x_this_node}
            if send_grad_args:
                arguments['grad_args'] = grad_args
            pypar.send((command, arguments), node_ii)
        sys.stdout.flush()

        # This will hold the function evaluations done by each node.
        vals_by_node = []
        # The master node does its share of the work now
        vals_by_node.append(eval_func_over_list(func_str, x_by_node[0], 
                                                *grad_args))
        # Now receive the work done by each of the other nodes
        for node_ii in range(1, num_procs):
            vals_by_node.append(pypar.receive(node_ii))

        # Reform the function value list that's broken apart by node.
        func_evals = numpy.zeros(len(x0)+1, numpy.float_)
        for node_ii,vals_this_node in enumerate(vals_by_node):
            func_evals[node_ii::num_procs] = vals_this_node

        # Now calculate the gradient
        grad = numpy.zeros(len(x0), numpy.float_)
        f0 = func_evals[0]
        for ii,func_val in enumerate(func_evals[1:]):
            grad[ii] = (func_val - f0)/epsilon

        return grad

def collect_arr(arr):
    """
    A useful collection routine for pypar.
    If you are using pypar to parallelize the set of nested loops and fill
    the resulting array, you usually need to combine the resulting array
    from several mpi threads. In that case you just can execute
    res=collect_arr(res)
    And it will add the arrays from all the threads and store them in
    the thread number 0
    """
    if pypar.rank() > 0:
        pypar.send(arr, 0)
    else:
        for i in range(1, pypar.size()):
            arr = arr + pypar.receive(i)
    return arr

    def test_longint_array(self):
        myid, ncpu = pp.rank(), pp.size()
        N = 17  # Number of elements

        if myid == 0:
            A = np.array(range(N)).astype('l')
            B = np.zeros(N).astype('l')

            pp.send(A, 1, use_buffer=True)
            B = pp.receive(ncpu - 1, buffer=B)

            self.assertTrue(np.allclose(A, B))
        else:
            X = np.zeros(N).astype('l')
            X = pp.receive(myid - 1, buffer=X)
            pp.send(X, (myid + 1) % ncpu, use_buffer=True)

	def broadcast(self, data, process):
	
		'''Broadcasts data from process to all other nodes'''
		
		if Environment.isParallel:
			import pypar
			if self.rank() == process:
				#NOTE: pypar broadcast won't work even when setting vanilla
				#It always returns message trucated error.
				for i in range(pypar.size()):
					if i != self.rank():
						pypar.send(data, i)
			else:	
				data = pypar.receive(process)
		
		return data