def __init__(self, workflow, **kwargs):
     super(GradientsCalculator, self).__init__(workflow, **kwargs)
     self.vbias_grad = Array()
     self.hbias_grad = Array()
     self.weights_grad = Array()
     self.demand("hbias1", "vbias1", "hbias0", "vbias0", "weights0",
                 "weights1")

 def clone(self):
     for unit, attrs in self.reals.items():
         for attr in attrs:
             value = getattr(unit, attr)
             if self.is_immutable(value):
                 setattr(self, attr, value)
                 continue
             if not isinstance(value, Array):
                 cloned = getattr(self, attr, None)
                 if cloned is None:
                     setattr(self, attr, deepcopy(value))
                     continue
                 if isinstance(value, list):
                     del cloned[:]
                     cloned.extend(value)
                 elif isinstance(value, (dict, set)):
                     cloned.clear()
                     cloned.update(value)
                 elif isinstance(value, Bool):
                     cloned <<= value
                 elif isinstance(value, numpy.ndarray):
                     cloned[:] = value
                 else:
                     setattr(self, attr, deepcopy(value))
                 continue
             vec = getattr(self, attr, None)
             if vec is None:
                 vec = Array()
                 self.vectors[value] = vec
                 setattr(self, attr, vec)
             else:
                 assert isinstance(vec, Array)
             if not vec and value:
                 vec.reset(value.mem.copy())

 def __init__(self, workflow, **kwargs):
     super(Uniform, self).__init__(workflow, **kwargs)
     self.num_states = kwargs.get("num_states", 256)
     self.states = Array()
     self.prng = kwargs.get("prng", get())
     self.output_bytes = kwargs.get("output_bytes", 0)
     self.output = Array()
     self.cl_const = numpy.zeros(1, dtype=numpy.int32)

 def __init__(self, workflow, **kwargs):
     super(EvaluatorSoftmax, self).__init__(workflow, **kwargs)
     self.compute_confusion_matrix = kwargs.get(
         "compute_confusion_matrix", True)
     self.confusion_matrix = Array()
     self.n_err = Array()
     self.max_err_output_sum = Array()
     self.demand("labels", "max_idx")

class Summator(AcceleratedUnit):
    """Multiplies two vectors pointwise.
    """
    def __init__(self, workflow, **kwargs):
        super(Summator, self).__init__(workflow, **kwargs)
        self.output = Array()
        self.demand("x", "y")

    def initialize(self, device, **kwargs):
        super(Summator, self).initialize(device, **kwargs)
        if not self.output:
            self.output.reset(numpy.zeros_like(self.x.mem))
        else:
            assert self.output.shape == self.x.shape
        self.init_vectors(self.x, self.y, self.output)

    def init_unpickled(self):
        super(Summator, self).init_unpickled()
        self.sources_["summator"] = {}

    def _gpu_init(self):
        self.build_program({"OUTPUT_SIZE": self.output.size},
                           "%s_%d" %
                           (self.__class__.__name__, self.output.size),
                           dtype=self.x.dtype)
        self.assign_kernel("add_forward")
        self.set_args(self.x, self.y, self.output)

    def cuda_init(self):
        self._gpu_init()
        block_size = self.device.suggest_block_size(self._kernel_)
        self._global_size = (
            int(numpy.ceil(self.output.size / block_size)), 1, 1)
        self._local_size = (block_size, 1, 1)

    def ocl_init(self):
        self._gpu_init()
        self._global_size = (self.output.size, 1, 1)
        self._local_size = None

    def numpy_init(self):
        pass  # nothing to init

    def _gpu_run(self):
        self.unmap_vectors(self.x, self.y, self.output)
        self.execute_kernel(self._global_size, self._local_size)

    def cuda_run(self):
        self._gpu_run()

    def ocl_run(self):
        self._gpu_run()

    def numpy_run(self):
        self.x.map_read()
        self.y.map_read()
        self.output.map_invalidate()
        numpy.add(self.x.mem, self.y.mem, self.output.mem)

 def __init__(self, workflow, **kwargs):
     super(EvaluatorMSE, self).__init__(workflow, **kwargs)
     self.metrics = Array()
     self.mse = Array()
     self.labels = None
     self.class_targets = None
     self.n_err = Array()
     self.root = kwargs.get("root", True)
     self.demand("target", "normalizer")

 def __init__(self, workflow, **kwargs):
     super(KohonenForward, self).__init__(workflow, **kwargs)
     self.demand("input", "weights")
     self.argmins = None
     self._distances = Array()
     self.output = Array()
     self._chunk_size_ = 0
     self.weights_transposed = False
     self.total = Array() if kwargs.get("total", False) else None
     if self.total is not None:
         self.minibatch_offset = None
         self.minibatch_size = None
         self.batch_size = None

 def __init__(self, workflow, **kwargs):
     super(Cutter1D, self).__init__(workflow, **kwargs)
     self.alpha = kwargs.get("alpha")
     self.beta = kwargs.get("beta")
     self.output_offset = kwargs.get("output_offset", 0)
     self.output = Array()
     self.demand("alpha", "beta", "input")

 def __init__(self, workflow, **kwargs):
     kwargs["view_group"] = kwargs.get("view_group", "WORKER")
     super(MeanDispNormalizer, self).__init__(workflow, **kwargs)
     self.output = Array()
     self.global_size = None
     self.local_size = None
     self.demand("input", "mean", "rdisp")

 def __init__(self, workflow, **kwargs):
     super(MultiHistogram, self).__init__(workflow, **kwargs)
     self.limit = kwargs.get("limit", 64)
     self.value = Array()
     self.n_bars = kwargs.get("n_bars", 25)
     self.hist_number = kwargs.get("hist_number", 16)
     self.demand("input")

 def __init__(self, workflow, **kwargs):
     kwargs["view_group"] = kwargs.get("view_group", "WORKER")
     super(Forward, self).__init__(workflow, **kwargs)
     self.weights_stddev = kwargs.get("weights_stddev")
     self.bias_stddev = kwargs.get("bias_stddev", self.weights_stddev)
     self.weights_filling = kwargs.get("weights_filling", "uniform")
     self.bias_filling = kwargs.get("bias_filling", "uniform")
     self.rand = kwargs.get("rand", prng.get())
     self.weights_transposed = kwargs.get("weights_transposed", False)
     self.include_bias = kwargs.get("include_bias", True)
     self.demand("input")
     self.output = Array(shallow_pickle=True)
     self.weights = Array()
     self.bias = Array()
     self.forward_mode = False
     self.exports = ["weights", "bias", "include_bias", "weights_transposed"]

class FixAccumulator(Unit):
    """
    Range accumulator.
    """
    def __init__(self, workflow, **kwargs):
        super(FixAccumulator, self).__init__(workflow)
        self.bars = kwargs.get("bars", 200)
        self.type = kwargs.get("type", "relu")
        self.input = None
        self.output = Array()
        self.reset_flag = Bool(True)
        self.n_bars = [0]
        self.max = 100
        self.min = 0

    def initialize(self, **kwargs):
        self.output.mem = numpy.zeros([self.bars + 2], dtype=numpy.int64)

    def run(self):
        if self.type == "relu":
            self.max = 10000
            self.min = 0
        elif self.type == "tanh":
            self.max = 1.7159
            self.min = -1.7159
        else:
            raise error.BadFormatError("Unsupported type %s" % self.type)

        d = self.max - self.min
        if not d:
            return
        self.output.map_write()
        self.input.map_read()
        d = (self.bars - 1) / d
        if self.reset_flag:
            self.output.mem[:] = 0
        self.n_bars[0] = self.bars + 2
        for y in self.input.mem.ravel():
            if y < self.min:
                self.output[0] += 1
                continue
            if y <= self.max and y > self.min:
                i = int(numpy.floor((y - self.min) * d))
                self.output[i] += 1
                continue
            self.output[self.bars + 1] += 1

 def __init__(self, workflow, **kwargs):
     super(FixAccumulator, self).__init__(workflow)
     self.bars = kwargs.get("bars", 200)
     self.type = kwargs.get("type", "relu")
     self.input = None
     self.output = Array()
     self.reset_flag = Bool(True)
     self.n_bars = [0]
     self.max = 100
     self.min = 0

 def __init__(self, workflow, **kwargs):
     super(Deconv, self).__init__(workflow, **kwargs)
     self.unsafe_padding = kwargs.get("unsafe_padding", False)
     self.hits = Array()
     self.krn_clear_output_ = None
     self._global_size = None
     self._local_size = None
     del self.bias
     self.demand("n_kernels", "kx", "ky", "padding", "sliding",
                 "input", "weights", "output_shape_source")

class GDSummator(AcceleratedUnit):
    """Gradient descent for Summator.
    """
    def __init__(self, workflow, **kwargs):
        super(GDSummator, self).__init__(workflow, **kwargs)
        self.err_x = Array()
        self.err_y = Array()
        self.demand("err_output")

    def initialize(self, device, **kwargs):
        super(GDSummator, self).initialize(device, **kwargs)

        if self.err_x:
            assert self.err_x.shape[1:] == self.err_output.shape[1:]
        if not self.err_x or self.err_x.shape[0] != self.err_output.shape[0]:
            self.err_x.reset(numpy.zeros_like(self.err_output.mem))
        if self.err_y:
            assert self.err_y.shape[1:] == self.err_output.shape[1:]
        if not self.err_y or self.err_y.shape[0] != self.err_output.shape[0]:
            self.err_y.reset(numpy.zeros_like(self.err_output.mem))
        self.init_vectors(self.err_x, self.err_y, self.err_output)

    def cuda_init(self):
        pass  # nothing to init

    def ocl_init(self):
        pass  # nothing to init

    def numpy_init(self):
        pass  # nothing to init

    def cuda_run(self):
        self.unmap_vectors(self.err_output, self.err_x, self.err_y)
        self.err_x.devmem.from_device_async(self.err_output.devmem)
        self.err_y.devmem.from_device_async(self.err_output.devmem)

    def ocl_run(self):
        self.unmap_vectors(self.err_output, self.err_x, self.err_y)
        self.device.queue_.copy_buffer(
            self.err_output.devmem, self.err_x.devmem, 0, 0,
            self.err_output.nbytes, need_event=False)
        self.device.queue_.copy_buffer(
            self.err_output.devmem, self.err_y.devmem, 0, 0,
            self.err_output.nbytes, need_event=False)

    def numpy_run(self):
        self.err_output.map_read()
        self.err_x.map_invalidate()
        self.err_y.map_invalidate()
        self.err_x.mem[:] = self.err_output.mem[:]
        self.err_y.mem[:] = self.err_output.mem[:]

    def __init__(self, workflow, **kwargs):
        kwargs["view_group"] = kwargs.get("view_group", "TRAINER")
        super(GradientDescentBase, self).__init__(workflow, **kwargs)
        self.err_input = Array(shallow_pickle=True)
        self.ocl_set_const_args = True
        self.weights = None
        self.bias = None
        self.demand("input", "err_output")
        self.learning_rate = kwargs.get("learning_rate", 0.01)
        self.learning_rate_bias = kwargs.get("learning_rate_bias",
                                             self.learning_rate)
        self.weights_decay = kwargs.get("weights_decay", 0.00005)
        self.weights_decay_bias = kwargs.get("weights_decay_bias", 0.0)
        self.l1_vs_l2 = kwargs.get("l1_vs_l2", 0)
        self.l1_vs_l2_bias = kwargs.get("l1_vs_l2_bias", self.l1_vs_l2)
        self.gradient_moment = kwargs.get("gradient_moment", 0)
        self.gradient_moment_bias = kwargs.get("gradient_moment_bias",
                                               self.gradient_moment)
        self.weights_transposed = kwargs.get("weights_transposed", False)
        self.need_err_input = kwargs.get("need_err_input", True)
        self.include_bias = kwargs.get("include_bias", True)
        self.factor_ortho = kwargs.get("factor_ortho", 0)
        self.col_sums = Array()  # for orthogonalization

        # Current gradient as it is without applying learning_rate etc.
        self.gradient_weights = Array()
        self.gradient_bias = Array()

        # Gradient with applied learning_rate etc.
        # optionally accumulated from the previous run
        self.accumulated_gradient_weights = Array()
        self.accumulated_gradient_bias = Array()

        # Gradient with accumulated moments
        self.gradient_weights_with_moment = Array()
        self.gradient_bias_with_moment = Array()

        # Sets to True when gradient changes
        self.gradient_changed = False

        # Gradient will be applied to weights immediately just after computing
        self.apply_gradient = kwargs.get("apply_gradient",
                                         not workflow.is_slave)

        # Accumulates gradient from the previous run:
        # OP_NONE: do not allocate array at all
        # OP_STORE: stores gradient with an applied learning_rate etc.
        # OP_ADD: adds current gradient to the array
        # OP_FLUSH: applies accumulated gradient, then resets it to zero
        self.accumulate_gradient = kwargs.get("accumulate_gradient",
                                              self.OP_NONE)

class MemCpy(AcceleratedUnit):
    def __init__(self, workflow, **kwargs):
        super(MemCpy, self).__init__(workflow, **kwargs)
        self.output = Array()
        self.demand("input")

    def initialize(self, device, **kwargs):
        super(MemCpy, self).initialize(device, **kwargs)
        if (self.output.mem is None or
                self.output.mem.size != self.input.mem.size):
            self.output.reset()
            self.output.mem = numpy.zeros(self.input.mem.shape,
                                          dtype=self.input.mem.dtype)
        self.input.initialize(self.device)
        self.output.initialize(self.device)

    def cuda_init(self):
        pass

    def ocl_init(self):
        pass

    def _gpu_run(self):
        self.input.unmap()
        self.output.unmap()

    def ocl_run(self):
        self._gpu_run()
        self.device.queue_.copy_buffer(self.input.devmem, self.output.devmem,
                                       0, 0, self.input.nbytes)

    def cuda_run(self):
        self._gpu_run()
        self.output.devmem.from_device_async(self.input.devmem)

    def numpy_run(self):
        self.input.map_read()
        self.output.map_invalidate()
        numpy.copyto(self.output.mem, self.input.mem)

 def __init__(self, workflow, **kwargs):
     super(ImageLoader, self).__init__(workflow, **kwargs)
     self.color_space = kwargs.get("color_space", "RGB")
     self._source_dtype = numpy.float32
     self._original_shape = tuple()
     self.class_keys = [[], [], []]
     self.verify_interface(IImageLoader)
     self.path_to_mean = kwargs.get("path_to_mean", None)
     self.add_sobel = kwargs.get("add_sobel", False)
     self.mirror = kwargs.get("mirror", False)  # True, False, "random"
     self.scale = kwargs.get("scale", 1.0)
     self.scale_maintain_aspect_ratio = kwargs.get(
         "scale_maintain_aspect_ratio", True)
     self.rotations = kwargs.get("rotations", (0.0,))  # radians
     self.crop = kwargs.get("crop", None)
     self.crop_number = kwargs.get("crop_number", 1)
     self._background = None
     self.background_image = kwargs.get("background_image", None)
     self.background_color = kwargs.get(
         "background_color", (0xff, 0x14, 0x93))
     self.smart_crop = kwargs.get("smart_crop", True)
     self.minibatch_label_values = Array()

 def __init__(self, workflow, **kwargs):
     super(All2AllSoftmax, self).__init__(workflow, **kwargs)
     self.max_idx = Array()
     self.reduce_size = 256

class All2AllSoftmax(All2All):
    """All2All with linear activation and softmax normalization.

    Must be assigned before initialize():

    Updates after run():
        max_idx

    Creates within initialize():
        max_idx

    Attributes:
        krn_sm_: kernel for softmax activation calculation.
        max_idx: indexes of element with maximum value for each sample.
    """
    __id__ = "420219fc-3e1a-45b1-87f8-aaa0c1540de4"

    MAPPING = {"softmax"}

    def __init__(self, workflow, **kwargs):
        super(All2AllSoftmax, self).__init__(workflow, **kwargs)
        self.max_idx = Array()
        self.reduce_size = 256

    def init_unpickled(self):
        super(All2AllSoftmax, self).init_unpickled()
        self.krn_sm_ = None
        self._force_gpu_apply_exp = False

    def initialize(self, device, **kwargs):
        self.reduce_size = min(self.reduce_size,
                               int(numpy.prod(self.output_sample_shape)))
        self.sources_["all2all/softmax"] = {
            "REDUCE_SIZE": self.reduce_size
        }
        retval = super(All2AllSoftmax, self).initialize(
            device=device, **kwargs)
        if retval:
            return retval
        if self.output.mem.size // self.output.mem.shape[0] <= 1:
            raise error.BadFormatError(
                "Output sample size should be greater than 1 for SoftMax.")

        if not self.max_idx:
            self.max_idx.reset(numpy.zeros(self.output.shape[0],
                                           dtype=numpy.int32))
        self.max_idx.initialize(self.device)
        return retval

    def numpy_apply_exp(self):
        self.output.map_write()
        self.max_idx.map_invalidate()
        out = self.output.mem
        out = reshape(out, (out.shape[0], out.size // out.shape[0]))
        for i, sample in enumerate(out):
            im = sample.argmax()
            self.max_idx[i] = im
            m = sample[im]
            sample -= m
            numpy.exp(sample, sample)
            smm = sample.sum()
            sample /= smm

    def ocl_apply_exp(self):
        self.unmap_vectors(self.output, self.max_idx)
        global_size = (self.output.shape[0] * self.reduce_size,)
        local_size = (self.reduce_size,)
        self.execute_kernel(global_size, local_size, self.krn_sm_)

    def cuda_apply_exp(self):
        self.unmap_vectors(self.output, self.max_idx)
        global_size = (self.output.shape[0], 1, 1)
        local_size = (self.reduce_size, 1, 1)
        self.execute_kernel(global_size, local_size, self.krn_sm_)

    def numpy_run(self):
        """Forward propagation from batch on CPU only.
        """
        super(All2AllSoftmax, self).numpy_run()
        if not self._force_gpu_apply_exp:
            self.numpy_apply_exp()

    def ocl_run(self):
        """Forward propagation from batch on GPU.
        """
        self._force_gpu_apply_exp = True
        super(All2AllSoftmax, self).ocl_run()
        self.ocl_apply_exp()

    def cuda_run(self):
        """Forward propagation from batch on GPU.
        """
        self._force_gpu_apply_exp = True
        super(All2AllSoftmax, self).cuda_run()
        self.cuda_apply_exp()

    def ocl_init(self):
        super(All2AllSoftmax, self).ocl_init()
        self.krn_sm_ = self.get_kernel("apply_exp")
        self.krn_sm_.set_args(self.output.devmem, self.max_idx.devmem)
#.........这里部分代码省略.........