def expression_iterator(self, layer, expression, geometryStorage):
     featReq = QgsFeatureRequest()
     expression = QgsExpression(expression)
     context = QgsExpressionContext()
     self.stopLoop = False
     i = 0
     for f in layer.getFeatures(featReq):
         QCoreApplication.processEvents()
         if self.stopLoop:
             break
         self.recordingSearchProgress.emit(i)
         i += 1
         context.setFeature(f)
         evaluated = unicode(expression.evaluate(context))
         if expression.hasEvalError():
             continue
         if f.geometry() is None or f.geometry().centroid() is None:
             continue
         centroid = f.geometry().centroid().asPoint()
         if geometryStorage == 'wkb':
             geom = binascii.b2a_hex(f.geometry().asWkb())
         elif geometryStorage == 'wkt':
             geom = f.geometry().exportToWkt()
         elif geometryStorage == 'extent':
             geom = f.geometry().boundingBox().asWktPolygon()
         yield ( evaluated, centroid.x(), centroid.y(), geom )

 def update_variables_gui(self):
     variables_scope = QgsExpressionContextScope(self.tr('Model Variables'))
     for k, v in self.model.variables().items():
         variables_scope.setVariable(k, v)
     variables_context = QgsExpressionContext()
     variables_context.appendScope(variables_scope)
     self.variables_editor.setContext(variables_context)
     self.variables_editor.setEditableScopeIndex(0)

    def processAlgorithm(self, feedback):
        layer = dataobjects.getLayerFromString(
            self.getParameterValue(self.INPUT_LAYER))

        geometry_type = self.getParameterValue(self.OUTPUT_GEOMETRY)
        wkb_type = None
        if geometry_type == 0:
            wkb_type = QgsWkbTypes.Polygon
        elif geometry_type == 1:
            wkb_type = QgsWkbTypes.LineString
        else:
            wkb_type = QgsWkbTypes.Point
        if self.getParameterValue(self.WITH_Z):
            wkb_type = QgsWkbTypes.addZ(wkb_type)
        if self.getParameterValue(self.WITH_M):
            wkb_type = QgsWkbTypes.addM(wkb_type)

        writer = self.getOutputFromName(
            self.OUTPUT_LAYER).getVectorWriter(
                layer.fields(),
                wkb_type,
                layer.crs())

        expression = QgsExpression(self.getParameterValue(self.EXPRESSION))
        if expression.hasParserError():
            raise GeoAlgorithmExecutionException(expression.parserErrorString())

        exp_context = QgsExpressionContext(QgsExpressionContextUtils.globalProjectLayerScopes(layer))

        if not expression.prepare(exp_context):
            raise GeoAlgorithmExecutionException(
                self.tr('Evaluation error: {0}').format(expression.evalErrorString()))

        features = vector.features(layer)
        total = 100.0 / len(features)
        for current, input_feature in enumerate(features):
            output_feature = input_feature

            exp_context.setFeature(input_feature)
            value = expression.evaluate(exp_context)
            if expression.hasEvalError():
                raise GeoAlgorithmExecutionException(
                    self.tr('Evaluation error: {0}').format(expression.evalErrorString()))

            if not value:
                output_feature.setGeometry(QgsGeometry())
            else:
                if not isinstance(value, QgsGeometry):
                    raise GeoAlgorithmExecutionException(
                        self.tr('{} is not a geometry').format(value))
                output_feature.setGeometry(value)

            writer.addFeature(output_feature)
            feedback.setProgress(int(current * total))

        del writer

 def evaluateExpression(self, text):
     context = QgsExpressionContext()
     context.appendScope(QgsExpressionContextUtils.globalScope())
     context.appendScope(QgsExpressionContextUtils.projectScope())
     exp = QgsExpression(text)
     if exp.hasParserError():
         raise Exception(exp.parserErrorString())
     result = exp.evaluate(context)
     if exp.hasEvalError():
         raise ValueError(exp.evalErrorString())
     return result

    def check_for_update_events(self, widget, value):
        if not self.feature:
            return

        from qgis.core import QgsExpression, QgsExpressionContext, QgsExpressionContextScope
        # If we don't have any events for this widgets just get out now
        if not widget.id in self.events:
            return

        events = self.events[widget.id]
        events = [event for event in events if event['event'].lower() == 'update']
        if not events:
            return

        feature = self.to_feature(no_defaults=True)

        for event in events:
            action = event['action'].lower()
            targetid = event['target']
            if targetid == widget.id:
                utils.log("Can't connect events to the same widget. ID {}".format(targetid))
                continue

            widget = self.get_widget_from_id(targetid)

            if not widget:
                utils.log("Can't find widget for id {} in form".format(targetid))
                continue

            condition = event['condition']
            expression = event['value']

            context = QgsExpressionContext()
            scope = QgsExpressionContextScope()
            scope.setVariable("value", value)
            scope.setVariable("field", widget.field)
            context.setFeature(feature)
            context.appendScope(scope)

            conditionexp = QgsExpression(condition)
            exp = QgsExpression(expression)

            if action.lower() == "show":
                widget.hidden = not conditionexp.evaluate(context)
            if action.lower() == "hide":
                widget.hidden = conditionexp.evaluate(context)
            if action == 'widget expression':
                if conditionexp.evaluate(context):
                    newvalue = self.widget_default(field, feature=feature)
                    widget.setvalue(newvalue)
            if action == 'set value':
                if conditionexp.evaluate(context):
                    newvalue = exp.evaluate(context)
                    widget.setvalue(newvalue)

def sum_fields(layer, output_field_key, input_fields):
    """Sum the value of input_fields and put it as output_field.

    :param layer: The vector layer.
    :type layer: QgsVectorLayer

    :param output_field_key: The output field definition key.
    :type output_field_key: basestring

    :param input_fields: List of input fields' name.
    :type input_fields: list
    """
    field_definition = definition(output_field_key)
    output_field_name = field_definition['field_name']
    # If the fields only has one element
    if len(input_fields) == 1:
        # Name is different, copy it
        if input_fields[0] != output_field_name:
            to_rename = {input_fields[0]: output_field_name}
            # We copy only, it will be deleted later.
            # We can't rename the field, we need to copy it as the same
            # field might be used many times in the FMT tool.
            copy_fields(layer, to_rename)
        else:
            # Name is same, do nothing
            return
    else:
        # Creating expression
        # Put field name in a double quote. See #4248
        input_fields = ['"%s"' % f for f in input_fields]
        string_expression = ' + '.join(input_fields)
        sum_expression = QgsExpression(string_expression)
        context = QgsExpressionContext()
        context.setFields(layer.fields())
        sum_expression.prepare(context)

        # Get the output field index
        output_idx = layer.fields().lookupField(output_field_name)
        # Output index is not found
        layer.startEditing()
        if output_idx == -1:
            output_field = create_field_from_definition(field_definition)
            layer.addAttribute(output_field)
            output_idx = layer.fields().lookupField(output_field_name)

        # Iterate to all features
        for feature in layer.getFeatures():
            context.setFeature(feature)
            result = sum_expression.evaluate(context)
            feature[output_idx] = result
            layer.updateFeature(feature)

        layer.commitChanges()

def sum_fields(layer, output_field_key, input_fields):
    """Sum the value of input_fields and put it as output_field.

    :param layer: The vector layer.
    :type layer: QgsVectorLayer

    :param output_field_key: The output field definition key.
    :type output_field_key: basestring

    :param input_fields: List of input fields' name.
    :type input_fields: list
    """
    field_definition = definition(output_field_key)
    output_field_name = field_definition['field_name']
    # If the fields only has one element
    if len(input_fields) == 1:
        # Name is different, copy it
        if input_fields[0] != output_field_name:
            copy_fields(layer, {
                input_fields[0]: output_field_name})
        # Name is same, do nothing
        else:
            return
    else:
        # Creating expression
        # Put field name in a double quote. See #4248
        input_fields = ['"%s"' % f for f in input_fields]
        string_expression = ' + '.join(input_fields)
        sum_expression = QgsExpression(string_expression)
        context = QgsExpressionContext()
        context.setFields(layer.pendingFields())
        sum_expression.prepare(context)

        # Get the output field index
        output_idx = layer.fieldNameIndex(output_field_name)
        # Output index is not found
        if output_idx == -1:
            output_field = create_field_from_definition(field_definition)
            layer.startEditing()
            layer.addAttribute(output_field)
            layer.commitChanges()
            output_idx = layer.fieldNameIndex(output_field_name)

        layer.startEditing()
        # Iterate to all features
        for feature in layer.getFeatures():
            context.setFeature(feature)
            result = sum_expression.evaluate(context)
            feature[output_idx] = result
            layer.updateFeature(feature)

        layer.commitChanges()

    def add_flooded_field(self, shapefile_path):
        """Create the layer from the local shp adding the flooded field.

        .. versionadded:: 3.3

        Use this method to add a calculated field to a shapefile. The shapefile
        should have a field called 'count' containing the number of flood
        reports for the field. The field values will be set to 0 if the count
        field is < 1, otherwise it will be set to 1.

        :param shapefile_path: Path to the shapefile that will have the flooded
            field added.
        :type shapefile_path: basestring

        :return: A vector layer with the flooded field added.
        :rtype: QgsVectorLayer
        """
        layer = QgsVectorLayer(
            shapefile_path, self.tr('Jakarta Floods'), 'ogr')
        # Add a calculated field indicating if a poly is flooded or not
        # from qgis.PyQt.QtCore import QVariant
        layer.startEditing()
        # Add field with integer from 0 to 4 which represents the flood
        # class. Its the same as 'state' field except that is being treated
        # as a string.
        # This is used for cartography
        flood_class_field = QgsField('floodclass', QVariant.Int)
        layer.addAttribute(flood_class_field)
        layer.commitChanges()
        layer.startEditing()
        flood_class_idx = layer.fields().lookupField('floodclass')
        flood_class_expression = QgsExpression('to_int(state)')
        context = QgsExpressionContext()
        context.setFields(layer.fields())
        flood_class_expression.prepare(context)

        # Add field with boolean flag to say if the area is flooded
        # This is used by the impact function
        flooded_field = QgsField('flooded', QVariant.Int)
        layer.dataProvider().addAttributes([flooded_field])
        layer.commitChanges()
        layer.startEditing()
        flooded_idx = layer.fields().lookupField('flooded')
        flood_flag_expression = QgsExpression('state > 0')
        flood_flag_expression.prepare(context)
        for feature in layer.getFeatures():
            context.setFeature(feature)
            feature[flood_class_idx] = flood_class_expression.evaluate(context)
            feature[flooded_idx] = flood_flag_expression.evaluate(context)
            layer.updateFeature(feature)
        layer.commitChanges()
        return layer

    def test_length_expression(self):
        # compare length using the ellipsoid in kms and the planimetric distance in meters
        self.lyr.fieldName = "round($length,5) || ' - ' || round(length($geometry),2)"
        self.lyr.isExpression = True

        QgsProject.instance().setCrs(QgsCoordinateReferenceSystem("EPSG:32613"))
        QgsProject.instance().setEllipsoid("WGS84")
        QgsProject.instance().setDistanceUnits(QgsUnitTypes.DistanceKilometers)

        ctxt = QgsExpressionContext()
        ctxt.appendScope(QgsExpressionContextUtils.projectScope(QgsProject.instance()))
        ctxt.appendScope(QgsExpressionContextUtils.layerScope(self.layer))
        self._TestMapSettings.setExpressionContext(ctxt)

        self.lyr.placement = QgsPalLayerSettings.Curved
        self.lyr.placementFlags = QgsPalLayerSettings.AboveLine | QgsPalLayerSettings.MapOrientation
        self.checkTest()

def add25dAttributes(cleanLayer, layer, canvas):
    provider = cleanLayer.dataProvider()
    provider.addAttributes([QgsField("height", QVariant.Double),
                            QgsField("wallColor", QVariant.String),
                            QgsField("roofColor", QVariant.String)])
    cleanLayer.updateFields()
    fields = cleanLayer.fields()
    renderer = layer.renderer()
    renderContext = QgsRenderContext.fromMapSettings(canvas.mapSettings())
    feats = layer.getFeatures()
    context = QgsExpressionContext()
    context.appendScope(QgsExpressionContextUtils.layerScope(layer))
    expression = QgsExpression('eval(@qgis_25d_height)')
    heightField = fields.indexFromName("height")
    wallField = fields.indexFromName("wallColor")
    roofField = fields.indexFromName("roofColor")
    renderer.startRender(renderContext, fields)
    cleanLayer.startEditing()
    for feat in feats:
        context.setFeature(feat)
        height = expression.evaluate(context)
        if isinstance(renderer, QgsCategorizedSymbolRenderer):
            classAttribute = renderer.classAttribute()
            attrValue = feat.attribute(classAttribute)
            catIndex = renderer.categoryIndexForValue(attrValue)
            categories = renderer.categories()
            symbol = categories[catIndex].symbol()
        elif isinstance(renderer, QgsGraduatedSymbolRenderer):
            classAttribute = renderer.classAttribute()
            attrValue = feat.attribute(classAttribute)
            ranges = renderer.ranges()
            for range in ranges:
                if (attrValue >= range.lowerValue() and
                        attrValue <= range.upperValue()):
                    symbol = range.symbol().clone()
        else:
            symbol = renderer.symbolForFeature(feat, renderContext)
        sl1 = symbol.symbolLayer(1)
        sl2 = symbol.symbolLayer(2)
        wallColor = sl1.subSymbol().color().name()
        roofColor = sl2.subSymbol().color().name()
        provider.changeAttributeValues({feat.id() + 1: {heightField: height,
                                                        wallField: wallColor,
                                                        roofField: roofColor}})
    cleanLayer.commitChanges()
    renderer.stopRender(renderContext)

    def processAlgorithm(self, progress):
        ns = {}
        ns["progress"] = progress
        ns["scriptDescriptionFile"] = self.descriptionFile

        for param in self.parameters:
            ns[param.name] = param.value

        for out in self.outputs:
            ns[out.name] = out.value

        variables = re.findall("@[a-zA-Z0-9_]*", self.script)
        script = "import processing\n"
        script += self.script

        context = QgsExpressionContext()
        context.appendScope(QgsExpressionContextUtils.globalScope())
        context.appendScope(QgsExpressionContextUtils.projectScope())
        for var in variables:
            varname = var[1:]
            if context.hasVariable(varname):
                script = script.replace(var, context.variable(varname))
            else:
                ProcessingLog.addToLog(ProcessingLog.LOG_WARNING, "Cannot find variable: %s" % varname)

        exec((script), ns)
        for out in self.outputs:
            out.setValue(ns[out.name])

    def processAlgorithm(self, progress):
        layer = layer = dataobjects.getObjectFromUri(self.getParameterValue(self.INPUT))

        expression = self.getParameterValue(self.EXPRESSION)
        qExp = QgsExpression(expression)
        if qExp.hasParserError():
            raise GeoAlgorithmExecutionException(qExp.parserErrorString())

        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
            layer.fields(), layer.wkbType(), layer.crs())

        context = QgsExpressionContext()
        context.appendScope(QgsExpressionContextUtils.globalScope())
        context.appendScope(QgsExpressionContextUtils.projectScope())
        context.appendScope(QgsExpressionContextUtils.layerScope(layer))

        count = layer.featureCount()
        step = 100.0 / count if count else 1

        request = QgsFeatureRequest(qExp, context)

        for current, f in enumerate(layer.getFeatures(request)):
            writer.addFeature(f)
            progress.setPercentage(int(current * step))

        del writer

    def processAlgorithm(self, parameters, context, feedback):
        ns = {}
        ns['feedback'] = feedback
        ns['scriptDescriptionFile'] = self.descriptionFile
        ns['context'] = context

        for param in self.parameterDefinitions():
            ns[param.name] = parameters[param.name()]

        for out in self.outputs:
            ns[out.name] = out.value

        variables = re.findall('@[a-zA-Z0-9_]*', self.script)
        script = 'import processing\n'
        script += self.script

        context = QgsExpressionContext()
        context.appendScope(QgsExpressionContextUtils.globalScope())
        context.appendScope(QgsExpressionContextUtils.projectScope(QgsProject.instance()))
        for var in variables:
            varname = var[1:]
            if context.hasVariable(varname):
                script = script.replace(var, context.variable(varname))
            else:
                QgsMessageLog.logMessage(self.tr('Cannot find variable: {0}').format(varname), self.tr('Processing'), QgsMessageLog.WARNING)

        exec((script), ns)
        for out in self.outputs:
            out.setValue(ns[out.name])

 def expressionContext(self):
     context = QgsExpressionContext()
     context.appendScope(QgsExpressionContextUtils.globalScope())
     context.appendScope(QgsExpressionContextUtils.projectScope())
     processingScope = QgsExpressionContextScope()
     for param in self.alg.parameters:
         processingScope.setVariable("%s_value" % param.name, "")
     context.appendScope(processingScope)
     return context

def _expressionContext(alg):
    context = QgsExpressionContext()
    context.appendScope(QgsExpressionContextUtils.globalScope())
    context.appendScope(QgsExpressionContextUtils.projectScope())
    processingScope = QgsExpressionContextScope()
    for param in alg.parameters:
        processingScope.setVariable('%s_value' % param.name, '')
    context.appendScope(processingScope)
    return context

def createExpressionContext():
    context = QgsExpressionContext()
    context.appendScope(QgsExpressionContextUtils.globalScope())
    context.appendScope(QgsExpressionContextUtils.projectScope(QgsProject.instance()))

    if iface.mapCanvas():
        context.appendScope(QgsExpressionContextUtils.mapSettingsScope(iface.mapCanvas().mapSettings()))

    processingScope = QgsExpressionContextScope()

    extent = iface.mapCanvas().fullExtent()
    processingScope.setVariable('fullextent_minx', extent.xMinimum())
    processingScope.setVariable('fullextent_miny', extent.yMinimum())
    processingScope.setVariable('fullextent_maxx', extent.xMaximum())
    processingScope.setVariable('fullextent_maxy', extent.yMaximum())
    context.appendScope(processingScope)
    return context

    def testConcurrency(self):
        """
        The connection pool has a maximum of 4 connections defined (+2 spare connections)
        Make sure that if we exhaust those 4 connections and force another connection
        it is actually using the spare connections and does not freeze.
        This situation normally happens when (at least) 4 rendering threads are active
        in parallel and one requires an expression to be evaluated.
        """
        # Acquire the maximum amount of concurrent connections
        iterators = list()
        for i in range(QgsApplication.instance().maxConcurrentConnectionsPerPool()):
            iterators.append(self.vl.getFeatures())

        # Run an expression that will also do a request and should use a spare
        # connection. It just should not deadlock here.

        feat = next(iterators[0])
        context = QgsExpressionContext()
        context.setFeature(feat)
        exp = QgsExpression('get_feature(\'{layer}\', \'pk\', 5)'.format(layer=self.vl.id()))
        exp.evaluate(context)

    def pointPicked(self, event):
        for b in self.rubberBands:
            del b
        self.rubberBands[:] = []

        fieldX = self.cmbXField.currentField()
        fieldY = self.cmbYField.currentField()

        artist = event.artist
        indices = event.ind
        for i in indices:
            x = self.xData[artist.name][i]
            y = self.yData[artist.name][i]

            if isinstance(x, int):
                expr = '"{}" = {} AND '.format(fieldX, x)
            elif isinstance(x, float):
                expr = 'abs("{}" - {}) <= 0.0000001 AND '.format(fieldX, x)
            elif isinstance(x, (str, unicode)):
                expr = """"{}" = '{}' AND """.format(fieldX, x)
            else:
                expr = """"{}" = '{}' AND """.format(fieldX, x.toString('yyyy-MM-dd'))

            if isinstance(y, float):
                expr += 'abs("{}" - {}) <= 0.0000001'.format(fieldY, y)
            elif isinstance(y, (str, unicode)):
                expr += """"{}" = '{}'""".format(fieldY, y)
            else:
                expr += '"{}" = {}'.format(fieldY, y)

            layer = self.cmbLayer.currentLayer()
            expression = QgsExpression(expr)
            context = QgsExpressionContext()
            context.appendScope(QgsExpressionContextUtils.globalScope())
            context.appendScope(QgsExpressionContextUtils.projectScope())
            context.appendScope(QgsExpressionContextUtils.mapSettingsScope(self.canvas.mapSettings()))
            context.appendScope(QgsExpressionContextUtils.layerScope(layer))

            request = QgsFeatureRequest(expression, context)
            for f in layer.getFeatures(request):
                hl = QgsHighlight(self.canvas, f.geometry(), layer)
                hl.setColor(QColor(255, 0, 0))
                hl.setWidth(2)
                self.rubberBands.append(hl)

    def __init__(self, provider):
        super(PyFeatureSource, self).__init__()
        self._provider = provider
        self._features = provider._features

        self._expression_context = QgsExpressionContext()
        self._expression_context.appendScope(QgsExpressionContextUtils.globalScope())
        self._expression_context.appendScope(QgsExpressionContextUtils.projectScope(QgsProject.instance()))
        self._expression_context.setFields(self._provider.fields())
        if self._provider.subsetString():
            self._subset_expression = QgsExpression(self._provider.subsetString())
            self._subset_expression.prepare(self._expression_context)
        else:
            self._subset_expression = None

    def testVariables(self):
        """ check through widget model to ensure it is populated with variables """
        w = QgsExpressionBuilderWidget()
        m = w.model()

        s = QgsExpressionContextScope()
        s.setVariable('my_var1', 'x')
        s.setVariable('my_var2', 'y')
        c = QgsExpressionContext()
        c.appendScope(s)

        # check that variables are added when setting context
        w.setExpressionContext(c)
        items = m.findItems('my_var1', Qt.MatchRecursive)
        self.assertEqual(len(items), 1)
        items = m.findItems('my_var2', Qt.MatchRecursive)
        self.assertEqual(len(items), 1)
        items = m.findItems('not_my_var', Qt.MatchRecursive)
        self.assertEqual(len(items), 0)
        # double check that functions are still only there once
        items = m.findItems('lower', Qt.MatchRecursive)
        self.assertEqual(len(items), 1)
        items = m.findItems('upper', Qt.MatchRecursive)
        self.assertEqual(len(items), 1)