def __calculate_statistics(self, thema, subthema, layers):
     #features = processing.features(self.coverage_layer)
     features = self.coverage_layer.selectedFeatures()
     for gstk in features:
         try:
             gnr = gstk[self.settings.fld_gnr()]
             flaeche = gstk.geometry().area()
             gstk_stats = VRPStatistik(gnr, thema.name, flaeche, self.gem_name)
             #pyqtRemoveInputHook()
             #pdb.set_trace()
             #go thru all data sources of subthema
             for quelle in subthema.quellen:
                 if VRP_DEBUG is True: QgsMessageLog.logMessage(u'quelle:{0}'.format(quelle.name), DLG_CAPTION)
                 #only use those with statistik == True
                 if quelle.statistik is False:
                     continue
                 lyr_curr_quelle = None
                 for lyr in layers:
                     if quelle.name == lyr.name():
                         lyr_curr_quelle = lyr
                 if lyr_curr_quelle is None:
                     continue
                 text_flaeche = self.__get_text_flaeche(quelle, gstk, lyr_curr_quelle, quelle.attribut)
                 sub_stat = VRPStatistikSubThema(quelle.name, text_flaeche)
                 gstk_stats.add_subthema(thema.name, sub_stat)
             if gnr in self.statistics:
                 self.statistics[gnr].append(gstk_stats)
             else:
                 self.statistics[gnr] = [gstk_stats]
         except:
             msg = '__calculate_statistics:\n\n{0}'.format(traceback.format_exc())
             QgsMessageLog.logMessage(msg, DLG_CAPTION, QgsMessageLog.CRITICAL)
             msg = msg.replace('\n', '')
             self.iface.messageBar().pushMessage(msg,  QgsMessageBar.CRITICAL)
             return

    def copy(self, destinationFolder):
        """
        Copy inventoried files considering the dataset
        destinationFolder: copy destination folder
        """
        for fileName in self.files:
            # adjusting the separators according to the OS
            fileName = fileName.replace('/', os.sep)
            file = fileName.split(os.sep)[-1]
            newFileName = os.path.join(destinationFolder, file)
            newFileName = newFileName.replace('/', os.sep)

            try:
                gdalSrc = gdal.Open(fileName)
                ogrSrc = ogr.Open(fileName)
                if ogrSrc:
                    self.copyOGRDataSource(ogrSrc, newFileName)
                elif gdalSrc:
                    self.copyGDALDataSource(gdalSrc, newFileName)
            except Exception as e:
                QgsMessageLog.logMessage(self.messenger.getCopyErrorMessage()+'\n'+':'.join(e.args), "DSG Tools Plugin", QgsMessageLog.INFO)
                return (0, self.messenger.getCopyErrorMessage()+'\n'+':'.join(e.args))
        
        QgsMessageLog.logMessage(self.messenger.getSuccessInventoryAndCopyMessage(), "DSG Tools Plugin", QgsMessageLog.INFO)
        return (1, self.messenger.getSuccessInventoryAndCopyMessage())

    def __init__(self, alg):
        super(HelpEditionDialog, self).__init__(None)
        self.setupUi(self)

        self.alg = alg
        self.descriptions = {}
        if isinstance(self.alg, QgsProcessingModelAlgorithm):
            self.descriptions = self.alg.helpContent()
        else:
            if self.alg.descriptionFile is not None:
                helpfile = alg.descriptionFile + '.help'
                if os.path.exists(helpfile):
                    try:
                        with open(helpfile) as f:
                            self.descriptions = json.load(f)
                    except Exception:
                        QgsMessageLog.logMessage(self.tr('Cannot open help file: {0}').format(helpfile), self.tr('Processing'), Qgis.Warning)

        self.currentName = self.ALG_DESC
        if self.ALG_DESC in self.descriptions:
            self.text.setText(self.descriptions[self.ALG_DESC])
        self.tree.itemClicked.connect(self.changeItem)

        self.fillTree()
        self.updateHtmlView()

    def testBlocker(self):
        app_log = QgsApplication.messageLog()

        spy = QSignalSpy(app_log.messageReceived)
        spy_received = QSignalSpy(app_log.messageReceived[bool])

        QgsMessageLog.logMessage('test', 'tag', Qgis.Warning, notifyUser=True)
        self.assertEqual(len(spy), 1)
        self.assertEqual(spy[-1], ['test', 'tag', Qgis.Warning])
        self.assertEqual(len(spy_received), 1)

        # block notifications
        b = QgsMessageLogNotifyBlocker()
        QgsMessageLog.logMessage('test', 'tag', Qgis.Warning, notifyUser=True)
        self.assertEqual(len(spy), 2) # should not be blocked
        self.assertEqual(len(spy_received), 1) # should be blocked

        # another blocker
        b2 = QgsMessageLogNotifyBlocker()
        QgsMessageLog.logMessage('test', 'tag', Qgis.Warning, notifyUser=True)
        self.assertEqual(len(spy), 3) # should not be blocked
        self.assertEqual(len(spy_received), 1) # should be blocked

        del b
        # still blocked because of b2
        QgsMessageLog.logMessage('test', 'tag', Qgis.Warning, notifyUser=True)
        self.assertEqual(len(spy), 4) # should not be blocked
        self.assertEqual(len(spy_received), 1) # should be blocked

        del b2
        # not blocked
        QgsMessageLog.logMessage('test', 'tag', Qgis.Warning, notifyUser=True)
        self.assertEqual(len(spy), 5) # should not be blocked
        self.assertEqual(len(spy_received), 2) # should not be blocked

 def killWorker(self):
     """Kill the worker thread."""
     if self.worker is not None:
         QgsMessageLog.logMessage(self.tr('Killing worker'),
                                  self.SDELLIPSE,
                                  Qgis.Info)
         self.worker.kill()

    def processAlgorithm(self, parameters, context, feedback):
        layerA = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(Difference.INPUT), context)
        layerB = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(Difference.OVERLAY), context)

        geomType = QgsWkbTypes.multiType(layerA.wkbType())
        writer = self.getOutputFromName(
            Difference.OUTPUT).getVectorWriter(layerA.fields(), geomType, layerA.crs(), context)

        outFeat = QgsFeature()
        index = QgsProcessingUtils.createSpatialIndex(layerB, context)
        selectionA = QgsProcessingUtils.getFeatures(layerA, context)
        total = 100.0 / layerA.featureCount() if layerA.featureCount() else 0
        for current, inFeatA in enumerate(selectionA):
            geom = inFeatA.geometry()
            diff_geom = QgsGeometry(geom)
            attrs = inFeatA.attributes()
            intersections = index.intersects(geom.boundingBox())

            request = QgsFeatureRequest().setFilterFids(intersections).setSubsetOfAttributes([])
            for inFeatB in layerB.getFeatures(request):
                tmpGeom = inFeatB.geometry()
                if diff_geom.intersects(tmpGeom):
                    diff_geom = QgsGeometry(diff_geom.difference(tmpGeom))

            try:
                outFeat.setGeometry(diff_geom)
                outFeat.setAttributes(attrs)
                writer.addFeature(outFeat, QgsFeatureSink.FastInsert)
            except:
                QgsMessageLog.logMessage(self.tr('Feature geometry error: One or more output features ignored due to invalid geometry.'), self.tr('Processing'), QgsMessageLog.WARNING)
                continue

            feedback.setProgress(int(current * total))

        del writer

 def validateFile(self,filename):
     # check that file exists
     if not os.path.exists(filename):
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "The file %s does not exist" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # check that file is an image
     if imghdr.what(filename) is None:
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "The file %s is not a supported data source" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # check if gdal can read file
     try:
         raster = gdal.Open(filename,gdal.GA_ReadOnly)
     except:
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "GDAL could not read file %s" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # check if there is an object raster
     if not raster:
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "GDAL could not read file %s" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # check that image has at least 3 bands
     if raster.RasterCount < 3:
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "The file %s has less than 3 raster bands" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # check if projection is set
     if raster.GetProjection() is '':
         self.bar0.clearWidgets()
         self.bar0.pushMessage(
             "CRITICAL",
             "Could not extract projection from file %s" % filename,
             level=QgsMessageBar.CRITICAL)
         return False
     # finally, check if bbox has valid data
     xy_points = [(0.0,0.0),(0.0,raster.RasterYSize),(raster.RasterXSize,0.0),(raster.RasterXSize,raster.RasterYSize)]
     for point in xy_points:
         if point != self.GDALInfoReportCorner(raster,point[0],point[1]):
             QgsMessageLog.logMessage(
                 'File %s is a valid data source' % filename,
                 'OAM',
                 level=QgsMessageLog.INFO)
             return True

def loadLessonsFromPaths(paths):
    hasErrors = False
    for path in paths:
        for folder in os.listdir(path):
            if os.path.isdir(os.path.join(path, folder)):
                groupFiles = [os.path.join(path, folder, f) for f in ["group.html", "group.md"]]
                for groupFile in groupFiles:
                    if os.path.exists(groupFile):
                        groups[folder.replace("_", " ")] = groupFile
                        break

                for subfolder in os.listdir(os.path.join(path, folder)):
                    if os.path.isdir(os.path.join(path, folder, subfolder)):
                        try:
                            f = os.path.join(path, folder, subfolder, "__init__.py")
                            if os.path.exists(f):
                                m = imp.load_source("{}.{}".format(folder, subfolder), f)
                                addLessonModule(m)
                        except Exception as e:
                            QgsMessageLog.logMessage("Can not load lesson from {}:\n{}".format(f, str(e)), "Lessons")
                            hasErrors = True

                        if isYamlLessonFolder(os.path.join(path, folder), subfolder):
                            lesson = lessonFromYamlFile(os.path.join(path, folder, subfolder, "lesson.yaml"))
                            if lesson:
                                _addLesson(lesson)
                            else:
                                hasErrors = True
    return hasErrors

    def unserialize(self, st):
        style = None
        orig_geom = None
        geom = None

        try:
            t = pickle.loads(st)
        except Exception as e:
            try:
                t = pickle.loads(st, encoding='utf-8')
                # strange, Exception says : No module named 'PyQt4'
            except Exception as e:
                for m in e.args:
                    QgsMessageLog.logMessage(str(m), 'Extensions')

                raise Exception("Mask - Error when loading mask")

        if len(t) == 12:  # older version
            (self.do_buffer,
             self.buffer_units,
             self.buffer_segments,
             self.do_simplify,
             self.simplify_tolerance,
             self.do_save_as,
             self.file_path,
             self.file_format,
             self.limited_layers_obsolete,
             style,
             self.polygon_mask_method,
             self.line_mask_method) = t
        else:
            (self.do_buffer,
             self.buffer_units,
             self.buffer_segments,
             self.do_simplify,
             self.simplify_tolerance,
             self.do_save_as,
             self.file_path,
             self.file_format,
             self.limited_layers_obsolete,
             style,
             self.polygon_mask_method,
             self.line_mask_method,
             orig_geom,
             geom
             ) = t
        self.style = None
        self.geometry = None
        if style is not None:
            self.style = style
        if geom is not None:
            self.geometry = QgsGeometry()
            self.geometry.fromWkb(geom)
        if orig_geom is not None:
            gl = []
            for g in orig_geom:
                geo = QgsGeometry()
                geo.fromWkb(g)
                gl.append(geo)
            self.orig_geometry = gl

 def showWarning(self, text):
     """Show a warning."""
     self.iface.messageBar().pushMessage(self.tr('Warning'), text,
                                         level=QgsMessageBar.WARNING,
                                         duration=2)
     QgsMessageLog.logMessage('Warning: ' + text, self.THINGREYSCALE,
                              QgsMessageLog.WARNING)

 def showInfo(self, text):
     """Show info."""
     self.iface.messageBar().pushMessage(self.tr('Info'), text,
                                         level=Qgis.Info,
                                         duration=2)
     QgsMessageLog.logMessage('Info: ' + text, self.THINGREYSCALE,
                              Qgis.Info)

 def showError(self, text):
     """Show an error."""
     self.iface.messageBar().pushMessage(self.tr('Error'), text,
                                         level=QgsMessageBar.CRITICAL,
                                         duration=3)
     QgsMessageLog.logMessage('Error: ' + text, self.THINGREYSCALE,
                              QgsMessageLog.CRITICAL)

 def do_aoi_search(self, order_params, csv_element):
     """
     Performs an AOI search for strips in GBD and generates stats from them
     :param order_params: The order params for the GBD query
     :param csv_element: The entry in the CSV row to update
     :return:
     """
     data = {
         KEY_DATA_SEARCH_AREA_WKT: order_params.polygon,
         KEY_DATA_START_DATE: order_params.time_begin.isoformat() + "Z",
         KEY_DATA_END_DATE: order_params.time_end.isoformat() + "Z",
         KEY_DATA_FILTERS: VALUE_DATA_FILTERS,
         KEY_DATA_TAG_RESULTS: VALUE_DATA_TAG_RESULTS,
         KEY_DATA_TYPES: VALUE_DATA_TYPES,
     }
     json_data = json.dumps(data)
     headers = self.headers.copy()
     headers[HEADER_CONTENT_TYPE] = CONTENT_TYPE_JSON
     try:
         request = urllib2.Request(GBD_SEARCH_AOI_AND_TIME_URL, json_data, headers)
         response = self.opener.open(request)
         response_data = response.read()
         result_data = json.loads(response_data, strict=False)
         self.update_csv_data(order_params.time_end, result_data, csv_element)
     except Exception, e:
         QgsMessageLog.instance().logMessage(
             "Exception detected during aoi search: " + str(e), TAG_NAME, level=QgsMessageLog.CRITICAL
         )

    def acquisition_search(self, order_params):
        """
        Performs a search for acquisitions.
        :param order_params: The order params for the GBD query
        :return:
        """
        self.log_in()

        # build request body json
        request_body = {
            KEY_DATA_SEARCH_AREA_WKT: order_params.polygon,
            KEY_DATA_FILTERS: order_params.filters,
            KEY_DATA_TAG_RESULTS: VALUE_DATA_TAG_RESULTS,
            KEY_DATA_TYPES: VALUE_DATA_TYPES
        }
        if order_params.time_begin:
            request_body[KEY_DATA_START_DATE] = order_params.time_begin
        if order_params.time_end:
            request_body[KEY_DATA_END_DATE] = order_params.time_end
        request_body_json = json.dumps(request_body)

        # build header
        headers = self.headers.copy()
        headers[HEADER_CONTENT_TYPE] = CONTENT_TYPE_JSON

        try:
            request = urllib2.Request(ACQUISITION_SEARCH_URL, request_body_json, headers)
            response = self.opener.open(request)
            response_data = response.read()
            result_data = json.loads(response_data, strict=False)
            return result_data
        except Exception, e:
            QgsMessageLog.instance().logMessage("Exception during acquisition search: " + str(e), TAG_NAME, level=QgsMessageLog.CRITICAL)

def search(url, callback):
    QgsMessageLog.logMessage("URL:" + url, "Gazetteer")

    def requestFinished(reply):
        # Disconnect from the signal
        networkAccessManager = QgsNetworkAccessManager.instance()
        networkAccessManager.finished.disconnect(requestFinished)
        # Handle the reply
        if reply.error() != QNetworkReply.NoError:
            QgsMessageLog.logMessage("Network error #{0}: {1}".format(reply.error(), reply.errorString()), "Gazetteer")
            callback(u'')
        else:
            charset = 'UTF-8'
            try:
                _, params = cgi.parse_header(reply.header(QNetworkRequest.ContentTypeHeader))
                charset = params['charset']
            except:
                pass
            QgsMessageLog.logMessage("charset: " + charset, "Gazetteer")
            data = unicode(reply.readAll(), charset)
            reply.deleteLater()
            callback(data)

    networkAccessManager = QgsNetworkAccessManager.instance()
    networkAccessManager.finished.connect(requestFinished)
    networkAccessManager.get(QNetworkRequest(QUrl(QUrl.fromPercentEncoding(url))))

    def install(self):
        """Install the processing scripts in the collection.

        We copy the processing scripts exist in the processing dir to the
        user's processing scripts directory and refresh the provider.
        """
        # Check if the dir exists, pass installing silently if it doesn't exist
        if not os.path.exists(self.resource_dir):
            return

        # Get all the script files under self.resource_dir
        processing_files = []
        for item in os.listdir(self.resource_dir):
            file_path = os.path.join(self.resource_dir, item)
            if fnmatch.fnmatch(file_path, '*.py'):
                processing_files.append(file_path)

        valid = 0
        for processing_file in processing_files:
            # Install silently the processing file
                try:
                    shutil.copy(processing_file, self.scripts_folder())
                    valid += 1
                except OSError as e:
                    QgsMessageLog.logMessage("Could not copy script '{}'\n{}".format(processing_file, str(e)),
                                             "Processing",
                                             Qgis.Warning)

        if valid > 0:
            self.refresh_script_provider()

    def _processGdalDatasource(self, layer, destDirectory):
        uri = layer.source()
        filePath = layer.source()
        newDirectory = destDirectory.replace(self.baseDirectory, '.')
        newSource = '{dirName}/{fileName}'.format(dirName=newDirectory, fileName=os.path.split(filePath)[1])
        if uri.startswith(('/vsizip/', '/vsigzip/', '/vsitar/')):
            m = GDAL_VSI.search(uri)
            if m is not None:
                prefix = m.group(1)
                filePath = m.group(2)
                layerPath = m.group(4)
                newSource = '{vsi}{dirName}/{fileName}/{layer}'.format(vsi=prefix, dirName=newDirectory, fileName=os.path.split(filePath)[1], layer=layerPath)
            else:
                QgsMessageLog.logMessage(self.tr('Failed to parse URI: "{uri}"'.format(uri=uri)), 'QConsolidate', Qgis.Warning)
                return
        else:
            # ignore other virtual filesystems
            if uri.startswith('/vsi'):
                prefix = uri.slit('/')[0]
                QgsMessageLog.logMessage(self.tr('Not supported GDAL virtual filesystem layer "{layerType}"'.format(layerType=prefix)), 'QConsolidate', Qgis.Info)
                return

            # handle multiple layers in the single dataset
            if '|' in uri:
                filePath = uri.split('|')[0]
                layerPath = uri.split('|')[1]
                newSource = '{dirName}/{fileName}|{layer}'.format(dirName=newDirectory, fileName=os.path.split(filePath)[1], layer=layerPath)

        self._copyLayerFiles(filePath, destDirectory)
        self.updateLayerSource(layer.id(), newSource)

  def accept(self):
    """Creates and loads the WMS layer."""
    self.close()
    currentIndex = self.comboBoxLayer.currentIndex()
    currentLayerId, unused_dataType = self.comboBoxLayer.itemData(currentIndex)
    currentLayerName = unicode(self.comboBoxLayer.currentText())
    mapId = self.labelMapId.text()
    # Create the WMS layer
    token = oauth2_utils.getToken()
    url = 'https://mapsengine.google.com/%s-4/wms/%s/'
    wmsUrl = url % (mapId, token.access_token)
    currentFormatIndex = self.comboBoxFormat.currentIndex()
    imageFormat = unicode(self.comboBoxFormat.itemData(currentFormatIndex))
    crs = self.comboBoxCrs.currentText()

    uri = QgsDataSourceURI()
    uri.setParam('url', wmsUrl)
    uri.setParam('layers', currentLayerId)
    uri.setParam('format', imageFormat)
    uri.setParam('crs', crs)
    uri.setParam('styles', '')

    rlayer = QgsRasterLayer(str(uri.encodedUri()), currentLayerName, 'wms')
    if rlayer.isValid():
      QgsMapLayerRegistry.instance().addMapLayer(rlayer)
    else:
      logText = 'Failed to add WMS layer %s with URI %s' % (
          currentLayerName, str(uri.encodedUri()))
      warnText = 'Failed to add WMS layer %s' % currentLayerName
      QgsMessageLog.logMessage(logText, 'GMEConnector', QgsMessageLog.CRITICAL)
      self.iface.messageBar().pushMessage(
          'Google Maps Engine Connector', warnText,
          level=QgsMessageBar.CRITICAL, duration=3)

    def run(self):
        if not os.path.isdir(self.dataDirectory):
            os.mkdir(self.dataDirectory)

        self.prepare()

        self.consolidateProject()

        layers = QgsProject.instance().mapLayers()
        total = 100.0 / len(layers)

        for count, layer in enumerate(layers.values()):
            QgsMessageLog.logMessage('Consolidating {layer}.'.format(layer=layer.name()), 'QConsolidate', Qgis.Info)
            if self.isCanceled():
                break

            layerType = layer.type()
            if layerType == QgsMapLayer.VectorLayer:
                self.consolidateVectorLayer(layer)
            elif layerType == QgsMapLayer.RasterLayer:
                self.consolidateRasterLayer(layer)
            elif layerType == QgsMapLayer.PluginLayer:
                self.consolidatePluginLayer(layer)
            elif layerType == QgsMapLayer.MeshLayer:
                self.consolidateMeshLayer(layer)

            self.setProgress(int(count * total))

        self.project.write(self.projectFile)

        return self.cleanup()

 def return_batch_route(self, features):
     """Save and/or display the routes retrieved"""
     osrm_batch_route_layer = QgsVectorLayer(
         "Linestring?crs=epsg:4326&field=id:integer"
         "&field=total_time:integer(20)&field=distance:integer(20)",
         "routes_osrm{}".format(self.nb_done), "memory")
     provider = osrm_batch_route_layer.dataProvider()
     provider.addFeatures(features)
     QgsMapLayerRegistry.instance().addMapLayer(osrm_batch_route_layer)
     if self.filename:
         error = QgsVectorFileWriter.writeAsVectorFormat(
             osrm_batch_route_layer, self.filename,
             self.encoding, None, "ESRI Shapefile")
         if error != QgsVectorFileWriter.NoError:
             self.iface.messageBar().pushMessage(
                 "Error",
                 "Can't save the result into {} - Output have been "
                 "added to the canvas (see QGis log for error trace"
                 "back)".format(self.filename), duration=10)
             QgsMessageLog.logMessage(
                 'OSRM-plugin error report :\n {}'.format(error),
                 level=QgsMessageLog.WARNING)
             self.iface.setActiveLayer(osrm_batch_route_layer)
             return -1
         else:
             QtGui.QMessageBox.information(
                 self.iface.mainWindow(), 'Info',
                 "Result saved in {}".format(self.filename))
     if self.check_add_layer.isChecked():
         self.iface.setActiveLayer(osrm_batch_route_layer)
     else:
         QgsMapLayerRegistry.instance().removeMapLayer(
             osrm_batch_route_layer.id())
     self.iface.messageBar().clearWidgets()