def HandleUnverifiedNotFoundOnStaging(self, existingDatafile):
     """
     File has a DataFile record, and a DataFileObject record,
     marked as unverified.  The file is not accessible on staging.
     So we need to upload the entire file and create any subdirectories
     required for it on staging.
     """
     dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
     self.verificationModel\
         .SetMessage("Unverified and not found on staging server.")
     self.verificationModel.SetStatus(
         VerificationStatus.NOT_FOUND_ON_STAGING)
     self.verificationsModel.MessageUpdated(self.verificationModel)
     logger.debug("Uploading \"%s\" to staging, because "
                  "it was not there. It does have a DataFileObject."
                  % dataFilePath)
     self.verificationsModel.SetComplete(self.verificationModel)
     wx.PostEvent(
         self.foldersController.notifyWindow,
         self.foldersController.unverifiedDatafileOnServerEvent(
             id=self.foldersController.EVT_UNVERIFIED_NOT_FOUND_ON_STAGING,
             foldersController=self.foldersController,
             folderModel=self.folderModel,
             dataFileIndex=self.dataFileIndex,
             existingUnverifiedDatafile=existingDatafile,
             bytesUploadedPreviously=None,
             verificationModel=self.verificationModel))

    def GetFacility(settingsModel, name):
        myTardisUrl = settingsModel.GetMyTardisUrl()
        myTardisUsername = settingsModel.GetUsername()
        myTardisApiKey = settingsModel.GetApiKey()

        url = myTardisUrl + "/api/v1/facility/?format=json&name=" + \
            urllib.quote(name)
        headers = {'Authorization': 'ApiKey ' + myTardisUsername + ":" +
                   myTardisApiKey}
        session = requests.Session()
        response = session.get(url=url, headers=headers, stream=False)
        logger.debug(response.text)
        if response.status_code != 200:
            message = response.text
            response.close()
            session.close()
            raise Exception(message)
        facilitiesJson = response.json()
        response.close()
        session.close()
        numFacilitiesFound = facilitiesJson['meta']['total_count']

        if numFacilitiesFound == 0:
            logger.warning("Facility \"%s\" was not found in MyTardis" % name)
            return None
        else:
            logger.debug("Found facility record for name '" + name + "'.")
            return FacilityModel(
                settingsModel=settingsModel, name=name,
                facilityJson=facilitiesJson['objects'][0])

 def GetDataFile(settingsModel, dataset, filename, directory):
     myTardisUrl = settingsModel.GetMyTardisUrl()
     myTardisUsername = settingsModel.GetUsername()
     myTardisApiKey = settingsModel.GetApiKey()
     url = myTardisUrl + "/api/v1/mydata_dataset_file/?format=json" + \
         "&dataset__id=" + str(dataset.GetId()) + \
         "&filename=" + urllib.quote(filename) + \
         "&directory=" + urllib.quote(directory)
     headers = {"Authorization": "ApiKey " + myTardisUsername + ":" +
                myTardisApiKey}
     response = requests.get(url=url, headers=headers)
     if response.status_code < 200 or response.status_code >= 300:
         logger.debug("Failed to look up datafile \"%s\" "
                      "in dataset \"%s\"."
                      % (filename, dataset.GetDescription()))
         logger.debug(response.text)
         return None
     dataFilesJson = response.json()
     numDataFilesFound = dataFilesJson['meta']['total_count']
     if numDataFilesFound == 0:
         raise DoesNotExist(
             message="Datafile \"%s\" was not found in MyTardis" % filename,
             url=url, response=response)
     elif numDataFilesFound > 1:
         raise MultipleObjectsReturned(
             message="Multiple datafiles matching %s were found in MyTardis"
             % filename,
             url=url, response=response)
     else:
         return DataFileModel(
             settingsModel=settingsModel,
             dataset=dataset,
             dataFileJson=dataFilesJson['objects'][0])

        def RenameInstrumentWorker():
            """
            Renames instrument in separate thread.
            """
            logger.debug("Starting run() method for thread %s"
                         % threading.current_thread().name)
            try:
                wx.CallAfter(BeginBusyCursorIfRequired)
                event.settingsModel.RenameInstrument(
                    event.facilityName,
                    event.oldInstrumentName,
                    event.newInstrumentName)

                wx.CallAfter(EndBusyCursorIfRequired, event)
                if event.nextEvent:
                    wx.PostEvent(wx.GetApp().GetMainFrame(), event.nextEvent)
            except DuplicateKey:
                wx.CallAfter(EndBusyCursorIfRequired, event)

                def NotifyUserOfDuplicateInstrumentName():
                    """
                    Notifies user of duplicate instrument name.
                    """
                    message = "Instrument name \"%s\" already exists in " \
                        "facility \"%s\"." \
                        % (event.newInstrumentName,
                           event.facilityName)
                    dlg = wx.MessageDialog(None, message, "MyData",
                                           wx.OK | wx.ICON_ERROR)
                    dlg.ShowModal()
                    event.settingsDialog.instrumentNameField.SetFocus()
                    event.settingsDialog.instrumentNameField.SelectAll()
                wx.CallAfter(NotifyUserOfDuplicateInstrumentName)
            logger.debug("Finishing run() method for thread %s"
                         % threading.current_thread().name)

 def HandleIncompleteResumableUpload(self, existingDatafile,
                                     bytesUploadedPreviously):
     """
     Resume partial upload.
     """
     dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
     self.verificationModel\
         .SetMessage("Found partially uploaded datafile "
                     "on staging server.")
     self.verificationModel\
         .SetStatus(VerificationStatus
                    .FOUND_UNVERIFIED_NOT_FULL_SIZE)
     self.verificationsModel.MessageUpdated(self.verificationModel)
     logger.debug("Re-uploading \"%s\" to staging, because "
                  "the file size is %s bytes in staging, "
                  "but it should be %s bytes."
                  % (dataFilePath,
                     bytesUploadedPreviously,
                     existingDatafile.GetSize()))
     self.verificationsModel.SetComplete(self.verificationModel)
     wx.PostEvent(
         self.foldersController.notifyWindow,
         self.foldersController.unverifiedDatafileOnServerEvent(
             id=self.foldersController.EVT_INCOMPLETE_FILE_ON_STAGING,
             foldersController=self.foldersController,
             folderModel=self.folderModel,
             dataFileIndex=self.dataFileIndex,
             existingUnverifiedDatafile=existingDatafile,
             bytesUploadedPreviously=bytesUploadedPreviously,
             verificationModel=self.verificationModel))

    def CalculateMd5Sum(self, filePath, fileSize, uploadModel,
                        progressCallback=None):
        """
        Calculate MD5 checksum.
        """
        md5 = hashlib.md5()

        defaultChunkSize = 128 * 1024
        maxChunkSize = 16 * 1024 * 1024
        chunkSize = defaultChunkSize
        while (fileSize / chunkSize) > 50 and chunkSize < maxChunkSize:
            chunkSize = chunkSize * 2
        bytesProcessed = 0
        with open(filePath, 'rb') as fileHandle:
            # Note that the iter() func needs an empty byte string
            # for the returned iterator to halt at EOF, since read()
            # returns b'' (not just '').
            for chunk in iter(lambda: fileHandle.read(chunkSize), b''):
                if self.IsShuttingDown() or uploadModel.Canceled():
                    logger.debug("Aborting MD5 calculation for "
                                 "%s" % filePath)
                    return None
                md5.update(chunk)
                bytesProcessed += len(chunk)
                del chunk
                if progressCallback:
                    progressCallback(bytesProcessed)
        return md5.hexdigest()

        def jobFunc(taskModel, tasksModel, row, col):

            def taskJobFunc():
                assert callable(taskModel.GetJobFunc())
                title = "Starting"
                message = taskModel.GetJobDesc()
                Notification.notify(message, title=title)
                taskModel.GetJobFunc()(*taskModel.GetJobArgs())
                taskModel.SetFinishTime(datetime.now())
                title = "Finished"
                message = taskModel.GetJobDesc()
                Notification.notify(message, title=title)
                wx.CallAfter(tasksModel.RowValueChanged, row, col)
                scheduleType = taskModel.GetScheduleType()
                if scheduleType == "Timer":
                    intervalMinutes = taskModel.GetIntervalMinutes()
                    newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
                    newStartTime = taskModel.GetStartTime() + \
                        timedelta(minutes=intervalMinutes)
                    newTaskModel = TaskModel(newTaskDataViewId,
                                             taskModel.GetJobFunc(),
                                             taskModel.GetJobArgs(),
                                             taskModel.GetJobDesc(),
                                             newStartTime,
                                             scheduleType="Timer",
                                             intervalMinutes=intervalMinutes)
                    timeString = newStartTime.strftime("%I:%M:%S %p")
                    dateString = \
                        "{d:%A} {d.day}/{d.month}/{d.year}".format(d=newStartTime)
                    wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
                                 "The \"%s\" task is scheduled "
                                 "to run at %s on %s "
                                 "(recurring every %d minutes)"
                                 % (taskModel.GetJobDesc(),
                                    timeString, dateString, intervalMinutes))
                    tasksModel.AddRow(newTaskModel)
                elif scheduleType == "Daily":
                    newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
                    newStartTime = taskModel.GetStartTime() + \
                        timedelta(days=1)
                    newTaskModel = TaskModel(newTaskDataViewId,
                                             taskModel.GetJobFunc(),
                                             taskModel.GetJobArgs(),
                                             taskModel.GetJobDesc(),
                                             newStartTime,
                                             scheduleType="Daily")
                    timeString = newStartTime.strftime("%I:%M:%S %p")
                    dateString = \
                        "{d:%A} {d.day}/{d.month}/{d.year}".format(d=newStartTime)
                    wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
                                 "The \"%s\" task is scheduled "
                                 "to run at %s on %s "
                                 "(recurring daily)"
                                 % (taskModel.GetJobDesc(),
                                    timeString, dateString))
                    tasksModel.AddRow(newTaskModel)

            thread = threading.Thread(target=taskJobFunc)
            logger.debug("Starting task %s" % taskModel.GetJobDesc())
            thread.start()

 def OnSyncNow(self, event):
     """
     Called when the "Sync Now" menu item is
     selected from MyData's system tray / menu bar icon menu.
     """
     logger.debug("Sync Now called from task bar menu item.")
     wx.GetApp().ScanFoldersAndUpload(event)

        def ScanDataDirs():
            """
            Scan data folders, looking for datafiles to look up on MyTardis
            and upload if necessary.
            """
            logger.debug("Starting run() method for thread %s" % threading.current_thread().name)
            wx.CallAfter(self.frame.SetStatusMessage, "Scanning data folders...")
            try:
                self.scanningFoldersThreadingLock.acquire()
                self.SetScanningFolders(True)
                logger.info("Just set ScanningFolders to True")
                self.toolbar.EnableTool(self.stopTool.GetId(), True)
                self.foldersModel.ScanFolders(WriteProgressUpdateToStatusBar, self.ShouldAbort)
                self.SetScanningFolders(False)
                self.scanningFoldersThreadingLock.release()
                logger.info("Just set ScanningFolders to False")
            except InvalidFolderStructure, ifs:

                def ShowMessageDialog():
                    """
                    Needs to run in the main thread.
                    """
                    dlg = wx.MessageDialog(None, str(ifs), "MyData", wx.OK | wx.ICON_ERROR)
                    dlg.ShowModal()

                wx.CallAfter(ShowMessageDialog)
                self.frame.SetStatusMessage(str(ifs))
                return

    def GetFacility(settingsModel, name):
        """
        Get facility by name.
        """
        myTardisUrl = settingsModel.GetMyTardisUrl()
        myTardisUsername = settingsModel.GetUsername()
        myTardisApiKey = settingsModel.GetApiKey()

        url = myTardisUrl + "/api/v1/facility/?format=json&name=" + urllib.quote(name)
        headers = {
            "Authorization": "ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
            "Content-Type": "application/json",
            "Accept": "application/json",
        }
        response = requests.get(url=url, headers=headers, stream=False)
        logger.debug(response.text)
        if response.status_code != 200:
            message = response.text
            response.close()
            raise Exception(message)
        facilitiesJson = response.json()
        response.close()
        numFacilitiesFound = facilitiesJson["meta"]["total_count"]

        if numFacilitiesFound == 0:
            logger.warning('Facility "%s" was not found in MyTardis' % name)
            return None
        else:
            logger.debug("Found facility record for name '" + name + "'.")
            return FacilityModel(settingsModel=settingsModel, name=name, facilityJson=facilitiesJson["objects"][0])

    def ScanFolders(self, incrementProgressDialog, shouldAbort):
        if self.GetCount() > 0:
            self.DeleteAllRows()
        if self.usersModel.GetCount() > 0:
            self.usersModel.DeleteAllRows()
        if self.groupsModel.GetCount() > 0:
            self.groupsModel.DeleteAllRows()
        dataDir = self.settingsModel.GetDataDirectory()
        folderStructure = self.settingsModel.GetFolderStructure()
        self.ignoreOldDatasets = self.settingsModel.IgnoreOldDatasets()
        if self.ignoreOldDatasets:
            seconds = {}
            seconds['day'] = 24 * 60 * 60
            seconds['week'] = 7 * seconds['day']
            seconds['year'] = int(365.25 * seconds['day'])
            seconds['month'] = seconds['year'] / 12
            singularIgnoreIntervalUnit = \
                self.settingsModel.GetIgnoreOldDatasetIntervalUnit().rstrip(
                    's')
            ignoreIntervalUnitSeconds = seconds[singularIgnoreIntervalUnit]

            self.ignoreIntervalNumber = \
                self.settingsModel.GetIgnoreOldDatasetIntervalNumber()
            self.ignoreIntervalUnit = \
                self.settingsModel.GetIgnoreOldDatasetIntervalUnit()
            self.ignoreIntervalSeconds = \
                self.ignoreIntervalNumber * ignoreIntervalUnitSeconds
        logger.debug("FoldersModel.ScanFolders(): Scanning " + dataDir + "...")
        if folderStructure.startswith("Username") or \
                folderStructure.startswith("Email"):
            self.ScanForUserFolders(incrementProgressDialog, shouldAbort)
        elif folderStructure.startswith("User Group"):
            self.ScanForGroupFolders(incrementProgressDialog, shouldAbort)
        else:
            raise InvalidFolderStructure("Unknown folder structure.")

    def GetUserByEmail(settingsModel, email):
        myTardisUrl = settingsModel.GetMyTardisUrl()
        myTardisUsername = settingsModel.GetUsername()
        myTardisApiKey = settingsModel.GetApiKey()

        url = myTardisUrl + "/api/v1/user/?format=json&email__iexact=" + \
            urllib2.quote(email)
        headers = {
            "Authorization": "ApiKey %s:%s" % (myTardisUsername,
                                               myTardisApiKey)}
        try:
            response = requests.get(url=url, headers=headers)
        except:
            raise Exception(traceback.format_exc())
        if response.status_code != 200:
            logger.debug(url)
            message = response.text
            raise Exception(message)
        try:
            userRecordsJson = response.json()
        except:
            logger.error(traceback.format_exc())
            raise
        numUserRecordsFound = userRecordsJson['meta']['total_count']

        if numUserRecordsFound == 0:
            raise DoesNotExist(
                message="User with email \"%s\" was not found in MyTardis"
                % email,
                url=url, response=response)
        else:
            logger.debug("Found user record for email '" + email + "'.")
            return UserModel(settingsModel=settingsModel,
                             userRecordJson=userRecordsJson['objects'][0])

    def ReadPublicKey(self):
        """
        Read public key, including "ssh-rsa "
        """
        if self.publicKeyFilePath is not None and \
                os.path.exists(self.publicKeyFilePath):
            with open(self.publicKeyFilePath, "r") as pubKeyFile:
                return pubKeyFile.read()
        elif os.path.exists(self.privateKeyFilePath):
            cmdList = [OPENSSH.DoubleQuote(OPENSSH.sshKeyGen),
                       "-y",
                       "-f", OPENSSH.DoubleQuote(
                           GetCygwinPath(self.privateKeyFilePath))]
            cmd = " ".join(cmdList)
            logger.debug(cmd)
            proc = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True,
                                    startupinfo=DEFAULT_STARTUP_INFO,
                                    creationflags=DEFAULT_CREATION_FLAGS)
            stdout, _ = proc.communicate()

            if proc.returncode != 0:
                raise SshException(stdout)

            return stdout
        else:
            raise SshException("Couldn't find MyData key files in ~/.ssh "
                               "while trying to read public key.")

 def HandleExistingUnverifiedDatafile(self, existingDatafile):
     """
     If the existing unverified DataFile was uploaded via POST, we just
     need to wait for it to be verified.  But if it was uploaded via
     staging, we might be able to resume a partial upload.
     """
     self.verificationModel.SetExistingUnverifiedDatafile(existingDatafile)
     dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
     replicas = existingDatafile.GetReplicas()
     message = "Found datafile record for %s " \
         "but it has no verified replicas." % dataFilePath
     logger.debug(message)
     message = "Found unverified datafile record on MyTardis."
     self.verificationModel.SetMessage(message)
     uploadToStagingRequest = self.settingsModel.GetUploadToStagingRequest()
     if self.foldersController.uploadMethod == \
             UploadMethod.VIA_STAGING and \
             uploadToStagingRequest is not None and \
             uploadToStagingRequest.IsApproved() and \
             len(replicas) > 0:
         # Can resume partial uploads:
         self.HandleResumableUpload(existingDatafile)
     else:
         # Can't resume partial uploads:
         self.HandleUnresumableUpload(existingDatafile)

            def ValidateSettings():
                """
                Validate settings.
                """
                logger.debug("Starting run() method for thread %s" % threading.current_thread().name)
                # pylint: disable=bare-except
                try:
                    wx.CallAfter(wx.BeginBusyCursor)
                    # pylint: disable=broad-except
                    try:
                        activeNetworkInterfaces = UploaderModel.GetActiveNetworkInterfaces()
                    except Exception, err:
                        logger.error(traceback.format_exc())
                        if type(err).__name__ == "WindowsError" and "The handle is invalid" in str(err):
                            message = (
                                "An error occurred, suggesting "
                                "that you have launched MyData.exe from a "
                                "Command Prompt window.  Please launch it "
                                "from a shortcut or from a Windows Explorer "
                                "window instead.\n"
                                "\n"
                                "See: https://bugs.python.org/issue3905"
                            )

                            def ShowErrorDialog(message):
                                """
                                Needs to run in the main thread.
                                """
                                dlg = wx.MessageDialog(None, message, "MyData", wx.OK | wx.ICON_ERROR)
                                dlg.ShowModal()

                            wx.CallAfter(ShowErrorDialog, message)
                    if len(activeNetworkInterfaces) == 0:
                        message = (
                            "No active network interfaces."
                            "\n\n"
                            "Please ensure that you have an active "
                            "network interface (e.g. Ethernet or WiFi)."
                        )

                        def ShowDialog():
                            """
                            Needs to run in the main thread.
                            """
                            dlg = wx.MessageDialog(None, message, "MyData", wx.OK | wx.ICON_ERROR)
                            dlg.ShowModal()
                            wx.CallAfter(EndBusyCursorIfRequired)
                            self.frame.SetStatusMessage("")
                            self.frame.SetConnected(self.settingsModel.GetMyTardisUrl(), False)

                        wx.CallAfter(ShowDialog)
                        return

                    self.settingsValidation = self.settingsModel.Validate()
                    event = mde.MyDataEvent(
                        mde.EVT_SETTINGS_VALIDATION_FOR_REFRESH_COMPLETE, needToValidateSettings=False
                    )
                    wx.PostEvent(self.frame, event)
                    wx.CallAfter(EndBusyCursorIfRequired)

 def CleanUp(self):
     """
     Perform clean up actions on each each folder model.
     """
     logger.debug("Cleaning up each FolderModel record's threads...")
     for row in range(0, self.GetRowCount()):
         self.foldersData[row].CleanUp()
     logger.debug("Cleaned up each FolderModel record's threads...")

 def OnRefreshFromToolbar(self, event):
     """
     The user pressed the Refresh icon on the main windows' toolbar.
     """
     logger.debug("OnRefreshFromToolbar")
     self.tasksModel.DeleteAllRows()
     self.settingsModel.SetScheduleType("Manually")
     self.settingsModel.SetLastSettingsUpdateTrigger(LastSettingsUpdateTrigger.UI_RESPONSE)
     self.scheduleController.ApplySchedule(event, runManually=True)

 def TryRowDeleted(self, row):
     try:
         if row < self.GetCount():
             self.RowDeleted(row)
         else:
             logger.warning("TryRowDeleted called with "
                            "row=%d, self.GetRowCount()=%d" %
                            (row, self.GetRowCount()))
     except:
         logger.debug(traceback.format_exc())

    def ShutdownForRefresh(event):
        """
        Shuts down upload threads before restarting them.
        """
        if event.GetEventId() != EVT_SHUTDOWN_FOR_REFRESH:
            event.Skip()
            return

        def ShutdownForRefreshWorker():
            """
            Shuts down upload threads (in dedicated worker thread)
            before restarting them.
            """
            logger.debug("Starting run() method for thread %s"
                         % threading.current_thread().name)
            logger.debug("Shutting down for refresh from %s."
                         % threading.current_thread().name)
            # pylint: disable=bare-except
            try:
                wx.CallAfter(BeginBusyCursorIfRequired)
                app = wx.GetApp()
                app.GetScheduleController().ApplySchedule(event)
                event.foldersController.ShutDownUploadThreads()
                shutdownForRefreshCompleteEvent = MyDataEvent(
                    EVT_SHUTDOWN_FOR_REFRESH_COMPLETE,
                    shutdownSuccessful=True)
                wx.PostEvent(app.GetMainFrame(),
                             shutdownForRefreshCompleteEvent)
                wx.CallAfter(EndBusyCursorIfRequired, event)
            except:
                logger.error(traceback.format_exc())
                message = "An error occurred while trying to shut down " \
                    "the existing data-scan-and-upload process in order " \
                    "to start another one.\n\n" \
                    "See the Log tab for details of the error."
                logger.error(message)

                def ShowDialog():
                    """
                    Show error dialog in main thread.
                    """
                    dlg = wx.MessageDialog(None, message, "MyData",
                                           wx.OK | wx.ICON_ERROR)
                    dlg.ShowModal()
                wx.CallAfter(ShowDialog)
            logger.debug("Finishing run() method for thread %s"
                         % threading.current_thread().name)

        shutdownForRefreshThread = \
            threading.Thread(target=ShutdownForRefreshWorker,
                             name="ShutdownForRefreshThread")
        MYDATA_THREADS.Add(shutdownForRefreshThread)
        logger.debug("Starting thread %s" % shutdownForRefreshThread.name)
        shutdownForRefreshThread.start()
        logger.debug("Started thread %s" % shutdownForRefreshThread.name)

 def TryRowValueChanged(self, row, col):
     # pylint: disable=bare-except
     try:
         if row < self.GetCount():
             self.RowValueChanged(row, col)
         else:
             logger.warning("TryRowValueChanged called with "
                            "row=%d, self.GetRowCount()=%d" %
                            (row, self.GetRowCount()))
             self.RowValueChanged(row, col)
     except:
         logger.debug(traceback.format_exc())