def getTimeOfLastUpdateInLogs(self):
     """
         
         @summary : Returns the time of the last update in iso format.
    
         @return : None if no update as found, EPCH is returned in iso format,
                   as to make sure an update is made since no prior updates exist.
         
     """
     
     timeOfLastUpdate = StatsDateLib.getIsoTodaysMidnight( StatsDateLib.getCurrentTimeInIsoformat() ) 
     
     paths = StatsPaths()
     paths.setPaths()
     
     updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/"
     
     if not os.path.isdir( updatesDirectory ):
         os.makedirs(updatesDirectory)       
     allEntries = os.listdir(updatesDirectory) 
     
     if allEntries !=[] :
         allEntries.sort()
         allEntries.reverse() 
         timeOfLastUpdate = os.path.basename( allEntries[0] ).replace( "_"," " )
         
         
     return timeOfLastUpdate

 def getPreviousMonitoringJob( self, currentTime ):
     """
         @summary : Gets the previous crontab from the pickle file.
         
         @return : Time of the previous monitoring job.
         
         @warning : Returns "" if file does not exist.
         
     """     
     
     statsPaths = StatsPaths()
     statsPaths.setPaths()         
     
     file  = "%spreviousMonitoringJob" %statsPaths.STATSMONITORING
     previousMonitoringJob = ""
     
     if os.path.isfile( file ):
         fileHandle      = open( file, "r" )
         previousMonitoringJob = pickle.load( fileHandle )
         fileHandle.close()
         #print previousMonitoringJob
         
     else:
         previousMonitoringJob = StatsDateLib.getIsoTodaysMidnight( currentTime )
         
     #print previousMonitoringJob   
     
     return previousMonitoringJob        

    def __getAutomaticUpdatesDoneDuringTimeSpan( self, startTime, endtime ):
        """
        
            @param startTime: Start time of the span in iso format 
            
            @param endtime: end time of the span in iso format

        """
        #set to fit file standard
        startTime = startTime.replace( " ", "_" )
        endtime = endtime.replace( " ", "_" )
        
        def afterEndTime(x):
            return x <= endtime
        
        def beforeStartTime(x):
            return x >= startTime
        
 
        
        paths = StatsPaths()
        paths.setPaths()
        
        updates = os.listdir( updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" ) 

        updates =  filter( afterEndTime, updates)

        updates =  filter( beforeStartTime, updates)     
  
        
        return updates

 def addAutomaticUpdateToLogs( self, timeOfUpdateInIsoFormat, currentUpdateFrequency  = None ):
    """
        @summary : Writes a new file in the log folder containing 
                   the current update frequency. 
    
        @param timeOfUpdateInIsoFormat: Time that the entries name will sport.
    
    """
    
    paths = StatsPaths()
    paths.setPaths()
    fileName = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + str( timeOfUpdateInIsoFormat ).replace( " ", "_" )
    
    #Safety to make sure 
    if not os.path.isdir( os.path.dirname( fileName ) ):
        os.makedirs( os.path.dirname( fileName ), 0777 )
    
    if currentUpdateFrequency  == None :
        currentUpdateFrequency = self.getCurrentUpdateFrequency()   
    
    CpickleWrapper.save( currentUpdateFrequency, fileName )
        
    allEntries = os.listdir(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/") 
    
    allEntries.sort()
    
    entriesToRemove = allEntries[ :-self.numberOfLogsToKeep]
    
    for entrytoRemove in entriesToRemove:
        os.remove(paths.STATSTEMPAUTUPDTLOGS  + self.updateType + "/" + entrytoRemove ) 

def restoreDatabaseUpdateTimes( timeToRestore, currentTime, nbBackupsToKeep ):
    """
       @summary : Copy all databases into a folder sporting the data of the backup.
        
       @param timeToRestore : Time of the DB backups to set as current DB.
       
       @param currentTime : Time of the call to the script.
       
       @param nbBackupsToKeep : total number of backups to keep.
       
    """

    statsPaths = StatsPaths()
    statsPaths.setPaths()
        
    source = statsPaths.STATSDBUPDATESBACKUPS + "/%s" %timeToRestore
    destination = statsPaths.STATSCURRENTDBUPDATES
    
    #Archive current Database
    backupRRDDatabases.backupDatabaseUpdateTimes( currentTime, nbBackupsToKeep, foldersToPreserve = [ source ] )
    
    #restore desired 
    status, output = commands.getstatusoutput( "rm -r %s" %( destination ) )
    os.makedirs(destination)
    status, output = commands.getstatusoutput( "cp -rf %s/* %s" %( source, destination ) )
    print output

    def __updateCsvFiles( self, type, clusters, cost ):
        """
        
            @summary    : Generate th rx and tx csv files
                          for yesterday for all clusters.
            
            @param type : daily | weekly | monthly | yearly 
             
            @param clusters :  List of currently running source clusters.

            @param cost : total operational cost for the perido specified 
                          by the type
            
            @return : None
    
        """   
        
        paths = StatsPaths()
        paths.setPaths()
        
        typeParameters = {  "daily" : "-d", "weekly" : "-w", "monthly" : "-m", "yearly" : "-y" }
        
        output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s'  %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) )
        #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s'  %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage )
           
        output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s'  %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) )
        #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s'  %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage )      
        
        fileName = self.__getFileNameFromExecutionOutput(output)
        
        if fileName != "":
            commands.getstatusoutput(paths.STATSWEBPAGESGENERATORS + 'csvDataFiltersForWebPages.py -c %s -f %s ' %(cost, fileName) )

def updateFilesAssociatedWithMachineTags( tagsNeedingUpdates, machineParameters ):   
    """
        @summary : For all the tags for wich 
                   a machine was change we rename all the 
                   files associated with that tag.
        
        @param tagsNeedingUpdates: List of tags that have been modified 
                                   since the last call.
                                             
    
    """
    
    paths = StatsPaths()
    paths.setPaths()
    
    previousParameters = getMachineParametersFromPreviousCall()
    
    for tag in tagsNeedingUpdates:
        previousCombinedMachineNames = ""
        previousCombinedMachineNames = previousCombinedMachineNames.join( [ x for x in previousParameters.getMachinesAssociatedWith( tag ) ] )
        
        currentCombinedMachineNames = ""
        currentCombinedMachineNames = currentCombinedMachineNames.join( [ x for x in machineParameters.getMachinesAssociatedWith( tag ) ]) 
        
        output = commands.getoutput( "%sfileRenamer.py -o %s  -n %s --overrideConfirmation" %( paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames  ) )
        print "%sfileRenamer.py -o %s  -n %s --overrideConfirmation" %( paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames  )
        print output 

def transferLogFiles():
    """
        @summary : Log files will not be tansferred if local machine
                   is not designed to be a pickling machine. 
                   
                   If log files are to be transferred, they will be straight
                  from the source."
    """
    
    paths = StatsPaths()
    paths.setPaths()
    
    parameters = StatsConfigParameters()    
    machineParameters = MachineConfigParameters()
    machineParameters.getParametersFromMachineConfigurationFile()
    parameters.getAllParameters()
    individualSourceMachines   = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags )
    individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines )
        
    for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) :      
               
        if picklingMachine == LOCAL_MACHINE :#pickling to be done here  
            
            userName = machineParameters.getUserNameForMachine(sourceMachine)
            remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName )
            
            
            print  "rsync -avzr --delete-before -e ssh %[email protected]%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine  )
            output = commands.getoutput( "rsync -avzr --delete-before -e ssh %[email protected]%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine  ) )
            print output

def cleanUp( configParameters, currentTime, daysOfPicklesToKeep ):
    """
    
        @summary: Based on current time and frequencies contained
                  within the time parameters, we will run 
                  the cleaners that need to be run.       
                            
        @param configParameters: StatsConfigParameters instance.
                               
        @param currenTime: currentTime in seconds since epoch format.
                                  
    """     
    
    paths = StatsPaths()
    paths.setPaths()
    
    updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner")
    
    if updateManager.updateIsRequired(currentTime) :
        
        output = commands.getoutput( paths.STATSTOOLS + "pickleCleaner.py %s" %int(daysOfPicklesToKeep) )
        print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep )
        updateManager.addAutomaticUpdateToLogs( currentTime )
        
    updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner")    
    
    if updateManager.updateIsRequired(currentTime) :
        commands.getstatusoutput( paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf"   )
        print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" 
        updateManager.addAutomaticUpdateToLogs( currentTime )

 def __getDocFilesToLinkTo(self, language):
     """    
         @summary : Gathers and returns all the documentation files
                    currently available 
         
         @summary : The list of fileNames to link to.
         
     """
     
     filesToLinkTo = []       
     
     statsPaths = StatsPaths()
     statsPaths.setPaths( self.mainLanguage )
     folder = statsPaths.STATSDOC + "html/"
     
     listOfFilesInFolder = os.listdir(folder)
     
     for file in listOfFilesInFolder:
         baseName = os.path.basename(file)
         if( fnmatch.fnmatch( baseName, "*_%s.html"%(language) ) ):
             filesToLinkTo.append( baseName )
     
     filesToLinkTo.sort()
     
     return filesToLinkTo

 def saveList( self, user, clients ):   
     """
         @summary : Saves list. 
         
         @note : Will include modification made in updateFileInlist method 
         
         @param clients : Client to wich the file is related(used to narrow down searchs)
         
         @param user   : Name of the client, person, etc.. wich has a relation with the 
                         file. 
         
         
     """
     statsPaths = StatsPaths()
     statsPaths.setPaths()
     directory = statsPaths.STATSDATA + "fileAccessVersions/"
      
     
     combinedName = ""
     for client in clients:
         combinedName = combinedName + client
     
     fileName  = combinedName + "_" + user 
     
     if not os.path.isdir( directory ):
         os.makedirs( directory, mode=0777 )
         #create directory
     completeFilename = directory + fileName 
     #print "saving %s" %completeFilename
             
     CpickleWrapper.save( object = self.savedFileList, filename = completeFilename )

  def getClientsCurrentFileList( self, clients ):
      """
          @summary : Gets all the files associated with the list of clients.
                  
          
          @note : Client list is used here since we need to find all the pickles that will be used in a merger.
                  Thus unlike all other methods we dont refer here to the combined name but rather to a list of
                  individual machine names. 
          
          @summary : Returns the all the files in a dictionnary associated
                     with each file associated with it's mtime.
          
      """  
      
      
      fileNames = []
      statsPaths = StatsPaths()
      statsPaths.setPaths()
      
      for client in clients : 
          filePattern = statsPaths.STATSPICKLES + client + "/*/*"  #_??
          folderNames = glob.glob( filePattern )
                      
          
          for folder in folderNames:
              if os.path.isdir( folder ):                    
                  filePattern = folder + "/" + "*_??"
                  fileNames.extend( glob.glob( filePattern ) )       
                  
  
          for fileName in fileNames :
              self.currentClientFileList[fileName] = os.path.getmtime( fileName )            
 
              
      return  self.currentClientFileList       

    def getSavedList( self, user, clients ):
        """
            @summary : Returns the checksum of the files contained in the saved list.
        
        """

        self.savedFileList         = {}
        
        statsPaths = StatsPaths()
        statsPaths.setPaths()
        directory = statsPaths.STATSDATA + "fileAccessVersions/"              
                
        combinedName = ""
        for client in clients:
            combinedName = combinedName + client
        
        fileName  = combinedName + "_" + user            
            
        try :
            
            self.savedFileList = CpickleWrapper.load( directory + fileName )
            
            if self.savedFileLis == None :
                self.savedFileList = {}
                
        except: # if file does not exist
            pass
        
        
        return self.savedFileList

def updatePickledTimes( dateToSet = "2006-10-23 09:00:00"  ):
    """
          @summary : Get all the keys then set all of them to the desired date.
    """
    
    statsPaths = StatsPaths()
    statsPaths.setPaths()
    
    folder = statsPaths.STATSPICKLESTIMEOFUPDATES
    
    files = os.listdir(folder)
    for fileName in files :
        if os.path.isfile( fileName ):
    
            fileHandle   = open( fileName, "r" )
            pickledTimes = pickle.load( fileHandle )
            fileHandle.close()
            
            
            keys = pickledTimes.keys()
            for key in keys:
                pickledTimes[key] = dateToSet
                
            fileHandle  = open( fileName, "w" )
    
            pickle.dump( pickledTimes, fileHandle )
    
            fileHandle.close()

def main():
    """
        @summary : Small test case scenario allows 
                   for unit-like testing of the LanguageTools
                   class. 
    """
    
    configParameters = StatsConfigParameters()
    configParameters.getAllParameters()
    language = configParameters.mainApplicationLanguage
    
    paths = StatsPaths()
    paths.setBasicPaths()
    
    print "Language set in config file : %s" %language
    
    print "Test1 : (Should show that the proper translation file will be used) "
    fileName =  LanguageTools.getTranslationFileName( language, paths.STATSLIB + 'StatsPlotter' )
    print "Translation file to be used : %s " %( fileName ) 
    
    print "Test2 : (Should translate the word into the specified language) "
    translator = LanguageTools.getTranslator( fileName )
    print "Translation for bytecount : %s" %( translator("bytecount") )
    
    print "Test3 : (Should be the same result as test 2) "
    translator = LanguageTools.getTranslatorForModule( paths.STATSLIB + 'StatsPlotter', language )
    print "Translation for bytecount : %s" %( translator("bytecount") )
    
    print "Test4 : Unless translation changes, this should print 'filecount' "
    print "Result : ", LanguageTools.translateTerm("nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py" )

    def __init__( self, displayedLanguage = 'en', filesLanguage='en', days = None, \
                  weeks = None, months = None, years = None, \
                  pathsTowardsGraphics = None, pathsTowardsOutputFiles = None  ):
        """
        
            @summary : Constructor 
            
            @param displayedLanguage: Languages in which to display 
                                      the different captions found within 
                                      the generated web page.  
            
            @param fileLanguages: Language in which the files that 
                                  will be referenced within this page
                                  have been generated.
                                  
            @param days : List of days that the web page covers.
        
            @note : Will set two global translators to be used throughout this module 
                    _ which translates every caption that is to be printed.
                    _F which translates every filename that is to be linked.     
        """
        
        configParameters = StatsConfigParameters()
        configParameters.getGeneralParametersFromStatsConfigurationFile()
      
        global _ 
        _ =  self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, displayedLanguage )
        
        if days == None:
            self.setDays()
        else:    
            self.days = days 
            
        if weeks == None:
            self.setWeeks()
        else:    
            self.weeks = weeks             
            
        if months == None:
            self.setMonths()
        else:    
            self.months = months 
                            
        if years == None:
            self.setYears()
        else:    
            self.years = years                 
                
        self.displayedLanguage = displayedLanguage
        self.filesLanguage     = filesLanguage
       
        self.pathsTowardsGraphics = StatsPaths()
        self.pathsTowardsGraphics.setPaths( filesLanguage )
        
        self.pathsTowardsOutputFiles = StatsPaths()
        self.pathsTowardsOutputFiles.setPaths( self.displayedLanguage )

        StatsDateLib.setLanguage(filesLanguage)

def updateCsvFiles():
    """    
        
        @summary : Runs the csv file update utility.
        
    """
    
    paths = StatsPaths()
    paths.setPaths()    
    output = commands.getoutput( "%sgetCsvFilesforWebPages.py" %paths.STATSWEBPAGESGENERATORS )

    def getGroupSettingsFromConfigurationFile( self ):
        """
            Reads all the group settings from 
            the configuration file.
        """
        
        groupParameters = GroupConfigParameters([], {}, {}, {},{} )
        
        machineParameters = MachineConfigParameters()        
        machineParameters.getParametersFromMachineConfigurationFile()
        
        paths = StatsPaths()
        paths.setBasicPaths()
        config = paths.STATSETC  + "config"
        fileHandle = open( config, "r" )
        
        line = fileHandle.readline()#read until groups section, or EOF
        while line != "" and "[specialGroups]" not in line: 
            
            line = fileHandle.readline()
            
            
        if line != "":#read until next section, or EOF     
            
            line = fileHandle.readline()            
            
            while line != ""  and "[" not in line:
                
                if line != '\n' and line[0] != '#' :
                    
                    splitLine = line.split()
                    if len( splitLine ) == 6:
                        groupName =  splitLine[0] 
                        if groupName not in (groupParameters.groups):
                            groupParameters.groups.append( groupName )
                            groupParameters.groupsMachines[groupName] = []
                            groupParameters.groupFileTypes[groupName] = []
                            groupParameters.groupsMembers[groupName] = []
                            groupParameters.groupsProducts[groupName] = []
                            
                            machines = splitLine[2].split(",")
                            
                            for machine in machines:
                                 groupParameters.groupsMachines[groupName].extend( machineParameters.getMachinesAssociatedWith(machine) )

                            groupParameters.groupFileTypes[groupName] = splitLine[3]
                            groupParameters.groupsMembers[groupName].extend( splitLine[4].split(",") )
                            groupParameters.groupsProducts[groupName].extend( splitLine[5].split(",") )
                    
                line = fileHandle.readline()     
                                
        fileHandle.close()            
        
        self.groupParameters = groupParameters

  def getTranslationFileName( language = 'en', moduleAbsPath = 'module' ):
      """
          
          @summary : Returns the filename containing the translation text required 
                     by the specified module for the spcified language.
          
          @Note : Will return "" if language is not supported.
          
          @param language: Language for which we need the translation file.
          
          @param moduleAbsPath: AbsolutePath name of the module for which we need the translation file.
      
          @return : Returns the filename containing the translation text required 
                    by the specified module for the spcified language. 
                    
                    Will return "" if language is not supported.
                      
          
      """
      
      translationfileName = ""
      moduleAbsPath = os.path.realpath(moduleAbsPath) #decodes symlinks.
      try : 
          
          paths = StatsPaths()
          paths.setBasicPaths()
 
          if language == 'en' : 
              correspondingPaths = { paths.STATSBIN : paths.STATSLANGENBIN, paths.STATSDEBUGTOOLS : paths.STATSLANGENBINDEBUGTOOLS \
                                    , paths.STATSTOOLS : paths.STATSLANGENBINTOOLS, paths.STATSWEBPAGESGENERATORS : paths.STATSLANGENBINWEBPAGES \
                                    , paths.STATSLIB : paths.STATSLANGENLIB  }
                       
          elif language == 'fr': 
              correspondingPaths = { paths.STATSBIN : paths.STATSLANGFRBIN, paths.STATSDEBUGTOOLS : paths.STATSLANGFRBINDEBUGTOOLS \
                    , paths.STATSTOOLS : paths.STATSLANGFRBINTOOLS, paths.STATSWEBPAGESGENERATORS : paths.STATSLANGFRBINWEBPAGES \
                    , paths.STATSLIB : paths.STATSLANGFRLIB  } 
          
          
          for key in correspondingPaths.keys():
              correspondingPaths[ key.split("pxStats")[-1:][0]] = correspondingPaths[ key]
          
          modulePath = str(os.path.dirname( moduleAbsPath ) + '/').split("pxStats")[-1:][0]
          moduleBaseName =  str(os.path.basename( moduleAbsPath )).replace( ".py", "" )
          
          #print "modulePath",modulePath
          
          #print "correspondingPaths", correspondingPaths
                     
          translationfileName = correspondingPaths[ modulePath ] + moduleBaseName
          
          #print translationfileName
          
      except Exception, instance:
          print instance

def giveOutPermissionsToFolders( currentlyUsedLanguages ):
    """    
        @summary : opens up permissions to folders that 
                   might be required by the web user.
                   
        @param currentlyUsedLanguages: Languages currently set to be 
                                       displayed in the web interface
    
    """
    
    for language in currentlyUsedLanguages:
        
        _ = LanguageTools.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, language)
        
        paths = StatsPaths()        
        paths.setPaths(language)        
        
        pathsToOpenUp = []
        
        pathsToOpenUp.append( paths.STATSLOGGING)
        pathsToOpenUp.append( paths.STATSPICKLES )
        
        pathsToOpenUp.append( paths.STATSDB)
        
        pathsToOpenUp.append( paths.STATSCURRENTDB )        
        pathsToOpenUp.append( paths.STATSCURRENTDB + _("bytecount") )
        pathsToOpenUp.append( paths.STATSCURRENTDB + _("errors")  )
        pathsToOpenUp.append( paths.STATSCURRENTDB + _("filecount") )
        pathsToOpenUp.append( paths.STATSCURRENTDB + _("filesOverMaxLatency"))
        pathsToOpenUp.append( paths.STATSCURRENTDB + _("latency"))      
        
        pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES)
        pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("rx") )
        pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("tx") )
        pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("totals") )        
        
        pathsToOpenUp.append( paths.STATSDBBACKUPS )
        pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("rx") )
        pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("tx") )
        pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("totals") )    
                
        pathsToOpenUp.append( paths.STATSGRAPHS )
        pathsToOpenUp.append( paths.STATSGRAPHS +_("others/"))
        pathsToOpenUp.append( paths.STATSGRAPHS +_("others/") + "gnuplot/")
        pathsToOpenUp.append( paths.STATSGRAPHS +_("others/") + "rrd/")
    
        pathsToOpenUp.append( paths.STATSWEBPAGESHTML + "/popUps/")
        
        for path in pathsToOpenUp:
            if not os.path.isdir(path):
                os.makedirs(path, 0777)
            commands.getstatusoutput( "chmod 0777 %s" %path )
            commands.getstatusoutput( "chmod 0777 %s/*" %path )