def start(self):
        """
          Starts the server. Call this after the handlers have been set.
        """

        import utils
        utils.makedirs(SOCKET_PATH)
        serversock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sockfile = os.path.join(SOCKET_PATH, DISPLAY)
        try:
            serversock.bind(sockfile)
        except socket.error:
            try:
                os.remove(sockfile)
            except OSError:
                log("Couldn't remove dead socket file \"%s\"." % sockfile)
                sys.exit(1)
            try:
                serversock.bind(sockfile)
            except socket.error:
                log("Couldn't bind to socket. Aborting.")
                sys.exit(1)

        serversock.listen(3)
        serversock.setblocking(False)

        atexit.register(self.__shutdown, serversock, sockfile)

        gobject.timeout_add(100, self.__server_handler, serversock, sockfile)

    def generate(self, in_base_path, out_base_path):
        self.in_base_path = in_base_path;
        self.out_base_path = out_base_path;
        
        utils.makedirs(out_base_path);                
        imgutils.init(in_base_path);
        utils.init(in_base_path);
        
        self.blog = Struct(json.load(utils.open_file(self.in_base_path + "/blog.json")));

        # copy static content
        cmd = "cp -rf " + in_base_path + "/static/* " + out_base_path;
        print("copy static content: " + cmd)
        proc = utils.execute_shell(cmd);
        
        # 'dynamic' content
        for c in ["sticky", "posts"]:
            setattr(self, c, []);
            self.generate_content(c);        
        
        # home page
        self.generate_home();

        # feed
        self.generate_feed();

    def create(self, source_path):
        if self.exists():
            raise Error("deck `%s' already exists" % self.name)

        if not isdir(source_path):
            raise Error("source `%s' is not a directory" % source_path)

        if is_deck(source_path):
            source = Deck(source_path)
            if source.storage.paths.path != self.paths.path:
                raise Error("cannot branch a new deck from a deck in another directory")

            levels = source.storage.get_levels()
            makedirs(self.stack_path)
            os.symlink(levels[0], join(self.stack_path, "0"))

            # we only need to clone last level if the deck is dirty
            if source.is_dirty():
                levels = levels[1:]
                source.add_level()
            else:
                levels = levels[1:-1] 
                
            for level in levels:
                level_id = basename(level)
                self.add_level(level_id)

            self.mounts_id = source.storage.mounts_id
        else:
            makedirs(self.stack_path)
            os.symlink(realpath(source_path), join(self.stack_path, "0"))

        self.add_level()

    def init_create(cls, source_path, deck_path):
        if exists(deck_path) and \
               (not isdir(deck_path) or len(os.listdir(deck_path)) != 0):
            raise Error("`%s' exists and is not an empty directory" % deck_path)

        storage = DeckStorage(deck_path)
        makedirs(deck_path)
        storage.create(source_path)

        if os.geteuid() == 0:
            mounts = Mounts(source_path)
            if len(mounts):
                id = deckcache.new_id()
                deckcache.blob(id, "w").write(str(mounts))

                storage.mounts_id = id
            elif storage.mounts_id:
                # make an identical copy of the blob
                newid = deckcache.new_id()
                print >> deckcache.blob(newid, "w"), deckcache.blob(storage.mounts_id).read()
                storage.mounts_id = newid

        deck = cls(deck_path)
        deck.mount()

        return deck

def _create_store(prefix):

    r1 = random.randrange(0xffffffL, 0x7fffffffL)
    r2 = random.randrange(0xffffffL, 0x7fffffffL)
    fname1 = ".!pwstore" + str(r1)
    fname2 = ".!" + str(r2)
    utils.makedirs(os.path.join(prefix, fname1))
    path = os.path.join(prefix, fname1, fname2)

    # is this equivalent to the commented code below? i think so
    # chars = [chr(a) for a in range(256)*16 if a!=26]
    # better :D
    chars = [chr(a) for a in range(256) if a!=26] * 16
    #chars = []
    #for i in xrange(4096):
    #    a = i % 256
    #    if (a == 26): continue
    #    chars.append(chr(a))
    #end for

    data = ""
    while chars:
        index = random.randrange(len(chars))
        c = chars.pop(index)
        data += c
    #end while

    fd = open(path, "w")
    fd.write(data)
    fd.close()
    os.chmod(path, 0400)

    def create_gdk_pixbuf_loaders_setup(self):
        modulespath = self.project.get_bundle_path("Contents/Resources/lib/gtk-2.0/" +
                                                   "${pkg:gtk+-2.0:gtk_binary_version}/"+
                                                   "loaders")
        modulespath = utils.evaluate_pkgconfig_variables (modulespath)

        cmd = "GDK_PIXBUF_MODULEDIR=" + modulespath + " gdk-pixbuf-query-loaders"
        f = os.popen(cmd)

        path = self.project.get_bundle_path("Contents/Resources/etc/gtk-2.0")
        utils.makedirs(path)
        fout = open(os.path.join(path, "gdk-pixbuf.loaders"), "w")

        prefix = "\"" + self.project.get_bundle_path("Contents/Resources")

        for line in f:
            line = line.strip()
            if line.startswith("#"):
                continue

            # Replace the hardcoded bundle path with @executable_path...
            if line.startswith(prefix):
                line = line[len(prefix):]
                line = "\"@executable_path/../Resources" + line
            fout.write(line)
            fout.write("\n")
        fout.close()

def setup_spy():
    # Nothing to do if already in LD_PRELOAD
    if "police-hook.so" in os.environ.get("LD_PRELOAD", ""):
        return

    # Setup environment
    os.environ["LD_PRELOAD"] = "police-hook.so"
    ldlibpath = [
        os.environ.get("LD_LIBRARY_PATH", ""),
        os.path.join(dragon.POLICE_HOME, "hook", "lib32"),
        os.path.join(dragon.POLICE_HOME, "hook", "lib64"),
    ]
    os.environ["LD_LIBRARY_PATH"] = ":".join(ldlibpath)
    os.environ["POLICE_HOOK_LOG"] = dragon.POLICE_SPY_LOG
    os.environ["POLICE_HOOK_RM_SCRIPT"] = os.path.join(
            dragon.POLICE_HOME, "police-rm.sh")
    os.environ["POLICE_HOOK_NO_ENV"] = "1"

    # Setup directory
    utils.makedirs(os.path.dirname(dragon.POLICE_SPY_LOG))

    # Keep previous logs. Spy will append to existing if any
    # If a fresh spy is required, user shall clean it before
    if not os.path.exists(dragon.POLICE_SPY_LOG):
        fd = open(dragon.POLICE_SPY_LOG, "w")
        fd.close()

    def __init__(self, path, architecture, cpp_opts=[]):
        self.base = path
        makedirs(self.base)

        cpp_opts.append("-I" + self.base)
        cpp_opts.append("-D%s=y" % architecture.upper())
        self.cpp_opts = cpp_opts

    def __init__(self, in_base_path, in_path, out_path):
        self.directory = os.path.basename(in_path);
        self.in_path = in_path;
        self.out_path = out_path + "/" + self.directory;
        self.small_dirname = "small";
        self.date_str = "";
        utils.makedirs(self.out_path + "/"  + self.small_dirname);

        try:
            self.date = time.strptime(self.directory[:10], "%Y-%m-%d");
        except ValueError as e:
            self.date = None;
            pass;

        self.attachments = [];

        for f in os.listdir(in_path):
            f_full = in_path + "/" + f;
            if (os.path.isfile(f_full)):
                if (f[-4:] == ".mml"):
                    if (hasattr(self, "page_path")):
                        utils.fatal("multipe content found in " + in_path);
                    self.page_path = f_full;
                    self.processHeader();
                else:
                    self.attachments.append(f);
        self.processAttachments();                

 def convert(self, acquisition, participantName):
     cmdTpl = self.mrconvertTpl[acquisition.getDataType()]
     dicomsDir = acquisition.getDicomsDir()
     outputDir = acquisition.getOutputDir()
     utils.makedirs(outputDir)
     outputWithoutExt = acquisition.getOutputWithoutExt()
     call(cmdTpl.format(dicomsDir, outputWithoutExt), shell=True)

    def install_addon(self, path):
        """Installs the given addon or directory of addons in the profile."""

        # if the addon is a directory, install all addons in it
        addons = [path]
        if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
            addons = [os.path.join(path, x) for x in os.listdir(path)]
           
        for addon in addons:
            if addon.endswith('.xpi'):
                tmpdir = tempfile.mkdtemp(suffix = "." + os.path.split(addon)[-1])
                compressed_file = zipfile.ZipFile(addon, "r")
                for name in compressed_file.namelist():
                    if name.endswith('/'):
                        makedirs(os.path.join(tmpdir, name))
                    else:
                        if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
                            makedirs(os.path.dirname(os.path.join(tmpdir, name)))
                        data = compressed_file.read(name)
                        f = open(os.path.join(tmpdir, name), 'wb')
                        f.write(data) ; f.close()
                addon = tmpdir

            # determine the addon id
            addon_id = Profile.addon_id(addon)
            assert addon_id is not None, "The addon id could not be found: %s" % addon
 
            # copy the addon to the profile
            addon_path = os.path.join(self.profile, 'extensions', addon_id)
            copytree(addon, addon_path, preserve_symlinks=1)
            self.addons_installed.append(addon_path)

    def get_resources(self, pattern=None):
        ''' fetch resources (images, css, javascript, video, ...)
        '''
        if not pattern:
            raise RuntimeError('Error! The pattern is not defined')

        pattern = re.compile(pattern)

        if self.cached and self.path:
            cache_dir = os.path.join(self.path, 'files/')
            fetcher = self._get_fetcher(headers=self.headers, cached=True, cache_dir=cache_dir)
        else:
            fetcher = self._get_fetcher(headers=self.headers)

        for link in self.content.links():
            if pattern and pattern.search(link):

                offline_filename = os.path.join(self.path, utils.offline_link(link))
                utils.makedirs(offline_filename)

                response = fetcher.fetch(link, to_file=offline_filename)
                response.pop(u'content')
                url = response.pop(u'url')
                if url is not self.metadata['resources']:
                    self.metadata['resources'][url] = response
                    response['filename'] = response['filename'].replace(self.path, '')
                    self.metadata['resources'][url]['filename'] = response['filename']

    def save(self, filename='index', metadata=True, resources=True):
        ''' save metadata and content

        filename - defines just filename for three files: 
            - <filename>.html 
            - <filename>.metadata 
            - <filename>.resources

        Only the first one is HTML file, the rest of files are JSON files

        metadata - if True, HTTP response information will be stored into .metadata file
        resources - If True, resources metadata will be stores into .resources file 
        '''
        if not self.path:
            raise RuntimeError('Error! The path for storing content is not defined')

        if not os.path.exists(self.path):
            utils.makedirs(self.path)

        # save content metadata
        if metadata:
            utils.save(os.path.join(self.path, '%s.metadata' % filename), 
                    content=unicode(json.dumps(self.metadata['headers'], indent=4, sort_keys=True)))

        # save resources metadata
        if resources and self.metadata['resources']:
            utils.save(os.path.join(self.path, '%s.resources' % filename), 
                    content=unicode(json.dumps(self.metadata['resources'], indent=4, sort_keys=True)))

        # save content
        offline_content = copy.deepcopy(self.content)
        offline_links = dict([(url, self.metadata['resources'][url]['filename']) for url in self.metadata['resources']])
        offline_content.make_links_offline(offline_links=offline_links)
        offline_content.save(os.path.join(self.path, '%s.html' % filename))

    def __init__(self, basedir, serial, name):
        self.path = os.path.join(basedir, str(serial))
        self.serial = serial
        self.name = name

        # check profile version, if not a new device
        if os.path.isdir(self.path):
            if self.version < self.PROFILE_VERSION:
                raise Device.ProfileVersionMismatch("Version on disk is too old")
            elif self.version > self.PROFILE_VERSION:
                raise Device.ProfileVersionMismatch("Version on disk is too new")

        # create directories
        utils.makedirs(self.path)
        for directory in DIRECTORIES:
            directory_path = os.path.join(self.path, directory)
            utils.makedirs(directory_path)

        # write profile version (if none)
        path = os.path.join(self.path, self.PROFILE_VERSION_FILE)
        if not os.path.exists(path):
            with open(path, 'wb') as f:
                f.write(str(self.PROFILE_VERSION))

        # write device name
        path = os.path.join(self.path, self.NAME_FILE)
        if not os.path.exists(path):
            with open(path, 'w') as f:
                f.write(self.name)

def write_tiff( hf, filename):
    #Make the directory path if it doesn't exist
    dirname  = os.path.dirname(filename)
    if not os.path.exists(dirname):
       makedirs(dirname)
       
    import struct
    #Get the GTiff driver
    driver = gdal.GetDriverByName("GTiff")

    #Create the dataset
    ds = driver.Create(filename, hf.width, hf.height, 1, gdal.GDT_Float32)
            
    band = ds.GetRasterBand( 1 )
    
    for r in range(0, hf.height):
        scanline = []
        for c in range(0, hf.width):
            height = hf.get_height(c,r)
            scanline.append( height )
        packed_data = struct.pack('f'*len(scanline), *scanline)
        line = hf.height-r-1
        band.WriteRaster(0, line, band.XSize, 1, packed_data, buf_type=gdal.GDT_Float32)
    ds.FlushCache()
    ds = None

def build_xsl_reports( 
          locate_root_dir
        , tag
        , expected_results_file
        , failures_markup_file
        , comment_file
        , results_dir
        , result_file_prefix
        , dont_collect_logs = 0
        , reports = report_types
        , warnings = []
        , user = None
        , upload = False
        ):

    ( run_date ) = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.gmtime() )

    root_paths.append( locate_root_dir )
    root_paths.append( results_dir )
    
    bin_boost_dir = os.path.join( locate_root_dir, 'bin', 'boost' )
    
    output_dir = os.path.join( results_dir, result_file_prefix )
    utils.makedirs( output_dir )
    
    if expected_results_file != '':
        expected_results_file = os.path.abspath( expected_results_file )
    else:
        expected_results_file = os.path.abspath( map_path( 'empty_expected_results.xml' ) )


    extended_test_results = os.path.join( output_dir, 'extended_test_results.xml' )
        
    execute_tasks(
          tag
        , user
        , run_date
        , comment_file
        , results_dir
        , output_dir
        , reports
        , warnings
        , extended_test_results
        , dont_collect_logs
        , expected_results_file
        , failures_markup_file
        )

    if upload:
        upload_dir = 'regression-logs/'
        utils.log( 'Uploading  results into "%s" [connecting as %s]...' % ( upload_dir, user ) )
        
        archive_name = '%s.tar.gz' % result_file_prefix
        utils.tar( 
              os.path.join( results_dir, result_file_prefix )
            , archive_name
            )
        
        utils.sourceforge.upload( os.path.join( results_dir, archive_name ), upload_dir, user )
        utils.sourceforge.untar( os.path.join( upload_dir, archive_name ), user, background = True )

    def _import_path(self, src_path):
        src_path_sha = sha1_for_path(src_path)
        dst_path = self._path_for_file_with_sha(src_path, src_path_sha)
        dst_path_gz = dst_path+'.gz'

        # TODO: this is lame
        if os.path.exists(dst_path):
            return (dst_path, os.path.getsize(dst_path))
        elif os.path.exists(dst_path_gz):
            return (dst_path_gz, os.path.getsize(dst_path_gz))

        # gzip the file first, and see if it passes the compression
        # threshhold

        makedirs(os.path.dirname(dst_path))
        mygzip(src_path, dst_path_gz)
        src_size = os.path.getsize(src_path)
        dst_gz_size = os.path.getsize(dst_path_gz)
        if float(dst_gz_size) / src_size <= self.config.gzip_threshhold:
            final_dst_path = dst_path_gz
            final_dst_size = dst_gz_size
        else:
            final_dst_path = dst_path
            final_dst_size = src_size
            copyfile(src_path, dst_path)
            os.remove(dst_path_gz)

        logging.info("Imported %s --> %s" % (src_path, final_dst_path))
        return (final_dst_path, final_dst_size)

    def save_lyric(self, data, sid, name, artist):
        save_path = os.path.expanduser(config.get("lyrics", "save_lrc_path"))
        if not os.path.exists(save_path):
            utils.makedirs(save_path)

        try:
            lrc = data['lrc']['lyric']
        except:
            lrc = "[00:00.00] "+name+' - '+artist+"\n[99:59:99] No lyric found\n"
        # deepin music 好像不支持tlyric, tlyric应该是英文歌词的翻译
        # 最好能把英文和翻译合并起来
        #try:
            #tlyric = data['tlyric']['lyric']
        #except:
            #tlyric = None
        #try:
            #klyric = data['klyric']['lyric']
        #except:
            #klyric = None
        #lrc_content = klyric or lrc or tlyric
        lrc_content = lrc
        lrc_path = os.path.join(save_path, str(sid)+'.lrc')
        if not os.path.exists(lrc_path) and lrc_content:
            with open(lrc_path, 'w') as f:
                f.write(str(lrc_content))

        return lrc_path

    def copy_path(self, Path):
        _doRecurse = False
        source = self.project.evaluate_path(Path.source)
        if Path.dest:
            dest = self.project.evaluate_path(Path.dest)
        else:
            # Source must begin with a prefix if we don't have a
            # dest. Skip past the source prefix and replace it with
            # the right bundle path instead.
            p = re.compile("^\${prefix(:.*?)?}/")
            m = p.match(Path.source)
            if m:
                relative_dest = self.project.evaluate_path(Path.source[m.end() :])
                dest = self.project.get_bundle_path("Contents/Resources", relative_dest)
            else:
                print "Invalid bundle file, missing or invalid 'dest' property: " + Path.dest
                sys.exit(1)

        (dest_parent, dest_tail) = os.path.split(dest)
        utils.makedirs(dest_parent)

        # Check that the source only has wildcards in the last component.
        p = re.compile("[\*\?]")
        (source_parent, source_tail) = os.path.split(source)
        if p.search(source_parent):
            print "Can't have wildcards except in the last path component: " + source
            sys.exit(1)

        if p.search(source_tail):
            source_check = source_parent
            if Path.recurse:
                _doRecurse = True
        else:
            source_check = source
        if not os.path.exists(source_check):
            print "Cannot find source to copy: " + source
            sys.exit(1)

        # If the destination has a wildcard as last component (copied
        # from the source in dest-less paths), ignore the tail.
        if p.search(dest_tail):
            dest = dest_parent

        if _doRecurse:
            for root, dirs, files in os.walk(source_parent):
                destdir = os.path.join(dest, os.path.relpath(root, source_parent))
                utils.makedirs(destdir)
                for globbed_source in glob.glob(os.path.join(root, source_tail)):
                    try:
                        #                        print "Copying %s to %s" % (globbed_source, destdir)
                        shutil.copy(globbed_source, destdir)
                    except EnvironmentError, e:
                        if e.errno == errno.ENOENT:
                            print "Warning, source file missing: " + globbed_source
                        elif e.errno == errno.EEXIST:
                            print "Warning, path already exits: " + dest
                        else:
                            print "Error %s when copying file: %s" % (str(e), globbed_source)
                            sys.exit(1)

def run_g2s(a_output_dir, a_filename, a_f107, a_f107a, a_ap):
    """
    $g2smodel_dir/$g2smodel -v -d $output_dir -i $F107 $F107a $Ap $ECMWF_ops/$ENfilename >& $logdir/$g2s_logfile
    """
    conf = Conf.get_instance()

    #get info from conf
    log_dir = conf.get('G2S', 'g2s_log_dir')
    #the_dir    = conf.get('G2S', 'dir')
    exe = conf.get('G2S', 'exe')

    log_file = '%s/g2s_%s.log' % (log_dir, os.path.basename(a_filename))

    print("log_dir = %s\n" % (log_dir))
    print("output_dir = %s\n" % (a_output_dir))

    # makedirs
    makedirs(a_output_dir)
    makedirs(log_dir)

    #check if the bin file already exists. If yes don't do anything
    the_date = get_datetime_from_ecmwf_file_name(a_filename)
    bin_file = 'G2SGCSx%s_HWM07.bin' % (the_date.strftime('%Y%m%d%H'))

    # look for bin_file
    if not os.path.exists('%s/%s' % (a_output_dir, bin_file)):

        # substitute command line
        exe = re.sub(r'\${log_file}', log_file, exe)
        exe = re.sub(r'\${output_dir}', a_output_dir, exe)

        # substitue params
        exe = re.sub(r'\${f107}', a_f107, exe)
        exe = re.sub(r'\${f107a}', a_f107a, exe)
        exe = re.sub(r'\${ap}', a_ap, exe)
        # substitute ecmwf file
        exe = re.sub(r'\${ecmwf_file}', a_filename, exe)

        command = '%s' % (exe)

        print('will run [%s]\n' % (command))

        # call the command
        retcode = call(command, shell=True)

        print("retcode %s\n" % (retcode))

        if retcode != 0:
            raise Exception("Cannot run g2s for %s. Check error in %s" %
                            (a_filename, log_file))

        print("G2S bin file %s/%s generated \n" % (a_output_dir, bin_file))

    else:
        print("G2S bin file %s/%s previously generated \n" % (a_output_dir,
                                                              bin_file))

    return '%s/%s' % (a_output_dir, bin_file)