def init(my):
        web = WebContainer.get_web()
        
        my.is_refresh = my.kwargs.get('is_refresh')
        my.search_type = my.kwargs.get('search_type')
        if not my.search_type:
            my.search_type = web.get_form_value('search_type_filter')
        my.close_cbfn = my.kwargs.get('close_cbfn')

        my.web_url = web.get_form_value("web_url")
        my.file_path = None
        if my.web_url:
            import urllib2
            response = urllib2.urlopen(my.web_url)
            csv = response.read()
            my.file_path = "/tmp/test.csv"
            f = open(my.file_path, 'w')
            f.write(csv)
            f.close()

        if not my.file_path:
            my.file_path =  web.get_form_value('file_path')


        if not my.file_path:
            file_name =  web.get_form_value('file_name')
            ticket =  web.get_form_value('html5_ticket')
            if not ticket:
                ticket =  web.get_form_value('csv_import|ticket')
                
            if file_name:
                # this is treated the same in FileUplaod class
                file_name = File.get_filesystem_name(str(file_name))
                my.file_path = '%s/%s' %(web.get_upload_dir(ticket=ticket), file_name)

    def _read_ref_file(my):
        '''read the reference file containing extra node information'''
        dir = my.get_upload_dir()
        xml = Xml()
        key = my.sobject.get_code()

        # make this filename good for the file system
        filename = File.get_filesystem_name(key)
        xml.read_file( "%s/%s-ref.xml" % (dir,filename) )
        return xml

 def _get_unique_filename(my):
     filename = my.file_object.get_full_file_name()
     # find if this filename has been used for this project
     file = File.get_by_filename(filename, skip_id=my.file_object.get_id())
     if file:
         root, ext = os.path.splitext(filename)
         parts = [root]
         filename = my.add_ending(parts, auto_version=True)
         return filename
     else:
         return None

 def _get_file_obj(my, snapshot, type='main'):
     if type:
         xpath = "snapshot/file[@type='%s']" %type
     else:
         xpath = "snapshot/file[@type]"
     xml = snapshot.get_xml_value('snapshot')
     node = xml.get_node(xpath)
     file = None
     if node is not None:
         file_code = Xml.get_attribute(node, "file_code")
         file = File.get_by_code(file_code)
     return file

    def _init_file_object(my):
        '''initialize the file object. Some fields are still empty before checkin postprocess'''

        # if set externally already, skip and return
        if my._file_object:
            return
        file_type = my.get_file_type()
        if file_type and my.snapshot:
            # get the file_object
            file_code = my.snapshot.get_file_code_by_type(file_type)
            from pyasm.biz import File 
            my._file_object = File.get_by_code(file_code)

    def add_ref_by_file_path(my, file_path, type='ref', node_name='', tag='main'):
        '''add a reference based on the file name.  If the file is unique, then
        a reference can be found based on the file name'''
        from pyasm.biz import File
        filename = os.path.basename(file_path)
        file = File.get_by_filename(filename, padding=4)
        if not file:
            Environment.add_warning("Unknown File Reference", "File reference [%s] does not exist in database" % file_path)
            my.add_unknown_ref(file_path)

            return
        else:
            snapshot_code = file.get_value("snapshot_code")
            return my.add_ref_by_snapshot_code(snapshot_code, type=type, node_name=node_name, tag=tag)

    def get_parser_by_path(cls, path):
        ext = File.get_extension(path)

        parser_str = None

        if ext in File.VIDEO_EXT:
            if HAS_FFMPEG:
                parser_str = "FFMPEG"
            else:
                parser_str = "PIL"
        else:
            if HAS_IMAGEMAGICK:
                parser_str = "ImageMagick"
            elif HAS_PIL:
                parser_str = "PIL"
            elif HAS_EXIF:
                parser_str = "EXIF"
            elif HAS_FFMPEG:
                parser_str = "FFMPEG"
        return cls.get_parser(parser_str, path)

    def execute(my):


        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir


        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)



        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}


        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """
 
    

    
        if not SearchType.column_exists(search_type, "name"):
            raise TacticException('The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.')

        for count, filename in enumerate(filenames):

            # first see if this sobjects still exists
            search = Search(search_type)
            search.add_filter("name", filename)
            if relative_dir and search.column_exists("relative_dir"):
                search.add_filter("relative_dir", relative_dir)
            sobject = search.get_sobject()

            # else create a new one
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)



            # extract metadata
            file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e


            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
#.........这里部分代码省略.........

    def execute(my):

        # extra data from the file check-in

        commit = my.kwargs.get("commit")
        if commit in ['false', False]:
            commit = False
        else:
            commit = True

        # metadata can only be associated with a single file on the
        # snapshot.
        files = my.kwargs.get("files")
        file_objects = my.kwargs.get("file_objects")

        snapshot_metadata = None

        parser_type = my.kwargs.get("parser")

        for i, file in enumerate(files):
            path = file
            ext = File.get_extension(path)
            file_object = file_objects[i]

            if not os.path.exists(path):
                continue
            elif parser_type != "auto" and parser_type != "true":
                pass
            elif ext in File.VIDEO_EXT:
                parser_type = "FFMPEG"
            elif ext in File.NORMAL_EXT:
                continue
            else:
                if HAS_IMAGEMAGICK:                    
                    parser_type = "ImageMagick" 
                else:
                    parser_type = "PIL"
            metadata = {}

            if parser_type == "FFMPEG":
                parser = FFProbeMetadataParser(path=path)
                metadata = parser.get_metadata()
            elif parser_type == "ImageMagick":
                parser = ImageMagickMetadataParser(path=path)
                metadata = parser.get_metadata()
            else:
                parser = PILMetadataParser(path=path)
                metadata = parser.get_metadata()

            metadata['__parser__'] = parser_type

            """

            if os.path.exists(path):
                try:
                    parser = ImageMagickMetadataParser(path=path)
                    metadata = parser.get_metadata()
                except:
                    try:
                        #parser = PILMetadataParser(path=path)
                        parser = FFProbeMetadataParser(path=path)
                        metadata = parser.get_metadata()
                    except:
                        metadata = {}
            else:
                metadata = {}
            """

            # add some default metadata
            basename = os.path.basename(path)
            path = file_object.get_lib_path()
            dirname = os.path.dirname(path)
            parts = basename.split(".")
            ext = parts[-1]
            try:
                # occasionally, people will put frame numbers as the last
                # part
                ext = int(ext)
                ext = parts[-2]
            except:
                pass

            metadata['Ext'] = ext
            metadata['Basename'] = basename
            metadata['Dirname'] = dirname


            if metadata and not file_object.get_value("metadata"):
                file_object.add_metadata(metadata, replace=True)
                searchable = my.get_searchable(metadata)
                file_object.set_value("metadata_search", searchable)

                file_object.commit()


                if i == 0:
                    snapshot = my.kwargs.get("snapshot")
                    snapshot.add_metadata(metadata, replace=True)
                    if commit:
                        snapshot.commit()

    def handle_dir_or_item(my, item_div, dirname, basename):
        spath = "%s/%s" % (dirname, basename)
        fspath = "%s/%s" % (dirname, File.get_filesystem_name(basename))

        md5 = my.md5s.get(fspath)
        changed = False
        context = None
        error_msg = None
        snapshot = None
        file_obj = my.checked_in_paths.get(fspath)
        if not file_obj:
            if fspath.startswith(my.base_dir):
                rel = fspath.replace("%s/" % my.base_dir, "")
                file_obj = my.checked_in_paths.get(rel)


        if file_obj != None:

            snapshot_code = file_obj.get_value("snapshot_code")
            snapshot = my.snapshots_dict.get(snapshot_code)
            if not snapshot:
                # last resort
                snapshot = file_obj.get_parent()

            if snapshot:
                context = snapshot.get_value("context")
                item_div.add_attr("spt_snapshot_code", snapshot.get_code())

                snapshot_md5 = file_obj.get_value("md5")
                item_div.add_attr("spt_md5", snapshot_md5)
                item_div.add_attr("title", "Checked-in as: %s" % file_obj.get_value("file_name"))

                if md5 and md5 != snapshot_md5:
                    item_div.add_class("spt_changed")
                    changed = True
            else:
                error_msg = 'snapshot not found'

            


        status = None
        if file_obj != None:
            if changed:
                check = IconWdg( "Checked-In", IconWdg.ERROR, width=12 )
                status = "changed"
            else:
                check = IconWdg( "Checked-In", IconWdg.CHECK, width=12 )
                status = "same"
            item_div.add_color("color", "color", [0, 0, 50])

        else:
            check = None
            item_div.add_style("opacity: 0.8")
            status = "unversioned"



        if check:
            item_div.add(check)
            check.add_style("float: left")
            check.add_style("margin-left: -16px")
            check.add_style("margin-top: 4px")


        # add the file name
        filename_div = DivWdg()
        item_div.add(filename_div)
        filename_div.add(basename)
        file_info_div = None
        if snapshot and status != 'unversioned':
            file_info_div = SpanWdg()
            filename_div.add(file_info_div)

        if error_msg:
            filename_div.add(' (%s)'%error_msg)
        filename_div.add_style("float: left")
        filename_div.add_style("overflow: hidden")
        filename_div.add_style("width: 65%")


        # DEPRECATED
        from pyasm.widget import CheckboxWdg, TextWdg, SelectWdg, HiddenWdg
        checkbox = CheckboxWdg("check")
      

        checkbox.add_style("display: none")
        checkbox.add_class("spt_select")
        checkbox.add_style("float: right")
        checkbox.add_style("margin-top: 1px")
        item_div.add(checkbox)

        subcontext_val = ''
        cat_input = None
        is_select = True
        if my.context_options:
            context_sel = SelectWdg("context")
            context_sel.add_attr('title', 'context')
            context_sel.set_option("show_missing", False)
            context_sel.set_option("values", my.context_options)
#.........这里部分代码省略.........

    def handle_system_commands(my, snapshot, files, file_objects, mode, md5s, source_paths=[], file_sizes=[]):
        '''move the tmp files in the appropriate directory'''

        # if mode is local then nothing happens here
        if mode == 'local':
            return

        sobject = snapshot.get_sobject()

        # inplace mode does not move the file.  It just registers the file
        # object
        if mode == 'inplace':
            for i, file in enumerate(files):
                file_object = file_objects[i]
                to_name = file_object.get_full_file_name()
                to_path = file

                # This is handled in create_file_types
                #file_type = snapshot.get_type_by_file_name(to_name)
                #file_object.set_value('type', file_type)


                if not os.path.isdir(to_path):
                    md5_checksum = None
                    if md5s:
                        md5_checksum = md5s[i]
                    if not md5_checksum:
                        md5_checksum = File.get_md5(to_path)

                    if md5_checksum:
                        file_object.set_value("md5", md5_checksum)

                file_object.commit(triggers=False)
            return
            
   
        for i, file in enumerate(files):

            file_object = file_objects[i]
            to_name = file_object.get_full_file_name()
            file_type = snapshot.get_type_by_file_name(to_name)

            lib_dir = snapshot.get_lib_dir(file_type=file_type, file_object=file_object)
            # it should have been created in postprocess_snapshot
            System().makedirs(lib_dir)

            to_path = "%s/%s" % (lib_dir, to_name )

            #print "path: ", i, files[i]
            #print to_path, os.path.exists(to_path)

            # first make sure that the to path does not exist, if so, just skip
            if os.path.exists(to_path) and mode not in ['inplace','preallocate']:
                raise CheckinException('This path [%s] already exists'%to_path) 



            # add the file
            try:

                # inplace undo used to not touch the file, 
                # now it will be moved to cache on undo
                io_action = True
                if mode in ['preallocate']:
                    io_action = False

                
                if mode == 'move':
                    FileUndo.move( source_paths[i], to_path )
                #elif mode == 'copy': # was free_copy
                   
                    #FileUndo.create( source_paths[i], to_path, io_action=io_action )
                # make it look like the files was created in the repository
                else: # mode ='create'
                    
                    md5 = file_object.get_value("md5")
                    st_size = file_object.get_value("st_size")
                    rel_dir = file_object.get_value("relative_dir")
                    if mode == 'copy':
                        io_action = 'copy'
                        src_path = source_paths[i]
                    else:
                        src_path = files[i]

                    file_name = to_name
                    rel_path = "%s/%s" % (rel_dir, file_name)
                    
                    FileUndo.create( src_path, to_path, io_action=io_action, extra={ "md5": md5, "st_size": st_size, "rel_path": rel_path } )


            except IOError, e:
                raise CheckinException('IO Error occurred. %s' %e.__str__())

            # check to see that the file exists.
            if not os.path.exists( to_path ):
                if mode in ["inplace", "preallocate"]:
                    raise CheckinException("File not found in repo at [%s]" % to_path )
                else:
                    raise CheckinException("Failed move [%s] to [%s]" % \
                (files[i], to_path) )
#.........这里部分代码省略.........

    def _test_inplace_checkin(self):

        # create a new test.txt file
        tmp_dir = Environment.get_tmp_dir()
        dir = "%s/temp" % tmp_dir
        if not os.path.exists(dir):
            os.makedirs(dir)
        file_path = "%s/test_inplace.txt" % dir

        if os.path.exists(file_path):
            os.unlink(file_path)

        file = open(file_path, 'w')
        file.write("whatever")
        file.close() 


        # inplace checkin: tell tactic that this is the correct path
        mode = "inplace"
        base_dir = tmp_dir
        context = "inplace"
        checkin = FileCheckin(self.person, file_path, context=context, mode=mode)
        checkin.execute()
        snapshot = checkin.get_snapshot()

        file_code = snapshot.get_file_code_by_type("main")
        file_object = File.get_by_code(file_code)

        relative_dir = file_object.get_value("relative_dir")
        # The relative dir is empty if the file is outside the repository
        self.assertEquals("", relative_dir)

        lib_dir = snapshot.get_lib_dir(file_type="main")
        file_name = snapshot.get_file_name_by_type("main")
        lib_path = "%s/%s" % (lib_dir, file_name)
        self.assertEquals( True, os.path.exists(lib_path) )
        self.assertEquals( file_path, lib_path)




        # check in a file alredy in the repository
        asset_dir = Config.get_value("checkin", "asset_base_dir", sub_key="default")
        file_path2 = "%s/unittest/text.txt" % asset_dir

        file = open(file_path2, 'w')
        file.write("whatever")
        file.close() 

        checkin = FileCheckin(self.person, file_path2, context=context, mode=mode)
        checkin.execute()
        snapshot = checkin.get_snapshot()

        file_code = snapshot.get_file_code_by_type("main")
        file_object = File.get_by_code(file_code)

        # check that the relative dir is as expected
        relative_dir = file_object.get_value("relative_dir")
        self.assertEquals( relative_dir, "unittest" )

        # check that the path returned correctly
        lib_path = snapshot.get_path_by_type("main")
        self.assertEquals( file_path2, lib_path )

        if os.path.exists(file_path):
            os.unlink(file_path)
        if os.path.exists(file_path2):
            os.unlink(file_path2)

    def init(my):

        WebContainer.register_cmd("pyasm.widget.AnnotateCbk")

        sobject = my.get_current_sobject()

        if not sobject:
            if not my.__dict__.has_key("search_type"):
                web = WebContainer.get_web()
                my.search_type = web.get_form_value("search_type")
                my.search_id = web.get_form_value("search_id")

            if not my.search_type:
                my.add("No search type")
                return


            search = Search(my.search_type)
            search.add_id_filter(my.search_id)
            sobject = search.get_sobject()


        snapshot = Snapshot.get_latest_by_sobject(sobject)

        # TODO:
        # this is a bit klunky
        snapshot_xml = snapshot.get_xml_value("snapshot")
        file_code = snapshot_xml.get_value("snapshot/file[@type='web']/@file_code")
        file = File.get_by_code(file_code)
        web_dir = snapshot.get_web_dir()
        path = "%s/%s" % (web_dir, file.get_full_file_name() )


        # add the annotate js object
        script = HtmlElement.script("annotate = new Annotate()")
        my.add(script)


        # add the image

        my.add("<h3>Image Annotations</h3>")

        width = 600
        img = HtmlElement.img(path)
        img.add_style("position: absolute")
        img.set_id("annotate_image")
        img.add_style("left: 0px")
        img.add_style("top: 0px")
        img.add_style("opacity: 1.0")
        img.add_style("z-index: 1")
        img.add_style("width", width)
        img.add_event("onmouseup", "annotate.add_new(event)")
        my.add(img)



        # test 
        version = snapshot.get_value("version")
        if version != 1:
            last_version = version - 1
            snapshot = Snapshot.get_by_version( \
                my.search_type, my.search_id, version=last_version )

            snapshot_xml = snapshot.get_xml_value("snapshot")
            file_code = snapshot_xml.get_value("snapshot/file[@type='web']/@file_code")
            file = File.get_by_code(file_code)
            web_dir = snapshot.get_web_dir()
            path = "%s/%s" % (web_dir, file.get_full_file_name() )

            img = HtmlElement.img(path)
            img.set_id("annotate_image_alt")
            img.add_style("position: absolute")
            img.add_style("left: 0px")
            img.add_style("top: 0px")
            img.add_style("opacity: 1.0")
            img.add_style("z-index: 0")
            img.add_style("width", width)
            img.add_event("onmouseup", "annotate.add_new(event)")
            my.add(img)


            #script = HtmlElement.script("align_element('%s','%s')" % \
            #    ("annotate_image", "annotate_image_alt") )
            #my.add(script)

            div = DivWdg()
            div.add_style("position: absolute")
            div.add_style("left: 620")
            div.add_style("top: 300")
            my.add(div)

            button = IconButtonWdg("Switch", IconWdg.REFRESH, True)
            button.add_event("onclick", "annotate.switch_alt()")
            div.add(button)

            button = IconButtonWdg("Opacity", IconWdg.LOAD, True)
            button.add_event("onclick", "annotate.set_opacity()")
            div.add(button)


#.........这里部分代码省略.........