def get_files(my):

        paths = []

        # remember this here for now
        my.files = {}

        my.snapshots = {}


        search_key = my.kwargs.get("search_key")
        search_keys = my.kwargs.get("search_keys")
        if search_key:
            sobject = SearchKey.get_by_search_key(search_key)
            my.sobjects = [sobject]

        if search_keys:
            if isinstance(search_keys, basestring):
                search_keys = search_keys.replace("'", '"')
                search_keys = jsonloads(search_keys)
            my.sobjects = Search.get_by_search_keys(search_keys)

        if not my.sobjects:
            return []

        my.sobject = my.sobjects[0]


        for sobject in my.sobjects:
            sobject_paths = my.get_sobject_files(sobject)
            paths.extend(sobject_paths)

        return paths

    def get_display(my):
        top = my.top
        # top.add_border()

        width = my.kwargs.get("width")
        if not width:
            width = "50px"
        height = my.kwargs.get("height")
        if not height:
            height = "50px"

        class_name = my.kwargs.get("class")
        if class_name:
            top.add_class(class_name)

        top.add_style("width: 100%")
        top.add_border()
        top.add_color("background", "background3")

        font_size = my.kwargs.get("font-size")
        if font_size:
            top.add_style("font-size: %s" % font_size)

        css = my.kwargs.get("css")
        if css:
            css = jsonloads(css)
            for name, value in css.items():
                top.add_style(name, value)

        top.add_style("height: %s" % height)
        top.add_style("width: %s" % width)

        return top

    def __init__(self, data=[]):

        if not data:
            self.data = []
        elif type(data) in types.StringTypes:
            try:
                # optimize the loading of json data
                json_data = Container.get("json_data")
                if json_data == None:
                    json_data = {}
                    Container.put("json_data", json_data)

                self.data = json_data.get(data)
                if self.data == None:
                    self.data = jsonloads(data)
                    json_data[data] = self.data

            except ValueError, e:
                if e.__str__().find('No JSON object') != -1:
                    raise SetupException('Data is not decodable as JSON.')
                # try a straight eval
                self.data = eval(data)

            except Exception as e:
                 if e.__str__().find('cannot parse JSON description') != -1:
                    raise SetupException('Data is not valid JSON.')

    def run(code, kwargs):
        code = jsondumps(code)
        kwargs = jsondumps(kwargs)

        install_dir = tacticenv.get_install_dir()
        cmd = '%s/src/tactic/command/js_cmd.py' % install_dir

        python_exec = Config.get_value("services", "python")
        cmd_list = [python_exec, cmd, code, kwargs]



        import subprocess
        program = subprocess.Popen(cmd_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        ret_val, error = program.communicate()

        lines = []
        start = False
        for line in ret_val.split("\n") :
            if line.startswith("~"*20):
                start = True
                continue

            if not start:
                continue

            lines.append(line)

        value = jsonloads("\n".join(lines))


        return value

    def execute(my):
        plugin = my.sobject

        web = WebContainer.get_web()
        value = web.get_form_value( my.get_input_name() )
        if not value:
            return
        src_search_keys = jsonloads(value)


        manifest = plugin.get_xml_value("manifest")

        top_node = manifest.get_node("manifest")

        for search_key in src_search_keys:
            sobject = SearchKey.get_by_search_key(search_key)

            node = manifest.create_element("sobject")

            # For now, a plugin must contain project specfic entries
            search_type = sobject.get_base_search_type()
            code = sobject.get_value("code")
            manifest.set_attribute(node, "search_type", search_type)
            manifest.set_attribute(node, "code", code)

            #search_key = SearchKey.get_by_sobject(sobject)
            #manifest.set_attribute(node, "search_key", search_key)

            manifest.append_child(top_node, node)

        plugin.set_value("manifest", manifest.to_string() )
        plugin.commit()

    def _test_csv_export(self):
        from tactic.ui.widget import CsvExportWdg
        view = 'table'
        search_type ='sthpw/task'
        search_view = 'auto_search:table'
        #search_view = ''
        simple_search_view = 'simple_search'
        search_class =''
        mode = 'export_matched'
        element_name=  'project_tasks'
        filter =  [{"prefix":"main_body","main_body_enabled":"on","main_body_column":"project_code","main_body_relation":"is","main_body_value":"{$PROJECT}"}, {"prefix":"main_body","main_body_enabled":"on","main_body_column":"search_type","main_body_relation":"is not","main_body_value":"sthpw/project"}]
        
        from pyasm.common import jsondumps, jsonloads
        values  = {'json': jsondumps(filter)}
        element_names = ['code','id','description']
        server = TacticServerStub(protocol='xmlrpc')
        current_project = 'vfx'
        server.set_project(current_project)

        rtn = server.get_widget('tactic.ui.widget.CsvExportWdg', args={'search_type':search_type, 'view':view,\
            'filter': filter, 'element_name': element_name, 'show_search_limit':'false', 'search_limit':-1, 'search_view':search_view, \
            'element_names': element_names, 'mode':mode, 'search_class':search_class, 'simple_search_view':simple_search_view,\
            'init_load_num':-1, 'test':True}, values=values )
        expected_columns = ['code','id','description']
        expected_sql = '''SELECT "sthpw"."public"."task".* FROM "sthpw"."public"."task" WHERE ( "task"."project_code" = \'%s\' AND ( "task"."search_type" != \'sthpw/project\' OR "task"."search_type" is NULL ) ) AND ("task"."s_status" != \'retired\' or "task"."s_status" is NULL) AND ("task"."s_status" != \'retired\' or "task"."s_status" is NULL) AND ("task"."s_status" != \'retired\' or "task"."s_status" is NULL) ORDER BY "task"."search_type", "task"."search_code"'''%current_project

        expr = "@COUNT(sthpw/task['project_code','%s']['search_type','!=','sthpw/project])"%current_project
        expected_count = Search.eval(expr, single=True)
        
        rtn = jsonloads(rtn)
        self.assertEquals(expected_columns, rtn.get('columns'))
        self.assertEquals(expected_sql, rtn.get('sql'))
        self.assertEquals(expected_count, rtn.get('count'))


        mode = 'export_displayed'
        selected_search_keys = ['sthpw/task?id=4385','sthpw/task?id=4386','sthpw/task?id=4387']
        rtn = server.get_widget('tactic.ui.widget.CsvExportWdg', args={'search_type':search_type, 'view':view,\
            'filter': filter, 'element_name': element_name, 'show_search_limit':'false', 'search_limit':-1, 'search_view':search_view, \
            'element_names': element_names, 'mode':mode, 'search_class':search_class, 'simple_search_view':simple_search_view,\
            'init_load_num':-1, 'test':True, 'selected_search_keys': selected_search_keys}, values=values )
        
        expected_count = 3
        rtn = jsonloads(rtn)
        self.assertEquals(expected_columns, rtn.get('columns'))
        self.assertEquals(expected_count, rtn.get('count'))

    def add_data(self, data):
        '''add data dictionary.'''
        if type(data) in types.StringTypes:
            try:
                data = jsonloads(data)
            except ValueError, e:
                if e.__str__().find('No JSON object') != -1:
                    raise SetupException('Data is not decodable as JSON.')

                # try a straight eval
                data = eval(data)

    def preprocess(my):
        my.options = my.get_option('options')
        if my.options:
            try:
                my.group_list = jsonloads(my.options)
            except:
                my.group_list = [{'label': 'Syntax Error', 'context':[]}]
        else:
            my.group_list = [{'label':'default', 'context': []}]

        super(TaskGroupCompletionWdg, my).preprocess()

    def execute(self):
        web = WebContainer.get_web()
        value = web.get_form_value( self.get_input_name() )
        if not value:
            value = self.get_data()

        if not value:
            return


        src_search_keys = jsonloads(value)
        #print "xxx: ", type(src_search_keys), src_search_keys

        # get all fo the sobjects from the search keys
        #src_sobjects = SearchKey.get_by_search_keys(src_search_keys)
        instance_type = self.get_option("instance_type")
        # path is used for self-relating in an instance table
        src_path = self.get_option("path")

        src_sobjects = []
        src_instances = []
        for src_search_key in src_search_keys:
            src_sobject = SearchKey.get_by_search_key(src_search_key)

            if src_sobject.get_base_search_type() == instance_type:
                src_instances.append(src_sobject)
            else:
                src_sobjects.append(src_sobject)


        dst_sobject = self.sobject

        
        # get all of the current instances and see if any were removed
        instances = dst_sobject.get_related_sobjects(instance_type)
        for instance in instances:
            exists = False
            for src_instance in src_instances:
                if src_instance.get_search_key() == instance.get_search_key():
                    exists = True

            if not exists:
                instance.delete()


        # add all the new sobjects
        for src_sobject in src_sobjects:

            instance = SearchType.create(instance_type)
            instance.add_related_connection(src_sobject, dst_sobject, src_path=src_path)

            instance.commit()

    def execute_func(self, js, kwargs={}):

        js = '''
        var func = function() {
            %s
        }
        var ret_val = func();
        ret_val = JSON.stringify(ret_val);
        ''' % js
        ret_val = self.execute(js, kwargs)
        ret_val = jsonloads(ret_val)

        return ret_val

    def execute(my, func_name, args=[], kwargs={}):

        server = TacticServerStub.get()

        if args:
            args = jsonloads(args)
        if kwargs:
            kwargs = jsonloads(kwargs)


        if kwargs:
            # Quirk ... when there is a kwargs, the last args is the kwargs
            if args:
                args.pop()
            call = "server.%s(*args, **kwargs)" % func_name
        else:
            call = "server.%s(*args)" % func_name

        try:
            ret_val = eval(call)
        except Exception, e:
            print "ERROR: ", e
            raise

    def execute(self):

        trigger_sobj = self.get_trigger_sobj()
        data = trigger_sobj.get_value("data")
        #data = """
        #{ "columns": [column1, column2]
        #"""

        data = jsonloads(data)

        column = data.get('column')
        src_status = data.get('src_status')

        

        item = self.get_caller()


        if isinstance(item, SObject):
            if isinstance(item, Task):
                if src_status != None:
                    if item.get_value("status") != src_status:
                        return

                item.set_now(column)
                item.commit()

            #Item can be a note when trigger input is adding or modifying notes
            else:
                process = item.get_value('process')
                expr = '@SOBJECT(parent.sthpw/task["process","%s"])'%process
                tasks = Search.eval(expr, sobjects=[item])

                if tasks:
                    for task in tasks:
                        task.set_now(column)
                        task.commit()

        #item can be a command such as check-in                 
        else:
            if hasattr(item, 'process'):
                process = item.process
                expr = '@SOBJECT(sthpw/task["process","%s"])'%process
                tasks = Search.eval(expr, sobjects=[item.sobject])

                if tasks:
                    for task in tasks:
                        task.set_now(column)
                        task.commit()

    def loads(self, search_type, sobjects_str):
        sobject_list = jsonloads(sobjects_str)

        sobjects = []
        for sobject_dict in sobject_list:
            sobject = SearchType.create(search_type)

            for name, value in sobject_dict.items():
                if value == None:
                    continue
                sobject.set_value(name, value)

            sobjects.append(sobject)

        return sobjects

    def set_data(self, data):
        '''add data dictionary or a JSON string'''
        self.data = []
        # protect against empty spaces/lines from xml
        if isinstance(data, basestring):
            data = data.strip()
        if not data:
            return

        if isinstance(data, basestring):
            try:
                data = data.replace("'", '"')
                data = jsonloads(data)
            except ValueError, e:
                if e.__str__().find('No JSON object') != -1:
                    raise SetupException('Data is not decodable as JSON. [%s]'%data)
                # try a straight eval
                data = eval(data)

    def execute(my):

        trigger_sobj = my.get_trigger_sobj()
        data = trigger_sobj.get_value("data")
        #data = """
        #{ "columns": [column1, column2]
        #"""

        data = jsonloads(data)

        column = data.get('column')
        src_status = data.get('src_status')
        
        task = my.get_caller()
        if task.get_value("status") != src_status:
            return

        task.set_now(column)
        task.commit()

    def get_files(self):

        paths = []

        # remember this here for now
        self.files = {}

        self.snapshots = {}


        search_key = self.kwargs.get("search_key")
        search_keys = self.kwargs.get("search_keys")
        if search_key:
            sobject = SearchKey.get_by_search_key(search_key)
            self.sobjects = [sobject]

        if search_keys:
            if isinstance(search_keys, basestring):
                search_keys = search_keys.replace("'", '"')
                search_keys = jsonloads(search_keys)
            self.sobjects = Search.get_by_search_keys(search_keys)

        if not self.sobjects:
            return []

        self.sobject = self.sobjects[0]


        for sobject in self.sobjects:
            
            if sobject.get_base_search_type() in ['sthpw/task', 'sthpw/note']:
                parent = sobject.get_parent()
                sobject_paths = self.get_sobject_files(parent)
                paths.extend(sobject_paths)

            else:
                sobject_paths = self.get_sobject_files(sobject)
                paths.extend(sobject_paths)


        return paths

    def POST(self):

        from pyasm.web import WebContainer
        web = WebContainer.get_web()

        method = web.get_form_value("method")
        print "method: ", method

        # make sure there are no special characters in there ie: ()
        p = re.compile('^\w+$')
        if not re.match(p, method):
            raise Exception("Mathod [%s] does not exist" % method)


        from tactic_client_lib import TacticServerStub
        server = TacticServerStub.get()

        if not eval("server.%s" % method):
            raise Exception("Mathod [%s] does not exist" % method)


        keys = web.get_form_keys()

        kwargs = {}
        for key in keys:
            if key in ["method", "login_ticket", "password"]:
                continue

            if key == 'kwargs':
                args = web.get_form_value(key)
                args = jsonloads(args)
                for name, value in args.items():
                    kwargs[name] = value
            else:
                kwargs[key] = web.get_form_value(key)

        call = "server.%s(**kwargs)" % method


        return eval(call)

    def execute(my):


        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir


        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)



        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}


        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """
 
    

    
        if not SearchType.column_exists(search_type, "name"):
            raise TacticException('The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.')

        for count, filename in enumerate(filenames):

            # first see if this sobjects still exists
            search = Search(search_type)
            search.add_filter("name", filename)
            if relative_dir and search.column_exists("relative_dir"):
                search.add_filter("relative_dir", relative_dir)
            sobject = search.get_sobject()

            # else create a new one
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)



            # extract metadata
            file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e


            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
#.........这里部分代码省略.........

    def get_display(self):

        top = self.top
        self.set_as_panel(top)
        top.add_class("spt_ingestion_top")
        top.add_color("background", "background", -5)

        self.data = {}

        rules_div = DivWdg()
        top.add(rules_div)
        rules_div.add_style("padding: 10px")

        rules_div.add("Rules: ")

        rules_select = SelectWdg("rule_code")
        rule_code = self.get_value('rule_code')
        if rule_code:
            rules_select.set_value(rule_code)
        rules_select.set_option("query", "config/ingest_rule|code|title")
        rules_select.add_empty_option("-- New --")
        rules_div.add(rules_select)
        rules_select.add_behavior( {
        'type': 'change',
        'cbjs_action': '''
        var top = bvr.src_el.getParent(".spt_ingestion_top");
        value = bvr.src_el.value;
        var class_name = 'tactic.ui.tools.IngestionToolWdg';
        spt.panel.load(top, class_name, {rule_code: value} );
        '''
        } )

        rules_div.add("<hr/>")

        # read from the database
        if rule_code:
            search = Search("config/ingest_rule")
            search.add_filter("code", rule_code)
            sobject = search.get_sobject()
        else:
            sobject = None
        if sobject:
            self.data = sobject.get_value("data")
            if self.data:
                self.data = jsonloads(self.data)

        session_code = self.kwargs.get("session_code")
        if session_code:
            session = Search.get_by_code("config/ingest_session", session_code)
        else:
            if sobject:
                session = sobject.get_related_sobject("config/ingest_session")
                print("sobject: ", sobject.get_code(), sobject.get_value("spt_ingest_session_code"))
                print("parent: ", session)
            else:
                session = None


        if not session:
            #session = SearchType.create("config/ingest_session")
            #session.set_value("code", "session101")
            #session.set_value("location", "local")
            ##session.set_value("base_dir", "C:")
            top.add("No session defined!!!")
            return top



        rule = ""
        filter = ""
        ignore = ""


        # get the base path
        if sobject:
            base_dir = sobject.get_value("base_dir")
        else:
            base_dir = ''

        #else:
        #    base_dir = self.get_value("base_dir")
        #if not base_dir:
        #    base_dir = ''

        if sobject:
            title = sobject.get_value("title")
        else:
            title = ''

        if sobject:
            code = sobject.get_value("code")
        else:
            code = ''


        file_list = self.get_value("file_list")
        scan_type = self.get_value("scan_type")
        action_type = self.get_value("action_type")
        rule = self.get_value("rule")
        if not rule:
#.........这里部分代码省略.........

    def _process_video(self, file_name):
        if not HAS_FFMPEG:
            return

        thumb_web_size = self.get_web_file_size()
        thumb_icon_size = (120, 100)

        exts = File.get_extensions(file_name)

        base, ext = os.path.splitext(file_name)
        icon_file_name = "%s_icon.png" % base
        web_file_name = "%s_web.jpg" % base

        tmp_icon_path = "%s/%s" % (self.tmp_dir, icon_file_name)
        tmp_web_path = "%s/%s" % (self.tmp_dir, web_file_name)

        #cmd = '''"%s" -i "%s" -r 1 -ss 00:00:01 -t 1 -s %sx%s -vframes 1 "%s"''' % (ffmpeg, self.file_path, thumb_web_size[0], thumb_web_size[1], tmp_web_path)
        #os.system(cmd)

        import subprocess
                
        try:
            # Attempt to resize only if necessary. Requires ffprobe call.
            # (More recent version of ffmpeg support the argument
            # -vf scale="'if(gt(iw, 640), 640, iw)':'if(gt(ih, 6400), 6400, -1)'"
            # allowing for scaling which preserves aspect ratio and only scales
            # when necessary. For now, it is necessary to query video size.)
            free_aspect_ratio = thumb_web_size[1] == -1 
            try:
                command = ["ffprobe", "-print_format", "json", "-select_streams", "v:0", "-show_entries", "stream=height,width",  self.file_path]
                p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                out, err = p.communicate()
                data = jsonloads(out)
                streams = data.get("streams") or []
                sample_stream = streams[0]
                width = int(sample_stream.get("width"))
                height = int(sample_stream.get("height"))
            
                max_width = thumb_web_size[0]
                max_height = max_width*10 if free_aspect_ratio else thumb_web_size[1]
               
                if width < max_width and height < max_height:
                    # Resizing is not necessary
                    size_option = ""
                    size = ""
                elif not free_aspect_ratio and (width > max_width or height > max_height):
                    size_option = "-s"
                    size =  "%sx%s" % (thumb_web_size[0], thumb_web_size[1])
                else: 
                    if width > height:
                        size_option = "-vf"
                        size = "scale=%s:-1" % thumb_web_size[0]
                        
                    elif height > width:
                        aspect_ratio = float(float(height)/(width))
                        if aspect_ratio >= 10:
                            size_option = "-vf"
                            size = "scale=-1:%s" % max_height
                        else:
                            new_height = max_height
                            new_width = float(new_height)/height
                            if new_width > max_width:
                                new_width = max_width
                                new_height = height*float(new_width)/width
                                size_option = "-vf"
                                size = "scale=%s:-1" % max_width
                            else:
                                size_option = "-vf"
                                size = "scale=-1:%s" % max_height

            except Exception as e:
                if free_aspect_ratio: 
                    size_option = "-vf"
                    size = "scale=%s:-1" % thumb_web_size[0]
                else:
                    size_option = "-s"
                    size =  "%sx%s" % (thumb_web_size[0], thumb_web_size[1])

            command = [ffmpeg_exe, '-i', self.file_path, "-y", "-ss", "00:00:00","-t","1"]
            if size_option and size:
                command.extend([size_option, size])
            command.extend(["-vframes","1","-f","image2", tmp_web_path])
            subprocess.call(command)

            if os.path.exists(tmp_web_path):
                self.web_path = tmp_web_path
            else:
                self.web_path = None

        except Exception as e:

            Environment.add_warning("Could not process file", \
                    "%s - %s" % (self.file_path, e.__str__()))
            pass
          
        try:
            subprocess.call([ffmpeg_exe, '-i', self.file_path, "-y", "-ss", "00:00:00","-t","1",\
                    "-s","%sx%s"%(thumb_icon_size[0], thumb_icon_size[1]),"-vframes","1","-f","image2", tmp_icon_path])
            
            if os.path.exists(tmp_icon_path):
#.........这里部分代码省略.........