def get_build_data(self, depth=1):
     build_url = "/".join([JENKINS["url"], 'job',
                           self.name,
                           str(self.number),
                           'api/json?depth={depth}'.format(depth=depth)])
     logger.debug("Request build data from {}".format(build_url))
     return json.load(urllib2.urlopen(build_url))

def apply_transactions(transactions, auto=False):
    ''' Apply renaming transactions.
    apply_transactions(transactions)
    transactions = [(old_path, new_path),(old_path),(new_path),...]
    Manual review of transactions is required.
    '''
    if auto:
        logger.warning('Auto is On. No confirmation required.')
    print('='*30)
    if not transactions:
        logger.debug('NO TRANSACTIONS')
        sys.exit('No Transactions to apply.')
        return

    for t in transactions:
        print('[{}] > [{}]'.format(t[0].name, t[1].name))
    print('{} Transactions to apply. Renaming...'.format(len(transactions)))
    count = 0
    if auto or input('EXECUTE ? [y]\n>') == 'y':
        for src, dst in transactions:
            try:
                src.rename(dst)
            except:
                logger.error(sys.exc_info()[0].__name__)
                logger.error('Could not rename: [{}]>[{}]'.format(src, dst))
            else:
                logger.debug('[{}] renamed to [{}]'.format(src, dst))
                count += 1

        print('{} folders renamed.'.format(count))

    def message(self, msg):
        if msg['type'] not in ('chat', 'normal'):
            logger.debug('Strange message type: %(type)s' % msg)
            return
        #logger.info('Message from %(from)s: %(body)s' % msg)
        
        msg_text = msg['body'].strip()
        # msg['from'] is a JID object
        # http://sleekxmpp.com/api/xmlstream/jid.html
        from_user = msg['from'].bare 
        logger.info('FROM:' + from_user)
        logger.info('MSG:' + msg_text)

        try:
            if (from_user in settings.accept_command_from) and msg_text.startswith("$"):
                resp = commands.execute(msg_text[1:])
                msg.reply('\n'+resp).send()
            else:
                msg.reply(msg_text).send()
                #self.send_message(  mto=msg['from'],
                #                    mtype='chat',
                #                    mbody=msg_text,
                #                    mhtml='''<a href="http://www.google.co.jp">%s</a>'''% (msg_text))
        except:
            exc = traceback.format_exc()
            msg.reply(exc).send()

def get_downstream_builds_from_html(url):
    """Return list of downstream jobs builds from specified job
    """
    url = "/".join([url, 'downstreambuildview/'])
    logger.debug("Request downstream builds data from {}".format(url))
    req = urllib2.Request(url)
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    s = opener.open(req).read()
    opener.close()
    jobs = []
    raw_downstream_builds = re.findall(
        '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', s)
    for raw_build in raw_downstream_builds:
        sub_job_name = raw_build.split('/')[2]
        sub_job_build = raw_build.split('/')[3]
        build = Build(name=sub_job_name, number=sub_job_build)
        jobs.append(
            {
                'name': build.name,
                'number': build.number,
                'result': build.build_data['result']
            }
        )

    return jobs

def upload_tests_descriptions(testrail_project, section_id, tests, check_all_sections):
    tests_suite = testrail_project.get_suite_by_name(TestRailSettings.tests_suite)
    check_section = None if check_all_sections else section_id
    existing_cases = [
        case["custom_test_group"]
        for case in testrail_project.get_cases(suite_id=tests_suite["id"], section_id=check_section)
    ]
    for test_case in tests:
        if test_case["custom_test_group"] in existing_cases:
            logger.debug(
                'Skipping uploading "{0}" test case because it '
                'already exists in "{1}" tests section.'.format(
                    test_case["custom_test_group"], TestRailSettings.tests_suite
                )
            )
            continue

        logger.debug(
            'Uploading test "{0}" to TestRail project "{1}", '
            'suite "{2}", section "{3}"'.format(
                test_case["custom_test_group"],
                TestRailSettings.project,
                TestRailSettings.tests_suite,
                TestRailSettings.tests_section,
            )
        )
        testrail_project.add_case(section_id=section_id, case=test_case)

    def get(self, *args, **kwargs):
        node = self._get_node(kwargs.get('path', ''))
        interface = self.get_argument("if")
        status, error_message = 200, None

        if not node or not interface:
            status = BAD_REQUEST
            error_message = "No node or interface specified"
            logger.exception(error_message)
        else:
            try:
                response = ControllerOperations().get_acls(node, interface)
                if response:
                    logger.debug(response)

                    self.set_header("Content-Type", "application/json")
                    self.write(json.dumps(response))
                    # self.finish()
                else:
                    status = NOT_FOUND
                    error_message = "No data found"
            except ValueError as e:
                status = INTERNAL_SERVER_ERROR
                error_message = e.message

        if error_message:
            logger.exception(error_message)
            if DEBUG:
                self.set_status(status, error_message)
            else:
                self.set_status(status)
        else:
            self.set_status(status)

    def stop(self):
        """Stop the daemon with checking is daemon already run"""
        # Get the pid from the pidfile
        logger.debug( "EWSoDS daemon is going to stop." )
        try:
            pf = file(self.pidfile,'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if not pid:
            logger.debug( "EWSoDS daemon check pid. [pidfile] : %s does not exist. Daemon not running?" % self.pidfile )
            return 

        try:
            while 1:
                os.kill(pid, SIGTERM)
                time.sleep(0.1)
        except OSError, err:
            err = str(err)
            if err.find("No such process") > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)

    def get_check_create_test_run(self, plan, cases):
        plan = self.project.plans.get(plan.id)
        suite_cases = self.suite.cases()
        run_name = self.get_run_name()
        runs = plan.runs.find_all(name=run_name)
        run = self.check_need_create_run(plan,
                                         runs,
                                         suite_cases)

        if run is None:
            logger.info('Run not found in plan "{}", create: "{}"'.format(
                plan.name, run_name))

            # Create new test run with cases from test suite
            suite_cases = self.get_suite_cases()

            if not suite_cases:
                logger.error('Empty test cases set.')
                return None

            # suite_cases = self.suite.cases.find(type_id=type_ids[0])
            run = Run(name=run_name,
                      description=self.run_description,
                      suite_id=self.suite.id,
                      milestone_id=self.milestone.id,
                      config_ids=[],
                      case_ids=[x.id for x in suite_cases]
                      )
            plan.add_run(run)
            logger.debug('Run created "{}"'.format(run_name))
        return run

def get_tests_groups_from_jenkins(runner_name, build_number, distros):
    runner_build = Build(runner_name, build_number)
    res = {}
    for b in runner_build.build_data['subBuilds']:

        if b['result'] is None:
            logger.debug("Skipping '{0}' job (build #{1}) because it's still "
                         "running...".format(b['jobName'], b['buildNumber'],))
            continue

        # Get the test group from the console of the job
        z = Build(b['jobName'], b['buildNumber'])
        console = z.get_job_console()
        groups = [keyword.split('=')[1]
                  for line in console
                  for keyword in line.split()
                  if 'run_tests.py' in line and '--group=' in keyword]
        if not groups:
            logger.error("No test group found in console of the job {0}/{1}"
                         .format(b['jobName'], b['buildNumber']))
            continue
        # Use the last group (there can be several groups in upgrade jobs)
        test_group = groups[-1]

        # Get the job suffix
        job_name = b['jobName']
        for distro in distros:
            if distro in job_name:
                sep = '.' + distro + '.'
                job_suffix = job_name.split(sep)[-1]
                break
        else:
            job_suffix = job_name.split('.')[-1]
        res[job_suffix] = test_group
    return res

def scrape_well():
    logger.debug("Scraping www.well.ca for deals on diapers...")
    root_url = settings.WELL['root_url']
    diaper_category_url = settings.WELL['diaper_category_url']

    # Parse main page
    response = requests.get(root_url + diaper_category_url)
    soup = BeautifulSoup(response.text, "lxml")
    diaper_size_links = [a.attrs.get('href') for a in soup.select(settings.WELL['size_category_css_selector'])]
    diaper_size_list = [p.text for p in soup.select(settings.WELL['size_category_css_selector'])]
    assert(len(diaper_size_links) == len(diaper_size_list))

    # Create product list
    product_list = []
    x = 0

    # For each size category parse for products
    for i in range(len(diaper_size_links)):
        size = diaper_size_list[i].strip()
        r = requests.get(diaper_size_links[i])
        s = BeautifulSoup(r.text, "lxml")
        product_category_links = [a.attrs.get('href') for a in s.select(settings.WELL['product_category_css_selector'])]
        for link in product_category_links:
            r = requests.get(link)
            s = BeautifulSoup(r.text, "lxml")
            product_name = [p.text.strip() for p in s.select(settings.WELL['product_name_css_selector'])]
            assert(len(product_name) == 1)
            product_list.append(WellProduct(product_name[0].strip(), link, 'well.ca', size=size))
            product_list[x].parse_data()

            if DEBUG:
                product_list[x].print_properties()
            else:
                product_list[x].save_to_db()
            x += 1

def save_as_csv(dfs,suffix='_scaled.csv'):
    for df in dfs:
        file_path_prefix, _ = os.path.splitext(df.fullpath)
        csv_path = file_path_prefix + suffix
        df.to_csv(csv_path)
        logger.debug("CSV File saved to %s"%csv_path)
    return dfs

def get_tests_descriptions(milestone_id, tests_include, tests_exclude, groups):
    import_tests()

    tests = []

    for jenkins_suffix in groups:
        group = groups[jenkins_suffix]
        for case in TestProgram(groups=[group]).cases:
            if not case.entry.info.enabled:
                continue
            if tests_include:
                if tests_include not in case.entry.home.func_name:
                    logger.debug(
                        "Skipping '{0}' test because it doesn't "
                        "contain '{1}' in method name".format(case.entry.home.func_name, tests_include)
                    )
                    continue
            if tests_exclude:
                if tests_exclude in case.entry.home.func_name:
                    logger.debug(
                        "Skipping '{0}' test because it contains"
                        " '{1}' in method name".format(case.entry.home.func_name, tests_exclude)
                    )
                    continue

            docstring = case.entry.home.func_doc or ""
            docstring = "\n".join([s.strip() for s in docstring.split("\n")])

            steps = [{"content": s, "expected": "pass"} for s in docstring.split("\n") if s and s[0].isdigit()]

            test_duration = re.search(r"Duration\s+(\d+[s,m])\b", docstring)
            title = docstring.split("\n")[0] or case.entry.home.func_name
            test_group = case.entry.home.func_name

            if case.entry.home.func_name in GROUPS_TO_EXPAND:
                """Expand specified test names with the group names that are
                   used in jenkins jobs where this test is started.
                """
                title = " - ".join([title, jenkins_suffix])
                test_group = "_".join([case.entry.home.func_name, jenkins_suffix])

            test_case = {
                "title": title,
                "type_id": 1,
                "milestone_id": milestone_id,
                "priority_id": 5,
                "estimate": test_duration.group(1) if test_duration else "3m",
                "refs": "",
                "custom_test_group": test_group,
                "custom_test_case_description": docstring or " ",
                "custom_test_case_steps": steps,
            }

            if not any([x["custom_test_group"] == test_group for x in tests]):
                tests.append(test_case)
            else:
                logger.warning("Testcase '{0}' run in multiple Jenkins jobs!".format(test_group))
    return tests

 def decorated_function(*args, **kwargs):
     if not hasattr(request,'survey') or not hasattr(request,'user'):
         logger.debug("with_admin: survey or user not loaded!")
         abort(404)
     print request.user.document_id,str(request.survey['user'])
     if not request.survey['user'] == request.user:
         logger.debug("with_admin: not an admin!")
         abort(403)
     return f(*args, **kwargs)

    def get_test_data(self, url, result_path=None):
        if result_path:
            test_url = "/".join(
                [url.rstrip("/"), 'testReport'] + result_path + ['api/json'])
        else:
            test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json'])

        logger.debug("Request test data from {}".format(test_url))
        response = urllib2.urlopen(test_url)
        return json.load(response)

def get_build_artifact(url, artifact):
    """Return content of job build artifact
    """
    url = "/".join([url, 'artifact', artifact])
    logger.debug("Request artifact content from {}".format(url))
    req = urllib2.Request(url)
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    s = opener.open(req).read()
    opener.close()
    return s

	def __connect(self):
		conn_cnt = 0
		logger.info('trying to connect to sqlserver on %s:%s' % (s.get('host'), s.get('port')))
		while conn_cnt < s.get('reconnect_cnt', 3):
			try:
				conn = pymssql.connect(host=s.get('host'), port=s.get('port'), user=s.get('user'),\
					password=s.get('password'), database=s.get('database'), charset=s.get('charset'))
				return conn
			except Exception, e:	# add a specified exception
				conn_cnt += 1
				logger.debug('connecting failed, times to reconnect: %d' % conn_cnt)

    def bugs_statistics(self):
        if self._bugs_statistics != {}:
            return self._bugs_statistics
        logger.info('Collecting stats for TestRun "{0}" on "{1}"...'.format(
            self.run['name'], self.run['config'] or 'default config'))

        for test in self.tests:
            logger.debug('Checking "{0}" test...'.format(
                test['title'].encode('utf8')))
            test_results = sorted(
                self.project.get_results_for_test(test['id'], self.results),
                key=lambda x: x['id'], reverse=True)

            linked_bugs = []
            is_blocked = False

            for result in test_results:
                if result['status_id'] in self.blocked_statuses:
                    if self.check_blocked:
                        new_bug_link = self.handle_blocked(test, result)
                        if new_bug_link:
                            linked_bugs.append(new_bug_link)
                            is_blocked = True
                            break
                    if result['custom_launchpad_bug']:
                        linked_bugs.append(result['custom_launchpad_bug'])
                        is_blocked = True
                        break
                if result['status_id'] in self.failed_statuses \
                        and result['custom_launchpad_bug']:
                    linked_bugs.append(result['custom_launchpad_bug'])

            bug_ids = set([re.search(r'.*bug/(\d+)/?', link).group(1)
                           for link in linked_bugs
                           if re.search(r'.*bug/(\d+)/?', link)])

            for bug_id in bug_ids:
                if bug_id in self._bugs_statistics:
                    self._bugs_statistics[bug_id][test['id']] = {
                        'group': test['custom_test_group'] or 'manual',
                        'config': self.run['config'] or 'default',
                        'blocked': is_blocked
                    }

                else:
                    self._bugs_statistics[bug_id] = {
                        test['id']: {
                            'group': test['custom_test_group'] or 'manual',
                            'config': self.run['config'] or 'default',
                            'blocked': is_blocked
                        }
                    }
        return self._bugs_statistics

def get_jobs_for_view(view):
    """Return list of jobs from specified view
    """
    view_url = "/".join([JENKINS["url"], 'view', view, 'api/json'])
    logger.debug("Request view data from {}".format(view_url))
    req = urllib2.Request(view_url)
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    s = opener.open(req).read()
    opener.close()
    view_data = json.loads(s)
    jobs = [job["name"] for job in view_data["jobs"]]
    return jobs

 def get_or_create_plan(self):
     """Get exists or create new TestRail Plan"""
     plan_name = self.get_plan_name()
     plan = self.project.plans.find(name=plan_name)
     if plan is None:
         plan = self.project.plans.add(name=plan_name,
                                       description=self.plan_description,
                                       milestone_id=self.milestone.id)
         logger.debug('Plan created"{}"'.format(plan_name))
     else:
         logger.debug('Plan found "{}"'.format(plan_name))
     return plan

    def delete(self, *args, **kwargs):
        logger.debug('Delete acl payload: {}'.format(self.request.body))
        result, msg = ControllerOperations().delete_acl(self.request.body)
        response = {
            'ret_code': result,
            'message': msg
        }

        if result is True:
            self.set_header('Content-Type', 'application/json')
            self.write(response)
        else:
            self.set_status(INTERNAL_SERVER_ERROR, msg)