def test_alive(ip_str="180.188.250.54", begin=45, end=60, interval=2):
    start_time = time.time()
    test_array = {}
    for i in range(begin, end, interval):
        sslsock, _, _ = connect_ssl(ip_str)
        try:
            result = test_app_check(sslsock, ip_str)
        except:
            break
        test_array[i] = {}
        test_array[i]["ssl"] = sslsock
        test_array[i]["start_time"] = time.time()
        time.sleep(interval)

    time_now = time.time()
    time.sleep(end - (time_now - start_time))

    for i in test_array:
        stat = test_array[i]
        sslsock = stat["ssl"]
        time_now = time.time()
        try:
            result = test_app_check(sslsock, ip_str)
            xlog.info("time alive:%d", time_now - stat["start_time"])
        except:
            xlog.info("time alive fail:%d", time_now - stat["start_time"])
            continue

    def scan_ip_worker(self):
        while self.searching_thread_count <= self.scan_ip_thread_num:
            if not connect_control.allow_scan():
                time.sleep(10)
                continue

            try:
                time.sleep(1)
                ip_int = ip_range.get_ip()
                ip_str = ip_utils.ip_num_to_string(ip_int)
                if self.is_bad_ip(ip_str):
                    continue

                result = check_ip.test_gws(ip_str)
                if not result:
                    continue

                if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type):
                    #logging.info("add  %s  CN:%s  type:%s  time:%d  gws:%d ", ip_str,
                    #     result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
                    xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time)
                    scan_ip_log.info("Add %s time:%d CN:%s type:%s", ip_str, result.handshake_time, result.domain, result.server_type)
                    self.remove_slowest_ip()
                    self.save_ip_list()
            except check_ip.HoneypotError as e:
                self.report_bad_ip(ip_str)
                connect_control.fall_into_honeypot()
                continue
            except Exception as e:
                xlog.exception("google_ip.runJob fail:%s", e)

        self.ncount_lock.acquire()
        self.searching_thread_count -= 1
        self.ncount_lock.release()
        xlog.info("scan_ip_worker exit")

    def test2(self):
        work_ciphers = ["AES128-SHA"]
        for cipher in self.cipher_list:
            if cipher in work_ciphers:
                continue
            else:
                work_ciphers.append(cipher)

            xlog.debug("%s", cipher)
            cipher_suites = (work_ciphers)

            openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
            try:
                ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
                if "gws" not in server_type:
                    work_ciphers.remove(cipher)
            except Exception as e:
                xlog.warn("err:%s", e)
                try:
                    work_ciphers.remove(cipher)
                except:
                    pass

        work_str = ""
        for cipher in work_ciphers:
            work_str += cipher + ":"
        xlog.info("work ciphers:%s", work_str)

    def do_POST(self):
        try:
            refer = self.headers.getheader('Referer')
            netloc = urlparse.urlparse(refer).netloc
            if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"):
                xlog.warn("web control ref:%s refuse", netloc)
                return
        except:
            pass
        xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path)
        try:
            ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
            if ctype == 'multipart/form-data':
                self.postvars = cgi.parse_multipart(self.rfile, pdict)
            elif ctype == 'application/x-www-form-urlencoded':
                length = int(self.headers.getheader('content-length'))
                self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1)
            else:
                self.postvars = {}
        except:
            self.postvars = {}

        path = urlparse.urlparse(self.path).path
        if path == '/deploy':
            return self.req_deploy_handler()
        elif path == "/config":
            return self.req_config_handler()
        elif path == "/scan_ip":
            return self.req_scan_ip_handler()
        elif path.startswith("/importip"):
            return self.req_importip_handler()
        else:
            self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')
            xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)

def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes

    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [
            ("dwOSVersionInfoSize", ctypes.c_ulong),
            ("dwMajorVersion", ctypes.c_ulong),
            ("dwMinorVersion", ctypes.c_ulong),
            ("dwBuildNumber", ctypes.c_ulong),
            ("dwPlatformId", ctypes.c_ulong),
            ("szCSDVersion", ctypes.c_wchar * 128),
            ("wServicePackMajor", ctypes.c_ushort),
            ("wServicePackMinor", ctypes.c_ushort),
            ("wSuiteMask", ctypes.c_ushort),
            ("wProductType", ctypes.c_byte),
            ("wReserved", ctypes.c_byte),
        ]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False

    def remove_slowest_ip(self):
        if len(self.gws_ip_list) <= self.max_good_ip_num:
            return

        self.try_sort_ip_by_handshake_time(force=True)

        self.ip_lock.acquire()
        try:
            ip_num = len(self.gws_ip_list)
            while ip_num > self.max_good_ip_num:

                ip_str = self.gws_ip_list[ip_num - 1]

                property = self.ip_dict[ip_str]
                server = property['server']
                handshake_time = property['handshake_time']
                xlog.info("remove_slowest_ip:%s handshake_time:%d", ip_str, handshake_time)
                del self.ip_dict[ip_str]

                if 'gws' in server and ip_str in self.gws_ip_list:
                    self.gws_ip_list.remove(ip_str)

                ip_num -= 1

        except Exception as e:
            xlog.exception("remove_slowest_ip err:%s", e)
        finally:
            self.ip_lock.release()

 def create_ca():
     key = OpenSSL.crypto.PKey()
     key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
     req = OpenSSL.crypto.X509Req()
     subj = req.get_subject()
     subj.countryName = 'CN'
     subj.stateOrProvinceName = 'Internet'
     subj.localityName = 'Cernet'
     subj.organizationName = CertUtil.ca_vendor
     subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
     subj.commonName = '%s XX-Net' % CertUtil.ca_vendor #TODO: here should be GoAgent
     req.set_pubkey(key)
     req.sign(key, CertUtil.ca_digest)
     ca = OpenSSL.crypto.X509()
     ca.set_version(2)
     ca.set_serial_number(0)
     ca.gmtime_adj_notBefore(0)
     ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652)
     ca.set_issuer(req.get_subject())
     ca.set_subject(req.get_subject())
     ca.set_pubkey(req.get_pubkey())
     ca.add_extensions([
         OpenSSL.crypto.X509Extension(
             'basicConstraints', False, 'CA:TRUE', ca, ca)
         ])
     ca.sign(key, CertUtil.ca_digest)
     #logging.debug("CA key:%s", key)
     xlog.info("create ca")
     return key, ca

def test(ip_str, loop=1):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str, check_cert=False)

    for i in range(loop):

        result = check.check(callback=test_app_head)
        if not result:
            if "gws" in check.result.server_type:
                xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type)

            xlog.warn("check fail")
            #continue
        else:
            xlog.debug("=======app check ok: %s", ip_str)
            check.result.appspot_ok = result


        result = check.check(callback=test_server_type, check_ca=True)
        if not result:
            xlog.debug("test server type fail")
            continue

        check.result.server_type = result
        xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type,
                     check.result.domain, check.result.handshake_time)

    return check.result

    def remove_ip_process(self):
        try:
            while True:

                try:
                    ip_str = self.to_remove_ip_list.get_nowait()
                except:
                    break

                result = check_ip.test(ip_str)
                if result and result.appspot_ok:
                    self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type)
                    xlog.debug("remove ip process, restore ip:%s", ip_str)
                    continue

                if not check_ip.network_is_ok():
                    self.to_remove_ip_list.put(ip_str)
                    xlog.warn("network is unreachable. check your network connection.")
                    return

                xlog.info("real remove ip:%s ", ip_str)
                self.iplist_need_save = 1
        finally:
            self.remove_ip_thread_num_lock.acquire()
            self.remove_ip_thread_num -= 1
            self.remove_ip_thread_num_lock.release()

    def report_connect_fail(self, ip_str, force_remove=False):
        self.ip_lock.acquire()
        try:
            time_now = time.time()
            if not ip_str in self.ip_dict:
                return

            self.ip_dict[ip_str]["links"] -= 1

            # ignore if system network is disconnected.
            if not force_remove:
                if not check_ip.network_is_ok():
                    xlog.debug("report_connect_fail network fail")
                    # connect_control.fall_into_honeypot()
                    return

            fail_time = self.ip_dict[ip_str]["fail_time"]
            if not force_remove and time_now - fail_time < 1:
                xlog.debug("fail time too near")
                return

            # increase handshake_time to make it can be used in lower probability
            self.ip_dict[ip_str]["handshake_time"] += 300

            if self.ip_dict[ip_str]["fail_times"] == 0:
                self.good_ip_num -= 1
            self.ip_dict[ip_str]["fail_times"] += 1
            self.append_ip_history(ip_str, "fail")
            self.ip_dict[ip_str]["fail_time"] = time_now

            if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50:
                property = self.ip_dict[ip_str]
                server = property["server"]
                del self.ip_dict[ip_str]

                if "gws" in server and ip_str in self.gws_ip_list:
                    self.gws_ip_list.remove(ip_str)

                if not force_remove:
                    self.to_remove_ip_list.put(ip_str)
                    self.try_remove_thread()
                    xlog.info(
                        "remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
                    )
                else:
                    xlog.info(
                        "remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
                    )

                if self.good_ip_num > len(self.ip_dict):
                    self.good_ip_num = len(self.ip_dict)

            self.iplist_need_save = 1
        except Exception as e:
            xlog.exception("set_ip err:%s", e)
        finally:
            self.ip_lock.release()

        if not self.is_ip_enough():
            self.search_more_google_ip()

    def remove_slowest_ip(self):
        if len(self.gws_ip_list) <= self.max_good_ip_num:
            return

        self.try_sort_gws_ip(force=True)

        self.ip_lock.acquire()
        try:
            ip_num = len(self.gws_ip_list)
            while ip_num > self.max_good_ip_num:

                ip_str = self.gws_ip_list[ip_num - 1]

                property = self.ip_dict[ip_str]
                server = property["server"]
                fails = property["fail_times"]
                handshake_time = property["handshake_time"]
                xlog.info("remove_slowest_ip:%s handshake_time:%d, fails:%d", ip_str, handshake_time, fails)
                del self.ip_dict[ip_str]

                if "gws" in server and ip_str in self.gws_ip_list:
                    self.gws_ip_list.remove(ip_str)

                ip_num -= 1

        except Exception as e:
            xlog.exception("remove_slowest_ip err:%s", e)
        finally:
            self.ip_lock.release()

def check_all_exist_ip():

    good_ip_file_name = "good_ip.txt"
    good_ip_file = os.path.abspath( os.path.join(config.DATA_PATH, good_ip_file_name))
    if not os.path.isfile(good_ip_file):
        print "open file ", good_ip_file_name, " fail."
        return

    with open(good_ip_file, "r") as fd:
        lines = fd.readlines()

    for line in lines:
        try:
            str_l = line.split(' ')
            if len(str_l) != 4:
                xlog.warning("line err: %s", line)
                continue
            ip_str = str_l[0]
            domain = str_l[1]
            server = str_l[2]
            handshake_time = int(str_l[3])

            xlog.info("test ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
            #test_with_app(ip_str)
            test_gws(ip_str)
            #self.add_ip(ip_str, handshake_time, domain, server)
        except Exception as e:
            xlog.exception("load_ip line:%s err:%s", line, e)

    def load_ip(self):
        if os.path.isfile(self.good_ip_file):
            file_path = self.good_ip_file
        else:
            file_path = self.default_good_ip_file

        with open(file_path, "r") as fd:
            lines = fd.readlines()

        for line in lines:
            try:
                if line.startswith("#"):
                    continue

                str_l = line.split(" ")

                if len(str_l) < 4:
                    xlog.warning("line err: %s", line)
                    continue
                ip_str = str_l[0]
                domain = str_l[1]
                server = str_l[2]
                handshake_time = int(str_l[3])
                if len(str_l) > 4:
                    fail_times = int(str_l[4])
                else:
                    fail_times = 0

                # logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
                self.add_ip(ip_str, handshake_time, domain, server, fail_times)
            except Exception as e:
                xlog.exception("load_ip line:%s err:%s", line, e)

        xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list))
        self.try_sort_gws_ip(force=True)

def load_tasks():
    tasks = []
    intervals = []

    import imp
    tasks_path = config.tasks_path
    for fileName in os.listdir(tasks_path):
        if fileName.lower().endswith('.py'):
            xlog.info('start load %s' % fileName)
            try:
                with open(os.path.join(tasks_path, fileName), 'rb') as fpy:
                    task = imp.load_source('Auto-tasks_%d_%s' % (len(tasks), fileName), tasks_path, fpy)

                    name = task.name
                    interval = task.run_interval

                    task.init()

                    tasks.append(task)
                    intervals.append(interval)

                    xlog.info('load task %s success.' % name)
            except:
                xlog.warn('load %s fail.' % fileName)
    del imp

    return (tasks, intervals)

def test_alive(ip_str="74.125.96.107", begin=50, end=60, interval=2):

    test_array = {}
    for i in range(begin, end, interval):
        sslsock, _, _ = connect_ssl(ip_str)
        try:
            result = test_app_check(sslsock, ip_str)
        except:
            break
        test_array[i] = {}
        test_array[i]["ssl"] = sslsock
        test_array[i]["start_time"] = time.time()
        time.sleep(interval)

    time.sleep(begin)

    for i in test_array:
        stat = test_array[i]
        sslsock = stat["ssl"]
        time_now = time.time()
        try:
            result = test_app_check(sslsock, ip_str)
            xlog.info("time alive:%d", time_now - stat["start_time"])
        except:
            xlog.info("time alive fail")
            break

    def context_builder(ca_certs=None, cipher_suites=('ALL:!RC4-SHA:!ECDHE-RSA-RC4-SHA:!ECDHE-RSA-AES128-GCM-SHA256:!AES128-GCM-SHA256',)):
        # 'ALL', '!aNULL', '!eNULL'
        global  ssl_version

        if not ssl_version:
            if hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
                ssl_version = "TLSv1_2"
            elif hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
                ssl_version = "TLSv1_1"
            elif hasattr(OpenSSL.SSL, "TLSv1_METHOD"):
                ssl_version = "TLSv1"
            else:
                ssl_version = "SSLv23"

            if sys.platform == "darwin":
                ssl_version = "TLSv1"
            
            # freenas openssl support fix from twitter user "himanzero"
            # https://twitter.com/himanzero/status/645231724318748672
            if sys.platform == "freebsd9":
                ssl_version = "TLSv1"

            xlog.info("SSL use version:%s", ssl_version)

        protocol_version = getattr(OpenSSL.SSL, '%s_METHOD' % ssl_version)
        ssl_context = OpenSSL.SSL.Context(protocol_version)
        if ca_certs:
            ssl_context.load_verify_locations(os.path.abspath(ca_certs))
            ssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok)
        else:
            ssl_context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda c, x, e, d, ok: ok)
        ssl_context.set_cipher_list(':'.join(cipher_suites))
        return ssl_context

    def import_mac_ca(common_name, certfile):
        commonname = "GoAgent XX-Net" #TODO: need check again
        ca_hash = CertUtil.ca_thumbprint.replace(':', '')

        def get_exist_ca_sha1():
            args = ['security', 'find-certificate', '-Z', '-a', '-c', commonname]
            output = subprocess.check_output(args)
            for line in output.splitlines(True):
                if len(line) == 53 and line.startswith("SHA-1 hash:"):
                    sha1_hash = line[12:52]
                    return sha1_hash

        exist_ca_sha1 = get_exist_ca_sha1()
        if exist_ca_sha1 == ca_hash:
            xlog.info("GoAgent CA exist")
            return

        import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../data/gae_proxy/CA.crt'# % certfile.decode('utf-8')
        if exist_ca_sha1:
            delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1
            exec_command = "%s;%s" % (delete_ca_command, import_command)
        else:
            exec_command = import_command

        admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command
        cmd = admin_command.encode('utf-8')
        xlog.info("try auto import CA command:%s", cmd)
        os.system(cmd)

def load_openssl():
    global loaded, libcrypto, buf

    from ctypes.util import find_library
    for p in ('crypto', 'eay32', 'libeay32'):
        libcrypto_path = find_library(p)
        if libcrypto_path:
            break
    else:
        raise Exception('libcrypto(OpenSSL) not found')
    xlog.info('loading libcrypto from %s', libcrypto_path)
    libcrypto = CDLL(libcrypto_path)
    libcrypto.EVP_get_cipherbyname.restype = c_void_p
    libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p

    libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p,
                                            c_char_p, c_char_p, c_int)

    libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p,
                                           c_char_p, c_int)

    libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,)
    libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,)
    if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'):
        libcrypto.OpenSSL_add_all_ciphers()

    buf = create_string_buffer(buf_size)
    loaded = True

 def __init__(self, address, handler, args=()):
     self.socket = None
     self.running = True
     self.server_address = address
     self.handler = handler
     self.args = args
     self.init_socket()
     xlog.info("server %s:%d started.", address[0], address[1])

    def roll_log(self):
        for i in range(1000):
            file_name = os.path.join(config.DATA_PATH, "scan_ip.%d.log" % i)
            if os.path.isfile(file_name):
                continue

            xlog.info("scan_ip_log roll %s -> %s", self.log_path, file_name)
            shutil.move(self.log_path, file_name)
            return