def float2json(value):
    """
    CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY
    :param value: float, int, long, Decimal
    :return: unicode
    """
    if value == 0:
        return u'0'
    try:
        sign = "-" if value < 0 else ""
        value = abs(value)
        sci = value.__format__(".15e")
        mantissa, str_exp = sci.split("e")
        digits, more_digits = _snap_to_base_10(mantissa)
        int_exp = int(str_exp) + more_digits
        if int_exp > 15:
            return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp)
        elif int_exp >= 0:
            return sign + (digits[:1 + int_exp] + '.' + digits[1 + int_exp:].rstrip('0')).rstrip('.')
        elif -4 < int_exp:
            digits = ("0" * (-int_exp)) + digits
            return sign + (digits[:1] + '.' + digits[1:].rstrip('0')).rstrip('.')
        else:
            return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp)
    except Exception as e:
        from mo_logs import Log
        Log.error("not expected", e)

def quote_value(value):
    """
    convert values to mysql code for the same
    mostly delegate directly to the mysql lib, but some exceptions exist
    """
    try:
        if value == None:
            return SQL_NULL
        elif isinstance(value, SQL):
            return quote_sql(value.template, value.param)
        elif is_text(value):
            return SQL("'" + "".join(ESCAPE_DCT.get(c, c) for c in value) + "'")
        elif is_data(value):
            return quote_value(json_encode(value))
        elif is_number(value):
            return SQL(text_type(value))
        elif isinstance(value, datetime):
            return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
        elif isinstance(value, Date):
            return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
        elif hasattr(value, '__iter__'):
            return quote_value(json_encode(value))
        else:
            return quote_value(text_type(value))
    except Exception as e:
        Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)

def utf82unicode(value):
    """
    WITH EXPLANATION FOR FAILURE
    """
    try:
        return value.decode("utf8")
    except Exception as e:
        if not _Log:
            _late_import()

        if not is_binary(value):
            _Log.error("Can not convert {{type}} to unicode because it's not bytes",  type= type(value).__name__)

        e = _Except.wrap(e)
        for i, c in enumerate(value):
            try:
                c.decode("utf8")
            except Exception as f:
                _Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])

        try:
            latin1 = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        try:
            a = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        _Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)

def note(template, **params):
    if not is_text(template):
        Log.error("Log.note was expecting a unicode template")

    if len(template) > 10000:
        template = template[:10000]

    log_params = wrap(
        {
            "template": template,
            "params": params,
            "timestamp": datetime.utcnow(),
            "machine": machine_metadata,
            "context": exceptions.NOTE,
        }
    )

    if not template.startswith("\n") and template.find("\n") > -1:
        template = "\n" + template

    if Log.trace:
        log_template = (
            '{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - "{{location.file}}:{{location.line}}" ({{location.method}}) - '
            + template.replace("{{", "{{params.")
        )
        f = sys._getframe(1)
        log_params.location = {
            "line": f.f_lineno,
            "file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
            "method": text_type(f.f_code.co_name),
        }
    else:
        log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.")

    Log.main_log.write(log_template, log_params)

    def wrap(cls, e, stack_depth=0):
        """
        ENSURE THE STACKTRACE AND CAUSAL CHAIN IS CAPTURED, PLUS ADD FEATURES OF Except

        :param e: AN EXCEPTION OF ANY TYPE
        :param stack_depth: HOW MANY CALLS TO TAKE OFF THE TOP OF THE STACK TRACE
        :return: A Except OBJECT OF THE SAME
        """
        if e == None:
            return Null
        elif isinstance(e, (list, Except)):
            return e
        elif is_data(e):
            e.cause = unwraplist([Except.wrap(c) for c in listwrap(e.cause)])
            return Except(**e)
        else:
            tb = getattr(e, '__traceback__', None)
            if tb is not None:
                trace = _parse_traceback(tb)
            else:
                trace = _extract_traceback(0)

            cause = Except.wrap(getattr(e, '__cause__', None))
            if hasattr(e, "message") and e.message:
                output = Except(context=ERROR, template=text_type(e.message), trace=trace, cause=cause)
            else:
                output = Except(context=ERROR, template=text_type(e), trace=trace, cause=cause)

            trace = extract_stack(stack_depth + 2)  # +2 = to remove the caller, and it's call to this' Except.wrap()
            output.trace.extend(trace)
            return output

 def to_python(self, not_null=False, boolean=False, many=False):
     return (
         "row["
         + text_type(self.var)
         + "] if 0<="
         + text_type(self.var)
         + "<len(row) else None"
     )

def box(script):
    """
    :param es_script:
    :return: TEXT EXPRESSION WITH NON OBJECTS BOXED
    """
    if script.type is BOOLEAN:
        return "Boolean.valueOf(" + text_type(script.expr) + ")"
    elif script.type is INTEGER:
        return "Integer.valueOf(" + text_type(script.expr) + ")"
    elif script.type is NUMBER:
        return "Double.valueOf(" + text_type(script.expr) + ")"
    else:
        return script.expr

 def _convert(v):
     if v is None:
         return NULL.to_es_script(schema)
     if v is True:
         return EsScript(
             type=BOOLEAN,
             expr="true",
             frum=self
         )
     if v is False:
         return EsScript(
             type=BOOLEAN,
             expr="false",
             frum=self
         )
     if isinstance(v, text_type):
         return EsScript(
             type=STRING,
             expr=quote(v),
             frum=self
         )
     if isinstance(v, int):
         return EsScript(
             type=INTEGER,
             expr=text_type(v),
             frum=self
         )
     if isinstance(v, float):
         return EsScript(
             type=NUMBER,
             expr=text_type(v),
             frum=self
         )
     if isinstance(v, dict):
         return EsScript(
             type=OBJECT,
             expr="[" + ", ".join(quote(k) + ": " + _convert(vv) for k, vv in v.items()) + "]",
             frum=self
         )
     if isinstance(v, (list, tuple)):
         return EsScript(
             type=OBJECT,
             expr="[" + ", ".join(_convert(vv).expr for vv in v) + "]",
             frum=self
         )
     if isinstance(v, Date):
         return EsScript(
             type=NUMBER,
             expr=text_type(v.unix),
             frum=self
         )

def table2csv(table_data):
    """
    :param table_data: expecting a list of tuples
    :return: text in nice formatted csv
    """
    text_data = [tuple(value2json(vals, pretty=True) for vals in rows) for rows in table_data]

    col_widths = [max(len(text) for text in cols) for cols in zip(*text_data)]
    template = ", ".join(
        "{{" + text_type(i) + "|left_align(" + text_type(w) + ")}}"
        for i, w in enumerate(col_widths)
    )
    text = "\n".join(expand_template(template, d) for d in text_data)
    return text

def tab(value):
    """
    convert single value to tab-delimited form, including a header
    :param value:
    :return:
    """
    if is_data(value):
        h, d = transpose(*wrap(value).leaves())
        return (
            "\t".join(map(value2json, h)) +
            CR +
            "\t".join(map(value2json, d))
        )
    else:
        text_type(value)

    def note(
        cls,
        template,
        default_params={},
        stack_depth=0,
        log_context=None,
        **more_params
    ):
        """
        :param template: *string* human readable string with placeholders for parameters
        :param default_params: *dict* parameters to fill in template
        :param stack_depth:  *int* how many calls you want popped off the stack to report the *true* caller
        :param log_context: *dict* extra key:value pairs for your convenience
        :param more_params: *any more parameters (which will overwrite default_params)
        :return:
        """
        if not isinstance(template, text_type):
            Log.error("Log.note was expecting a unicode template")

        if len(template) > 10000:
            template = template[:10000]

        params = dict(unwrap(default_params), **more_params)

        log_params = set_default({
            "template": template,
            "params": params,
            "timestamp": datetime.utcnow(),
            "machine": machine_metadata
        }, log_context, {"context": exceptions.NOTE})

        if not template.startswith("\n") and template.find("\n") > -1:
            template = "\n" + template

        if cls.trace:
            log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace("{{", "{{params.")
            f = sys._getframe(stack_depth + 1)
            log_params.location = {
                "line": f.f_lineno,
                "file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
                "method": text_type(f.f_code.co_name)
            }
            thread = _Thread.current()
            log_params.thread = {"name": thread.name, "id": thread.id}
        else:
            log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.")

        cls.main_log.write(log_template, log_params)

    def encode(self, value, pretty=False):
        if pretty:
            return pretty_json(value)

        try:
            with Timer("scrub", too_long=0.1):
                scrubbed = scrub(value)
            with Timer("encode", too_long=0.1):
                return text_type(self.encoder(scrubbed))
        except Exception as e:
            from mo_logs.exceptions import Except
            from mo_logs import Log

            e = Except.wrap(e)
            Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e)
            raise e

    def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None):
        self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD)
        self.rate = coalesce(rate, HG_REQUEST_PER_SECOND)
        self.cache_locker = Lock()
        self.cache = {}  # MAP FROM url TO (ready, headers, response, timestamp) PAIR
        self.no_cache = {}  # VERY SHORT TERM CACHE
        self.workers = []
        self.todo = Queue(APP_NAME+" todo")
        self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds))
        self.url = URL(source.url)
        self.db = Sqlite(database)
        self.inbound_rate = RateLogger("Inbound")
        self.outbound_rate = RateLogger("hg.mo")

        if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data:
            with self.db.transaction() as t:
                t.execute(
                    "CREATE TABLE cache ("
                    "   path TEXT PRIMARY KEY, "
                    "   headers TEXT, "
                    "   response TEXT, "
                    "   timestamp REAL "
                    ")"
                )

        self.threads = [
            Thread.run(APP_NAME+" worker" + text_type(i), self._worker)
            for i in range(CONCURRENCY)
        ]
        self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter)
        self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)

 def __init__(self, ident):
     self.id = ident
     if ident != -1:
         self.name = "Unknown Thread " + text_type(ident)
     self.child_lock = allocate_lock()
     self.children = []
     self.cprofiler = None

    def format(self, format="%Y-%m-%d %H:%M:%S"):
        try:
            return text_type(unix2datetime(self.unix).strftime(format))
        except Exception as e:
            from mo_logs import Log

            Log.error("Can not format {{value}} with {{format}}", value=unix2datetime(self.unix), format=format, cause=e)

def latin12unicode(value):
    if isinstance(value, text_type):
        Log.error("can not convert unicode from latin1")
    try:
        return text_type(value.decode('latin1'))
    except Exception as e:
        Log.error("Can not convert {{value|quote}} to unicode", value=value)

 def to_es_script(self, schema):
     return EsScript(
         type=NUMBER,
         expr=text_type(Date(self.value).unix),
         frum=self,
         schema=schema
     )

def queue_consumer(pull_queue, please_stop=None):
    queue = aws.Queue(pull_queue)
    time_offset = None
    request_count = 0

    while not please_stop:
        request = queue.pop(till=please_stop)
        if please_stop:
            break
        if not request:
            Log.note("Nothing in queue, pausing for 5 seconds...")
            (please_stop | Till(seconds=5)).wait()
            continue

        if SKIP_TRY_REQUESTS and 'try' in request.where['and'].eq.branch:
            Log.note("Skipping try revision.")
            queue.commit()
            continue

        now = Date.now().unix
        if time_offset is None:
            time_offset = now - request.meta.request_time

        next_request = request.meta.request_time + time_offset
        if next_request > now:
            Log.note("Next request in {{wait_time}}", wait_time=Duration(seconds=next_request - now))
            Till(till=next_request).wait()

        Thread.run("request "+text_type(request_count), one_request, request)
        request_count += 1
        queue.commit()

    def _find_revision(self, revision):
        please_stop = False
        locker = Lock()
        output = []
        queue = Queue("branches", max=2000)
        queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"])
        queue.add(THREAD_STOP)

        problems = []
        def _find(please_stop):
            for b in queue:
                if please_stop:
                    return
                try:
                    url = b.url + "json-info?node=" + revision
                    rev = self.get_revision(Revision(branch=b, changeset={"id": revision}))
                    with locker:
                        output.append(rev)
                    Log.note("Revision found at {{url}}", url=url)
                except Exception as f:
                    problems.append(f)

        threads = []
        for i in range(3):
            threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop))

        for t in threads:
            with assert_no_exception:
                t.join()

        return output

def problem_serializing(value, e=None):
    """
    THROW ERROR ABOUT SERIALIZING
    """
    from mo_logs import Log

    try:
        typename = type(value).__name__
    except Exception:
        typename = "<error getting name>"

    try:
        rep = text_type(repr(value))
    except Exception as _:
        rep = None

    if rep == None:
        Log.error(
            "Problem turning value of type {{type}} to json",
            type=typename,
            cause=e
        )
    else:
        Log.error(
            "Problem turning value ({{value}}) of type {{type}} to json",
            value=rep,
            type=typename,
            cause=e
        )