本文整理汇总了Python中remotespark.utils.log.Log类的典型用法代码示例。如果您正苦于以下问题:Python Log类的具体用法?Python Log怎么用?Python Log使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Log类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: Command
class Command(ObjectWithGuid):
def __init__(self, code, spark_events=None):
super(Command, self).__init__()
self.code = textwrap.dedent(code)
self.logger = Log(u"Command")
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
def __eq__(self, other):
return self.code == other.code
def __ne__(self, other):
return not self == other
def execute(self, session):
self._spark_events.emit_statement_execution_start_event(session.guid, session.kind, session.id, self.guid)
statement_id = -1
try:
session.wait_for_idle()
data = {u"code": self.code}
response = session.http_client.post_statement(session.id, data)
statement_id = response[u'id']
output = self._get_statement_output(session, statement_id)
except Exception as e:
self._spark_events.emit_statement_execution_end_event(session.guid, session.kind, session.id,
self.guid, statement_id, False, e.__class__.__name__,
str(e))
raise
else:
self._spark_events.emit_statement_execution_end_event(session.guid, session.kind, session.id,
self.guid, statement_id, True, "", "")
return output
def _get_statement_output(self, session, statement_id):
statement_running = True
out = u""
while statement_running:
statement = session.http_client.get_statement(session.id, statement_id)
status = statement[u"state"]
self.logger.debug(u"Status of statement {} is {}.".format(statement_id, status))
if status == u"running":
session.sleep()
else:
statement_running = False
statement_output = statement[u"output"]
if statement_output[u"status"] == u"ok":
out = (True, statement_output[u"data"][u"text/plain"])
elif statement_output[u"status"] == u"error":
out = (False,
statement_output[u"evalue"] + u"\n" + u"".join(statement_output[u"traceback"]))
else:
raise LivyUnexpectedStatusException(u"Unknown output status from Livy: '{}'"
.format(statement_output[u"status"]))
return out
开发者ID:msftristew,项目名称:sparkmagic,代码行数:59,代码来源:command.py
示例2: __init__
class EventsHandler:
def __init__(self):
self.logger = Log("EventsHandler")
def handle_event(self, kwargs_list):
"""
Storing the Event details using the logger.
"""
event_line = ",".join("{}: {}".format(key, arg) for key, arg in kwargs_list)
self.logger.info(event_line)
开发者ID:cfregly,项目名称:sparkmagic,代码行数:10,代码来源:eventshandler.py
示例3: __init__
def __init__(self, code, spark_events=None):
super(Command, self).__init__()
self.code = textwrap.dedent(code)
self.logger = Log(u"Command")
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
开发者ID:msftristew,项目名称:sparkmagic,代码行数:7,代码来源:command.py
示例4: __init__
def __init__(self, shell, data=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
try:
should_serialize = conf.serialize()
if should_serialize:
self.logger.debug("Serialization enabled.")
self.magics_home_path = get_magics_home_path()
path_to_serialize = join_paths(self.magics_home_path, "state.json")
self.logger.debug("Will serialize to {}.".format(path_to_serialize))
self.spark_controller = SparkController(self.ipython_display, serialize_path=path_to_serialize)
else:
self.logger.debug("Serialization NOT enabled.")
except KeyError:
self.logger.error("Could not read env vars for serialization.")
self.logger.debug("Initialized spark magics.")
开发者ID:gdtm86,项目名称:sparkmagic,代码行数:25,代码来源:sparkmagicsbase.py
示例5: __init__
def __init__(self, implementation, implementation_version, language, language_version, language_info,
session_language, user_code_parser=None, **kwargs):
# Required by Jupyter - Override
self.implementation = implementation
self.implementation_version = implementation_version
self.language = language
self.language_version = language_version
self.language_info = language_info
# Override
self.session_language = session_language
super(SparkKernelBase, self).__init__(**kwargs)
self.logger = Log("_jupyter_kernel".format(self.session_language))
self._fatal_error = None
self.ipython_display = IpythonDisplay()
if user_code_parser is None:
self.user_code_parser = UserCodeParser()
else:
self.user_code_parser = user_code_parser
# Disable warnings for test env in HDI
requests.packages.urllib3.disable_warnings()
if not kwargs.get("testing", False):
self._load_magics_extension()
self._change_language()
if conf.use_auto_viz():
self._register_auto_viz()
开发者ID:mindis,项目名称:sparkmagic,代码行数:31,代码来源:sparkkernelbase.py
示例6: __init__
def __init__(self, reader_writer):
assert reader_writer is not None
self.logger = Log("ClientManagerStateSerializer")
self._ipython_display = IpythonDisplay()
self._reader_writer = reader_writer
开发者ID:gdtm86,项目名称:sparkmagic,代码行数:7,代码来源:clientmanagerstateserializer.py
示例7: SparkMagicBase
class SparkMagicBase(Magics):
def __init__(self, shell, data=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
try:
should_serialize = conf.serialize()
if should_serialize:
self.logger.debug("Serialization enabled.")
self.magics_home_path = get_magics_home_path()
path_to_serialize = join_paths(self.magics_home_path, "state.json")
self.logger.debug("Will serialize to {}.".format(path_to_serialize))
self.spark_controller = SparkController(self.ipython_display, serialize_path=path_to_serialize)
else:
self.logger.debug("Serialization NOT enabled.")
except KeyError:
self.logger.error("Could not read env vars for serialization.")
self.logger.debug("Initialized spark magics.")
def execute_sqlquery(self, sqlquery, session, output_var, quiet):
try:
df = self.spark_controller.run_cell_sql(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
except DataFrameParseException as e:
self.ipython_display.send_error(e.out)
return None
@staticmethod
def print_endpoint_info(info_sessions):
sessions_info = [" {}".format(i) for i in info_sessions]
print("""Info for endpoint:
Sessions:
{}
""".format("\n".join(sessions_info)))
开发者ID:gdtm86,项目名称:sparkmagic,代码行数:47,代码来源:sparkmagicsbase.py
示例8: __init__
def __init__(self, client_factory, reader_writer):
assert client_factory is not None
assert reader_writer is not None
self.logger = Log("ClientManagerStateSerializer")
self._client_factory = client_factory
self._reader_writer = reader_writer
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:8,代码来源:clientmanagerstateserializer.py
示例9: Command
class Command(ObjectWithGuid):
def __init__(self, code):
super(Command, self).__init__()
self.code = textwrap.dedent(code)
self.logger = Log("Command")
def __eq__(self, other):
return self.code == other.code
def __ne__(self, other):
return not self == other
def execute(self, session):
session.wait_for_idle()
data = {"code": self.code}
response = session.http_client.post_statement(session.id, data)
statement_id = response['id']
return self._get_statement_output(session, statement_id)
def _get_statement_output(self, session, statement_id):
statement_running = True
out = ""
while statement_running:
statement = session.http_client.get_statement(session.id, statement_id)
status = statement["state"]
self.logger.debug("Status of statement {} is {}.".format(statement_id, status))
if status == "running":
session.sleep()
else:
statement_running = False
statement_output = statement["output"]
if statement_output["status"] == "ok":
out = (True, statement_output["data"]["text/plain"])
elif statement_output["status"] == "error":
out = (False,
statement_output["evalue"] + "\n" + "".join(statement_output["traceback"]))
else:
raise ValueError("Unknown output status: '{}'".format(statement_output["status"]))
return out
开发者ID:cfregly,项目名称:sparkmagic,代码行数:43,代码来源:command.py
示例10: __init__
def __init__(self, ipython_display, serialize_path=None):
self.logger = Log("SparkController")
self.ipython_display = ipython_display
self.client_factory = LivyClientFactory()
if serialize_path is not None:
serializer = ClientManagerStateSerializer(self.client_factory, FileSystemReaderWriter(serialize_path))
self.client_manager = ClientManager(serializer)
else:
self.client_manager = ClientManager()
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:10,代码来源:sparkcontroller.py
示例11: __init__
def __init__(self, endpoint, headers, retry_policy):
self._endpoint = endpoint
self._headers = headers
self._retry_policy = retry_policy
self.logger = Log("ReliableHttpClient")
self.verify_ssl = not conf.ignore_ssl_errors()
if not self.verify_ssl:
self.logger.debug("ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks.")
requests.packages.urllib3.disable_warnings()
开发者ID:mindis,项目名称:sparkmagic,代码行数:10,代码来源:reliablehttpclient.py
示例12: __init__
def __init__(self, url, headers, username, password, retry_policy):
self._url = url.rstrip("/")
self._headers = headers
self._username = username
self._password = password
self._retry_policy = retry_policy
self.logger = Log("ReliableHttpClient")
self._do_not_authenticate = self._username == "" and self._password == ""
self.verify_ssl = not conf.ignore_ssl_errors()
if self.verify_ssl:
self.logger.debug("ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks.")
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:13,代码来源:reliablehttpclient.py
示例13: __init__
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
开发者ID:mindis,项目名称:sparkmagic,代码行数:13,代码来源:sparkmagicsbase.py
示例14: SparkMagicBase
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction):
return SQLQuery(cell, samplemethod, maxrows, samplefraction)
@staticmethod
def print_endpoint_info(info_sessions):
sessions_info = [" {}".format(i) for i in info_sessions]
print("""Info for endpoint:
Sessions:
{}
""".format("\n".join(sessions_info)))
开发者ID:mindis,项目名称:sparkmagic,代码行数:37,代码来源:sparkmagicsbase.py
示例15: __init__
def __init__(self, http_client, properties, ipython_display,
session_id=-1, sql_created=None, spark_events=None):
super(LivySession, self).__init__()
assert u"kind" in list(properties.keys())
kind = properties[u"kind"]
self.properties = properties
self.ipython_display = ipython_display
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
status_sleep_seconds = conf.status_sleep_seconds()
statement_sleep_seconds = conf.statement_sleep_seconds()
wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds()
assert status_sleep_seconds > 0
assert statement_sleep_seconds > 0
assert wait_for_idle_timeout_seconds > 0
if session_id == -1 and sql_created is True:
raise BadUserDataException(u"Cannot indicate sql state without session id.")
self.logger = Log(u"LivySession")
kind = kind.lower()
if kind not in constants.SESSION_KINDS_SUPPORTED:
raise BadUserDataException(u"Session of kind '{}' not supported. Session must be of kinds {}."
.format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED)))
if session_id == -1:
self.status = constants.NOT_STARTED_SESSION_STATUS
sql_created = False
else:
self.status = constants.BUSY_SESSION_STATUS
self._app_id = None
self._logs = u""
self._http_client = http_client
self._status_sleep_seconds = status_sleep_seconds
self._statement_sleep_seconds = statement_sleep_seconds
self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds
self.kind = kind
self.id = session_id
self.created_sql_context = sql_created
开发者ID:msftristew,项目名称:sparkmagic,代码行数:45,代码来源:livysession.py
示例16: __init__
def __init__(self, serializer=None):
serialize_periodically = False
serialize_period = 3
if serializer is not None:
serialize_periodically = conf.serialize_periodically()
serialize_period = conf.serialize_period_seconds()
self.logger = Log("ClientManager")
self._livy_clients = dict()
self._serializer = serializer
self._serialize_timer = None
if self._serializer is not None:
for (name, client) in self._serializer.deserialize_state():
self.add_client(name, client)
if serialize_periodically:
self._serialize_state_periodically(serialize_period)
开发者ID:gdtm86,项目名称:sparkmagic,代码行数:20,代码来源:clientmanager.py
示例17: __init__
def __init__(self, implementation, implementation_version, language, language_version, language_info,
kernel_conf_name, session_language, client_name, **kwargs):
# Required by Jupyter - Override
self.implementation = implementation
self.implementation_version = implementation_version
self.language = language
self.language_version = language_version
self.language_info = language_info
# Override
self.kernel_conf_name = kernel_conf_name
self.session_language = session_language
self.client_name = client_name
super(SparkKernelBase, self).__init__(**kwargs)
self._logger = Log(self.client_name)
self._session_started = False
self._fatal_error = None
self._ipython_display = IpythonDisplay()
self.user_command_parser = UserCommandParser()
# Disable warnings for test env in HDI
requests.packages.urllib3.disable_warnings()
if not kwargs.get("testing", False):
configuration = self._get_configuration()
if not configuration:
# _get_configuration() sets the error for us so we can just return now.
# The kernel is not in a good state and all do_execute calls will
# fail with the fatal error.
return
(username, password, url) = configuration
self.connection_string = get_connection_string(url, username, password)
self._load_magics_extension()
if conf.use_auto_viz():
self._register_auto_viz()
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:38,代码来源:sparkkernelbase.py
示例18: __init__
def __init__(self, ipython_display, http_client, session_id, sql_created, properties):
assert "kind" in properties.keys()
kind = properties["kind"]
self.properties = properties
self.ipython_display = ipython_display
status_sleep_seconds = conf.status_sleep_seconds()
statement_sleep_seconds = conf.statement_sleep_seconds()
create_sql_context_timeout_seconds = conf.create_sql_context_timeout_seconds()
assert status_sleep_seconds > 0
assert statement_sleep_seconds > 0
assert create_sql_context_timeout_seconds > 0
if session_id == "-1" and sql_created is True:
raise ValueError("Cannot indicate sql state without session id.")
self.logger = Log("LivySession")
kind = kind.lower()
if kind not in Constants.session_kinds_supported:
raise ValueError("Session of kind '{}' not supported. Session must be of kinds {}."
.format(kind, ", ".join(Constants.session_kinds_supported)))
if session_id == "-1":
self._status = Constants.not_started_session_status
sql_created = False
else:
self._status = Constants.busy_session_status
self._logs = ""
self._http_client = http_client
self._status_sleep_seconds = status_sleep_seconds
self._statement_sleep_seconds = statement_sleep_seconds
self._create_sql_context_timeout_seconds = create_sql_context_timeout_seconds
self._state = LivySessionState(session_id, http_client.connection_string,
kind, sql_created)
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:37,代码来源:livysession.py
示例19: LivySession
class LivySession(object):
"""Session that is livy specific."""
def __init__(self, ipython_display, http_client, session_id, sql_created, properties):
assert "kind" in properties.keys()
kind = properties["kind"]
self.properties = properties
self.ipython_display = ipython_display
status_sleep_seconds = conf.status_sleep_seconds()
statement_sleep_seconds = conf.statement_sleep_seconds()
create_sql_context_timeout_seconds = conf.create_sql_context_timeout_seconds()
assert status_sleep_seconds > 0
assert statement_sleep_seconds > 0
assert create_sql_context_timeout_seconds > 0
if session_id == "-1" and sql_created is True:
raise ValueError("Cannot indicate sql state without session id.")
self.logger = Log("LivySession")
kind = kind.lower()
if kind not in Constants.session_kinds_supported:
raise ValueError("Session of kind '{}' not supported. Session must be of kinds {}."
.format(kind, ", ".join(Constants.session_kinds_supported)))
if session_id == "-1":
self._status = Constants.not_started_session_status
sql_created = False
else:
self._status = Constants.busy_session_status
self._logs = ""
self._http_client = http_client
self._status_sleep_seconds = status_sleep_seconds
self._statement_sleep_seconds = statement_sleep_seconds
self._create_sql_context_timeout_seconds = create_sql_context_timeout_seconds
self._state = LivySessionState(session_id, http_client.connection_string,
kind, sql_created)
def __str__(self):
return "Session id: {}\tKind: {}\tState: {}".format(self.id, self.kind, self._status)
def get_state(self):
return self._state
def start(self):
"""Start the session against actual livy server."""
self.logger.debug("Starting '{}' session.".format(self.kind))
r = self._http_client.post("/sessions", [201], self.properties)
self._state.session_id = str(r.json()["id"])
self._status = str(r.json()["state"])
self.ipython_display.writeln("Creating SparkContext as 'sc'")
self.logger.debug("Session '{}' started.".format(self.kind))
def create_sql_context(self):
"""Create a sqlContext object on the session. Object will be accessible via variable 'sqlContext'."""
if self.started_sql_context:
return
self.logger.debug("Starting '{}' sql and hive session.".format(self.kind))
self.ipython_display.writeln("Creating SqlContext as 'sqlContext'")
self._create_context(Constants.context_name_sql)
self.ipython_display.writeln("Creating HiveContext as 'hiveContext'")
self._create_context(Constants.context_name_hive)
self._state.sql_context_created = True
def _create_context(self, context_type):
if context_type == Constants.context_name_sql:
command = self._get_sql_context_creation_command()
elif context_type == Constants.context_name_hive:
command = self._get_hive_context_creation_command()
else:
raise ValueError("Cannot create context of type {}.".format(context_type))
try:
self.wait_for_idle(self._create_sql_context_timeout_seconds)
self.execute(command)
self.logger.debug("Started '{}' {} session.".format(self.kind, context_type))
except LivyClientTimeoutError:
raise LivyClientTimeoutError("Failed to create the {} context in time. Timed out after {} seconds."
.format(context_type, self._create_sql_context_timeout_seconds))
@property
def id(self):
return self._state.session_id
@property
def started_sql_context(self):
return self._state.sql_context_created
@property
def kind(self):
return self._state.kind
#.........这里部分代码省略.........
开发者ID:MohamedElKamhawy,项目名称:sparkmagic,代码行数:101,代码来源:livysession.py
示例20: ClientManagerStateSerializer
class ClientManagerStateSerializer(object):
"""Livy client manager state serializer"""
def __init__(self, reader_writer):
assert reader_writer is not None
self.logger = Log("ClientManagerStateSerializer")
self._ipython_display = IpythonDisplay()
self._reader_writer = reader_writer
def deserialize_state(self):
self.logger.debug("Deserializing state.")
clients_to_return = []
lines = self._reader_writer.read_lines()
line = ''.join(lines).strip()
if line != '':
self.logger.debug("Read content. Converting to JSON.")
json_str = json.loads(line)
clients = json_str["clients"]
for client in clients:
# Ignore version for now
name = client["name"]
session_id = client["id"]
sql_context_created = client["sqlcontext"]
kind = client["kind"].lower()
connection_string = client["connectionstring"]
session = self._create_livy_session(connection_string, {"kind": kind}, self._ipython_display,
session_id, sql_context_created)
# Do not start session automatically. Just create it but skip is not existent.
try:
# Get status to know if it's alive or not.
status = session.status
if not session.is_final_status(status):
self.logger.debug("Adding session {}".format(session_id))
client_obj = self._create_livy_client(session)
clients_to_return.append((name, client_obj))
else:
self.logger.error("Skipping serialized session '{}' because session was in status {}."
.format(session.id, status))
except (ValueError, ConnectionError) as e:
self.logger.error("Skipping serialized session '{}' because {}".format(session.id, str(e)))
else:
self.logger.debug("Empty manager state found.")
return clients_to_return
def serialize_state(self, name_client_dictionary):
self.logger.debug("Serializing state.")
serialized_clients = []
for name in list(name_client_dictionary.keys()):
client = name_client_dictionary[name]
serialized_client = client.serialize()
serialized_client["name"] = name
serialized_clients.append(serialized_client)
serialized_str = json.dumps({"clients": serialized_clients})
self._reader_writer.overwrite_with_line(serialized_str)
def _create_livy_session(self, connection_string, properties, ipython_display,
session_id, sql_context_created):
return LivySession.from_connection_string(connection_string, properties, ipython_display,
session_id, sql_context_created)
def _create_livy_client(self, session):
return LivyClient(session)
开发者ID:gdtm86,项目名称:sparkmagic,代码行数:73,代码来源:clientmanagerstateserializer.py
注:本文中的remotespark.utils.log.Log类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论