本文整理汇总了Python中qds_sdk.qubole.Qubole类的典型用法代码示例。如果您正苦于以下问题:Python Qubole类的具体用法?Python Qubole怎么用?Python Qubole使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Qubole类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: __init__
def __init__(self, *args, **kwargs):
conn = self.get_connection(kwargs['qubole_conn_id'])
Qubole.configure(api_token=conn.password, api_url=conn.host)
self.task_id = kwargs['task_id']
self.dag_id = kwargs['dag'].dag_id
self.kwargs = kwargs
self.cls = COMMAND_CLASSES[self.kwargs['command_type']]
self.cmd = None
开发者ID:sstm2,项目名称:incubator-airflow,代码行数:8,代码来源:qubole_hook.py
示例2: _configure_qubole
def _configure_qubole(self):
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('qds_connection')
logger.propagate = False
qdslog = logging.getLogger('qds')
if not self.config.API_TOKEN:
raise Exception("You didn't specify your QUBOLE_API_TOKEN in "
"your environment before running commands on "
"Qubole!\n. It can be found at http://api.qubole"
".com/users/edit")
Qubole.configure(api_token=self.config.API_TOKEN,
api_url=self.config.API_URL,
version=self.config.API_VERSION,
poll_interval=self.config.POLL_INTERVAL_SEC)
return qdslog
开发者ID:Betterment,项目名称:pinball,代码行数:15,代码来源:qubole_executor.py
示例3: update
def update(cls, cluster_id_label, cluster_info):
"""
Update the cluster with id/label `cluster_id_label` using information provided in
`cluster_info`.
"""
conn = Qubole.agent(version="v2")
return conn.put(cls.element_path(cluster_id_label), data=cluster_info)
开发者ID:rahul26goyal,项目名称:qds-sdk-py,代码行数:7,代码来源:clusterv2.py
示例4: terminate
def terminate(cls, cluster_id_label):
"""
Terminate the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "terminate"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:7,代码来源:cluster.py
示例5: clone
def clone(cls, cluster_id_label, cluster_info):
"""
Update the cluster with id/label `cluster_id_label` using information provided in
`cluster_info`.
"""
conn = Qubole.agent()
return conn.post(cls.element_path(cluster_id_label) + '/clone', data=cluster_info)
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:7,代码来源:cluster.py
示例6: find_by_name
def find_by_name(name):
conn = Qubole.agent()
if name is not None:
schedjson = conn.get(Scheduler.rest_entity_path, params={"name":name})
if schedjson["schedules"]:
return Scheduler(schedjson["schedules"][0])
return None
开发者ID:harshshah87,项目名称:qds-sdk-py,代码行数:7,代码来源:scheduler.py
示例7: poke
def poke(self, context):
conn = BaseHook.get_connection(self.qubole_conn_id)
Qubole.configure(api_token=conn.password, api_url=conn.host)
this.log.info('Poking: %s', self.data)
status = False
try:
status = self.sensor_class.check(self.data)
except Exception as e:
logging.exception(e)
status = False
this.log.info('Status of this Poke: %s', status)
return status
开发者ID:caseybrown89,项目名称:airflow,代码行数:16,代码来源:qubole_sensor.py
示例8: add_node
def add_node(cls, cluster_id_label, parameters=None):
"""
Add a node to an existing cluster
"""
conn = Qubole.agent()
parameters = {} if not parameters else parameters
return conn.post(cls.element_path(cluster_id_label) + "/nodes", data={"parameters" : parameters})
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:7,代码来源:cluster.py
示例9: index
def index(cls):
"""
Shows a list of all available reports by issuing a GET request to the
/reports endpoint.
"""
conn = Qubole.agent()
return conn.get(cls.rest_entity_path)
开发者ID:raghav-ghaiee-komli,项目名称:qds-sdk-py,代码行数:7,代码来源:report.py
示例10: start
def start(cls, cluster_id_label):
"""
Start the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "start"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:7,代码来源:cluster.py
示例11: find
def find(cls, name="default", **kwargs):
if (name is None) or (name == "default"):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError("Bad name %s" % name,
"Hadoop Clusters can only be named 'default' currently")
开发者ID:guptavishal,项目名称:qds-sdk-py,代码行数:7,代码来源:hadoop_cluster.py
示例12: get_results
def get_results(self, fp=sys.stdout, inline=True, delim=None):
"""
Fetches the result for the command represented by this object
Args:
`fp`: a file object to write the results to directly
"""
result_path = self.meta_data['results_resource']
conn = Qubole.agent()
r = conn.get(result_path, {'inline': inline})
if r.get('inline'):
if sys.version_info < (3, 0, 0):
fp.write(r['results'].encode('utf8'))
else:
import io
if isinstance(fp, io.TextIOBase):
fp.buffer.write(r['results'].encode('utf8'))
elif isinstance(fp, io.BufferedIOBase) or isinstance(fp, io.RawIOBase):
fp.write(r['results'].encode('utf8'))
else:
# Can this happen? Don't know what's the right thing to do in this case.
pass
else:
acc = Account.find()
boto_conn = boto.connect_s3(aws_access_key_id=acc.storage_access_key,
aws_secret_access_key=acc.storage_secret_key)
log.info("Starting download from result locations: [%s]" % ",".join(r['result_location']))
#fetch latest value of num_result_dir
num_result_dir = Command.find(self.id).num_result_dir
for s3_path in r['result_location']:
# In Python 3, in this case, `fp` should always be binary mode.
_download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim)
开发者ID:guptavishal,项目名称:qds-sdk-py,代码行数:35,代码来源:commands.py
示例13: list
def list(cls, label=None, cluster_id=None, state=None):
"""
List existing clusters present in your account.
Kwargs:
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
"""
if cluster_id is not None:
return cls.show(cluster_id)
if label is not None:
return cls.show(label)
conn = Qubole.agent(version="v2")
cluster_list = conn.get(cls.rest_entity_path)
if state is None:
# return the complete list since state is None
return conn.get(cls.rest_entity_path)
# filter clusters based on state
result = []
if 'clusters' in cluster_list:
for cluster in cluster_list['clusters']:
if state.lower() == cluster['state'].lower():
result.append(cluster)
return result
开发者ID:rahul26goyal,项目名称:qds-sdk-py,代码行数:26,代码来源:clusterv2.py
示例14: update_node
def update_node(cls, cluster_id_label, command, private_dns, parameters=None):
"""
Add a node to an existing cluster
"""
conn = Qubole.agent()
parameters = {} if not parameters else parameters
data = {"command" : command, "private_dns" : private_dns, "parameters" : parameters}
return conn.put(cls.element_path(cluster_id_label) + "/nodes", data)
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:8,代码来源:cluster.py
示例15: check
def check(cls, data):
"""
Method to call the sensors api with json payload
:param data: valid json object
:return: True or False
"""
conn = Qubole.agent()
return conn.post(cls.rest_entity_path, data=data)['status']
开发者ID:harshshah87,项目名称:qds-sdk-py,代码行数:8,代码来源:sensors.py
示例16: __init__
def __init__(self, access=None, secret = None, testmode=False, db_parallelism=None, mode=None, db_table=None, db_where=None, db_columns=None, db_boundary_query=None, db_extract_query=None, db_split_column=None, hive_table=None, part_spec=None, db_user=None, db_passwd=None, db_host=None, db_port=None, db_type=None, db_name=None, api_token = None, api_url=None, fetch_size = None):
self.temp_location = "/tmp/sqoop/"+uuid.uuid1().hex
self.tmp_dir = tempfile.mkdtemp(prefix="/media/ephemeral0/logs"+"/sqoop")
logger.info("Temp Directory is:" + self.tmp_dir)
self.access = access
self.secret = secret
self.api_token = api_token
self.api_url = api_url
self.fetch_size = fetch_size
self.redshift_sink = False
self.__loadImportParamsFromCid(testmode, db_parallelism, mode, db_table, db_where, db_columns, db_boundary_query, db_extract_query, db_split_column, hive_table, part_spec, db_user, db_passwd, db_host, db_port, db_type, db_name)
self.sqoop_cmd=["/usr/lib/sqoop-h2/bin/sqoop"]
self.sqoop_cmd.extend(["import"])
self.__addBasicOptions()
self.__extendCmdSpecificOptions()
Qubole.configure(api_token=api_token, api_url=api_url)
self.cluster_label = Cluster.show(os.popen("cat /usr/lib/hustler/bin/nodeinfo_src.sh | grep cluster_id").read().split("=")[1].strip().replace('"',''))['cluster']['label'][0]
开发者ID:quboletest,项目名称:solutions,代码行数:17,代码来源:db_import_client_hadoop2.py
示例17: create_update_clone_parser
def create_update_clone_parser(subparser, action=None):
# cloud config parser
cloud = Qubole.get_cloud()
cloud.create_parser(subparser)
# cluster info parser
ClusterInfoV2.cluster_info_parser(subparser, action)
# engine config parser
Engine.engine_parser(subparser)
开发者ID:rahul26goyal,项目名称:qds-sdk-py,代码行数:10,代码来源:clusterv2.py
示例18: cancel_id
def cancel_id(cls, id):
"""
Cancels command denoted by this id
Args:
`id`: command id
"""
conn = Qubole.agent()
data = {"status": "kill"}
return conn.put(cls.element_path(id), data)
开发者ID:pingali,项目名称:qds-sdk-py,代码行数:10,代码来源:commands.py
示例19: get_log_id
def get_log_id(cls, id):
"""
Fetches log for the command represented by this id
Args:
`id`: command id
"""
conn = Qubole.agent()
r = conn.get_raw(cls.element_path(id) + "/logs")
return r.text
开发者ID:pingali,项目名称:qds-sdk-py,代码行数:10,代码来源:commands.py
示例20: snapshot
def snapshot(cls, cluster_id_label, s3_location, backup_type):
"""
Create hbase snapshot full/incremental
"""
conn = Qubole.agent()
parameters = {}
parameters['s3_location'] = s3_location
if backup_type:
parameters['backup_type'] = backup_type
return conn.post(cls.element_path(cluster_id_label) + "/snapshots", data=parameters)
开发者ID:Ninelives21,项目名称:qds-sdk-py,代码行数:10,代码来源:cluster.py
注:本文中的qds_sdk.qubole.Qubole类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论