本文整理汇总了Python中pywebhdfs.webhdfs.PyWebHdfsClient类的典型用法代码示例。如果您正苦于以下问题:Python PyWebHdfsClient类的具体用法?Python PyWebHdfsClient怎么用?Python PyWebHdfsClient使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了PyWebHdfsClient类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: close
def close(self):
# drop the temp database
self._cursor.execute('USE %s' % self._temp_db)
self._cursor.execute('SHOW TABLES')
temp_tables = [x[0] for x in self._cursor.fetchall()]
for table in temp_tables:
self._cursor.execute('DROP TABLE IF EXISTS %s.%s' % (self._temp_db, table))
self._cursor.execute('SHOW FUNCTIONS')
temp_udfs = [x[1] for x in self._cursor.fetchall()]
for udf in temp_udfs:
self._cursor.execute('DROP FUNCTION IF EXISTS %s.%s' % (self._temp_db, udf))
self._cursor.execute('SHOW AGGREGATE FUNCTIONS')
temp_udas = [x[1] for x in self._cursor.fetchall()]
for uda in temp_udas:
self._cursor.execute('DROP AGGREGATE FUNCTION IF EXISTS %s.%s' % (self._temp_db, uda))
self._cursor.execute('USE default')
self._cursor.execute('DROP DATABASE IF EXISTS %s' % self._temp_db)
# drop the temp dir in HDFS
try:
from requests.exceptions import ConnectionError
from pywebhdfs.webhdfs import PyWebHdfsClient
hdfs_client = PyWebHdfsClient(host=self._nn_host,
port=self._webhdfs_port, user_name=self._hdfs_user)
hdfs_client.delete_file_dir(self._temp_dir.lstrip('/'), recursive=True)
except ImportError:
import sys
sys.stderr.write("Could not import requests or pywebhdfs. "
"You must delete the temporary directory manually: %s" % self._temp_dir)
except ConnectionError:
import sys
sys.stderr.write("Could not connect via pywebhdfs. "
"You must delete the temporary directory manually: %s" % self._temp_dir)
开发者ID:carlotorniai,项目名称:impyla,代码行数:32,代码来源:context.py
示例2: upload_file
def upload_file():
"""
Upload file
---
tags:
- Files
consumes: "multipart/form-data"
parameters:
- name: file
in: formData
required: true
paramType: body
dataType: file
type: file
responses:
200:
description: Return a successful message
401:
description: Unauthorized
400:
description: Bad Request
500:
description: Server Internal error
"""
# hard-code config information. You should imporove it.
hdfs = PyWebHdfsClient(host='webhdfs',port='50070', user_name='thanhson1085')
if request.method == 'POST':
file = request.files['file']
if file and allowed_file(file.filename):
filename = secure_filename(str(time.time()) + file.filename)
my_file = 'tmp/thanhson1085/data/' + filename
hdfs.create_file(my_file, file)
return jsonify({'success':'true'})
return jsonify({'success':'false'})
开发者ID:thanhson1085,项目名称:flask-webhdfs,代码行数:35,代码来源:controllers.py
示例3: HDFS
class HDFS(NDArray):
'''
HDFS storage
Parameters
----------
name : str
Name of directory to store text files (Path to the directory) without
a leading '/'
model : Model
If None, the model is taken from the 'with' context
vars : list of variables
Sampling values will be stored for these variables. If None.
'model.unobserved_RVs' is used
host : str
The IP address or hostname of the HDFS namenode. By default,
it is 'localhost'
port : str
The port number for WebHDFS on the namenode. By default, it
is '50070'
user_name : str
WebHDFS user_name used for authentication. By default, it is
None
'''
def __init__(self, name, model=None, vars=None, host='localhost', port='50070', user_name=None):
self.hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
try:
self.hdfs.list_dir(name)
except FileNotFound:
self.hdfs.make_dir(name)
super(HDFS, self).__init__(name, model, vars)
def close(self):
super(HDFS, self).close()
_dump_trace(self.name, self)
开发者ID:bkanuka,项目名称:pymc,代码行数:35,代码来源:hdfs.py
示例4: WhenTestingRenameOperation
class WhenTestingRenameOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs/old_dir'
self.new_path = '/user/hdfs/new_dir'
self.response = MagicMock()
self.rename = {"boolean": True}
self.response.json = MagicMock(return_value=self.rename)
def test_rename_throws_exception_for_not_ok(self):
self.response.status_code = http_client.BAD_REQUEST
self.requests.put.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.rename_file_dir(self.path, self.new_path)
def test_rename_returns_true(self):
self.response.status_code = http_client.OK
self.requests.put.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.rename_file_dir(self.path, self.new_path)
self.assertEqual(result, {"boolean": True})
开发者ID:avaranovich,项目名称:pywebhdfs,代码行数:32,代码来源:test_webhdfs.py
示例5: setup_common_oozie_libs
def setup_common_oozie_libs(name_node):
webhdfs_port = '14000'
webhdfs_user = 'hdfs'
platform_dir = 'user/deployment/platform'
lib_path_list = ['/opt/cloudera/parcels/CDH/lib/hbase/hbase-client.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-common.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-protocol.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-server.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/lib/htrace-core.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-hadoop-compat.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-it.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-prefix-tree.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/lib/zookeeper.jar',
'/opt/cloudera/parcels/CDH/lib/pig/piggybank.jar',
'/opt/cloudera/parcels/CDH/lib/spark/lib/spark-examples.jar']
# Setup a connection with hdfs using namenode.
hdfs_client = PyWebHdfsClient(host=name_node, port=webhdfs_port, user_name=webhdfs_user, timeout=None)
# Create directory on hadoop file system (HDFS).
hdfs_client.make_dir(platform_dir)
# Creates a new file on HDFS and write contents from local FS.
for path in lib_path_list:
platform_file = '%s/%s' % (platform_dir, os.path.basename(path))
logging.info('Copying source file: %s to HDFS path %s', path, platform_file)
with open(path) as file_data:
hdfs_client.create_file(platform_file, file_data, overwrite=True)
开发者ID:pndaproject,项目名称:platform-salt,代码行数:26,代码来源:cm_setup.py
示例6: WhenTestingOpenOperation
class WhenTestingOpenOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs'
self.file_data = '010101'
self.response = MagicMock()
self.response.text = self.file_data
def test_read_throws_exception_for_not_ok(self):
self.response.status_code = httplib.BAD_REQUEST
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.read_file(self.path)
def test_read_returns_file(self):
self.response.status_code = httplib.OK
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.read_file(self.path)
self.assertEqual(result, self.file_data)
开发者ID:waliaashish85,项目名称:pywebhdfs,代码行数:31,代码来源:test_webhdfs.py
示例7: WhenTestingDeleteOperation
class WhenTestingDeleteOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs/old_dir'
self.response = MagicMock()
def test_rename_throws_exception_for_not_ok(self):
self.response.status_code = httplib.BAD_REQUEST
self.requests.delete.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.delete_file_dir(self.path)
def test_rename_returns_true(self):
self.response.status_code = httplib.OK
self.requests.delete.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.delete_file_dir(self.path)
self.assertTrue(result)
开发者ID:waliaashish85,项目名称:pywebhdfs,代码行数:29,代码来源:test_webhdfs.py
示例8: submit
def submit(self, bund, files=[]):
hdfs = PyWebHdfsClient(host=os.environ["WEBHDFS_HOST"], port='14000', user_name='oozie')
for f in files:
hdfs.create_file("{}/{}".format(bund.path, f.name), f.read())
doc, tag, text = Doc().tagtext()
with tag("configuration"):
with tag("property"):
with tag("name"):
text("user.name")
with tag("value"):
text("oozie")
with tag("property"):
with tag("name"):
text("oozie.bundle.application.path")
with tag("value"):
text("/"+bund.path + "/" + bund.name)
configuration = doc.getvalue()
response = post("{0}/oozie/v1/jobs".format(self.url), data=configuration, headers={'Content-Type': 'application/xml'})
if response.status_code > 399:
print response.headers["oozie-error-message"]
print response.status_code
print response.content
开发者ID:orenmazor,项目名称:oozie.py,代码行数:27,代码来源:oozie_server.py
示例9: ship_udf
def ship_udf(ic, function, hdfs_path=None, udf_name=None, database=None,
overwrite=False):
# extract some information from the function
if udf_name is None:
udf_name = function.name
symbol = function.llvm_func.name
ir = function.llvm_module.to_bitcode()
return_type = udf_to_impala_type[function.signature.return_type.name]
arg_types = [udf_to_impala_type[arg.name]
for arg in function.signature.args[1:]]
# ship the IR to the cluster
hdfs_client = PyWebHdfsClient(host=ic._nn_host, port=ic._webhdfs_port,
user_name=ic._hdfs_user)
if hdfs_path is None:
hdfs_path = os.path.join(ic._temp_dir, udf_name + '.ll')
if not hdfs_path.endswith('.ll'):
raise ValueError("The HDFS file name must end with .ll")
hdfs_client.create_file(hdfs_path.lstrip('/'), ir, overwrite=overwrite)
# register the function in Impala
if database is None:
database = ic._temp_db
impala_name = '%s.%s(%s)' % (database, udf_name, ', '.join(arg_types))
if overwrite:
ic._cursor.execute("DROP FUNCTION IF EXISTS %s" % impala_name)
register_query = "CREATE FUNCTION %s RETURNS %s LOCATION '%s' SYMBOL='%s'" % (impala_name,
return_type, hdfs_path, symbol)
ic._cursor.execute(register_query)
开发者ID:carlotorniai,项目名称:impyla,代码行数:29,代码来源:__init__.py
示例10: WhenTestingDeleteXattrOperation
class WhenTestingDeleteXattrOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs/old_dir'
self.xattr = 'user.test'
self.response = MagicMock()
def test_delete_xattr_throws_exception_for_not_ok(self):
self.response.status_code = http_client.BAD_REQUEST
self.requests.return_value = self.response
with patch('requests.sessions.Session.put', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.delete_xattr(self.path, self.xattr)
def test_delete_xattr_returns_true(self):
self.response.status_code = http_client.OK
self.requests.return_value = self.response
with patch('requests.sessions.Session.put', self.requests):
result = self.webhdfs.delete_xattr(self.path, self.xattr)
self.assertTrue(result)
开发者ID:duedil-ltd,项目名称:pywebhdfs,代码行数:31,代码来源:test_webhdfs.py
示例11: WhenTestingCreateUri
class WhenTestingCreateUri(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.path = 'user/hdfs'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
def test_create_uri_no_kwargs(self):
op = operations.CREATE
uri = 'http://{host}:{port}/webhdfs/v1/' \
'{path}?op={op}&user.name={user}'\
.format(
host=self.host, port=self.port, path=self.path,
op=op, user=self.user_name)
result = self.webhdfs._create_uri(self.path, op)
self.assertEqual(uri, result)
def test_create_uri_with_kwargs(self):
op = operations.CREATE
mykey = 'mykey'
myval = 'myval'
uri = 'http://{host}:{port}/webhdfs/v1/' \
'{path}?op={op}&{key}={val}' \
'&user.name={user}' \
.format(
host=self.host, port=self.port, path=self.path,
op=op, key=mykey, val=myval, user=self.user_name)
result = self.webhdfs._create_uri(self.path, op, mykey=myval)
self.assertEqual(uri, result)
开发者ID:waliaashish85,项目名称:pywebhdfs,代码行数:32,代码来源:test_webhdfs.py
示例12: run
def run(self):
if ("agg" in self.arg):
#reading from a file to memory to stream later
with open(self.path,"rb") as f:
self.data_holder['data'] = json.dumps(cPickle.load(f));
#indicating that reading in memory is finished for this data
self.data_holder["indicator"]='ready';
elif("raw" in self.arg):
from pywebhdfs.webhdfs import PyWebHdfsClient;
hdfs = PyWebHdfsClient(host='cshadoop.boisestate.edu',port='50070', user_name='uacharya');
file_path = 'user/uacharya/flow/'+str(self.arg['d'])+'/node_'+str(self.arg['n'])+'/output.csv'
#reading the csv files in the memory
self.data_holder['data']= hdfs.read_file(file_path,buffersize=4096)
self.data_holder["indicator"]='ready';
elif("bitmap" in self.arg):
#putting the line data into a object to stream
with open(self.path+"/data.json","rb")as f:
self.data_holder['data'] = json.dumps(cPickle.load(f));
# with open(self.path+"\\data.json","rb")as f:
# output = cPickle.load(f);
#not loading images into memory if there is none images
if(self.data_holder['data']=='""'):
#indicating that reading in memory is finished for this data
self.data_holder['frames']=(0,[]);
self.data_holder["indicator"]='ready';
return;
# if(not output):
# self.data_holder['data']= msgpack.packb(output,use_bin_type=True);
# self.data_holder["indicator"]='ready';
# return;
#just in case there is some data to stream add all the PNGS to a list
# output['frames']=[];
content_length =0; #calculate the content length in bytes of all images to stream in total
PNGS=[]; #list to hold all the pngs data in memory
#reading all the images to memory to stream
for x in xrange(1,31):
buf_string = cStringIO.StringIO();
Image.open(self.path+"/imgs/"+str(x)+".png").save(buf_string, format="PNG", quality=100);
content_length = content_length+(buf_string.tell()+4);
PNGS.append(struct.pack('>I',buf_string.tell())+buf_string.getvalue());
buf_string.close();
# for x in xrange(1,31):
# buf_string = cStringIO.StringIO();
# Image.open(self.path+"\\imgs\\"+str(x)+".png").save(buf_string, format="PNG", quality=100);
# output['frames'].append(buf_string.getvalue());
# buf_string.close();
self.data_holder['frames']=(content_length,PNGS);
# self.data_holder['data']=msgpack.packb(output,use_bin_type=True);
#indicating that reading in memory is finished for this data
self.data_holder["indicator"]='ready';
else:
raise InvalidFormatError("the type of format is not available to read in memory");
开发者ID:uacharya,项目名称:WebServer,代码行数:58,代码来源:DataCreator.py
示例13: main
def main(argv):
"""
Main method.
This method performs the following tasks:
1. Parse command line arguments
2. Retrieve credentials and connect to Cloudant and WebHDFS
3. Connect to the Cloudant `_changes` feed for checkpointed document
consumption
4. Process each change individually.
5. Upon exception throwing, store the latest checkpoint to local file and
exit.
"""
# add options into the parser
parser = configureOptions()
(options, args) = parser.parse_args()
checkRequiredArguments(options, parser)
print options
# configurations
last_seq = options.last_seq
# get credential
perm_file = "%s/.clou" % os.environ["HOME"]
creds = get_creds(perm_file)
# connect to source database
s = Server("https://%s:%[email protected]%s" % (creds["cloudant_user"], creds["cloudant_pwd"], options.uri))
db = s[options.dbname]
# print db.info()
# connect to target hdfs cluster
hdfs = PyWebHdfsClient(host=options.hdfs_host, port=options.hdfs_port, user_name=creds["hdfs_user"])
hdfs.make_dir(options.hdfs_path)
# and here we consume the cloudant `_changes` feed
counter = 0
changestream = ChangesStream(db, include_docs=True, heartbeat=True, since=last_seq)
for c in changestream:
# print c
try:
if counter % 100 == 0:
checkpoint(last_seq)
seq = processChange(hdfs, c, options.hdfs_path)
if seq: # protect against the last line being blank
last_seq = seq
counter += 1
except Exception:
traceback.print_exc()
checkpoint(last_seq)
os._exit(1)
checkpoint(last_seq)
开发者ID:phnz,项目名称:hdfs,代码行数:54,代码来源:cloudant2hdfs.py
示例14: __init__
def __init__(self, remote=None, namenodes=None, **kwargs):
self.remote = remote
self.namenodes = namenodes or []
PyWebHdfsClient.__init__(self, **kwargs)
if self.namenodes and 'path_to_hosts' not in kwargs:
self.path_to_hosts = [('.*', self.namenodes)]
# Override base uri
self.base_uri_pattern = kwargs.get('base_uri_pattern', "http://{host}/webhdfs/v1/").format(
host="{host}")
开发者ID:djKooks,项目名称:omniduct,代码行数:12,代码来源:_webhdfs_helpers.py
示例15: WhenTestingCreateUri
class WhenTestingCreateUri(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.path = 'user/hdfs'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
def test_create_uri_no_kwargs(self):
op = operations.CREATE
uri = 'http://{{host}}:{port}/webhdfs/v1/' \
'{path}?op={op}&user.name={user}'\
.format(port=self.port, path=self.path,
op=op, user=self.user_name)
result = self.webhdfs._create_uri(self.path, op)
self.assertEqual(uri, result)
def test_create_uri_with_kwargs(self):
op = operations.CREATE
mykey = 'mykey'
myval = 'myval'
uri = 'http://{{host}}:{port}/webhdfs/v1/' \
'{path}?op={op}&{key}={val}' \
'&user.name={user}' \
.format(
port=self.port, path=self.path,
op=op, key=mykey, val=myval, user=self.user_name)
result = self.webhdfs._create_uri(self.path, op,
mykey=myval)
self.assertEqual(uri, result)
def test_create_uri_with_leading_slash(self):
op = operations.CREATE
uri_path_no_slash = self.webhdfs._create_uri(self.path, op)
uri_path_with_slash = self.webhdfs._create_uri('/' + self.path, op)
self.assertEqual(uri_path_no_slash, uri_path_with_slash)
def test_create_uri_with_unicode_path(self):
op = operations.CREATE
mykey = 'mykey'
myval = 'myval'
path = u'die/Stra\xdfe'
quoted_path = 'die/Stra%C3%9Fe'
uri = 'http://{{host}}:{port}/webhdfs/v1/' \
'{path}?op={op}&{key}={val}' \
'&user.name={user}' \
.format(
port=self.port, path=quoted_path,
op=op, key=mykey, val=myval, user=self.user_name)
result = self.webhdfs._create_uri(path, op, mykey=myval)
self.assertEqual(uri, result)
开发者ID:avaranovich,项目名称:pywebhdfs,代码行数:53,代码来源:test_webhdfs.py
示例16: WhenTestingFileExistsOperation
class WhenTestingFileExistsOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs/old_dir'
self.response = MagicMock()
self.file_status = {
"FileStatus": {
"accessTime": 0,
"blockSize": 0,
"group": "supergroup",
"length": 0,
"modificationTime": 1320173277227,
"owner": "webuser",
"pathSuffix": "",
"permission": "777",
"replication": 0,
"type": "DIRECTORY"
}
}
self.response.json = MagicMock(return_value=self.file_status)
def test_exists_throws_exception_for_error(self):
self.response.status_code = http_client.BAD_REQUEST
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.exists_file_dir(self.path)
def test_exists_returns_true(self):
self.response.status_code = http_client.OK
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
self.assertTrue(self.webhdfs.exists_file_dir(self.path))
def test_exists_returns_false(self):
self.response.status_code = http_client.NOT_FOUND
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
self.assertFalse(self.webhdfs.exists_file_dir(self.path))
开发者ID:avaranovich,项目名称:pywebhdfs,代码行数:50,代码来源:test_webhdfs.py
示例17: WhenTestingCreateOperation
class WhenTestingCreateOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.location = 'redirect_uri'
self.path = 'user/hdfs'
self.file_data = '010101'
self.init_response = MagicMock()
self.init_response.headers = {'location': self.location}
self.response = MagicMock()
self.expected_headers = {'content-type': 'application/octet-stream'}
def test_create_throws_exception_for_no_redirect(self):
self.init_response.status_code = httplib.BAD_REQUEST
self.response.status_code = httplib.CREATED
self.requests.put.side_effect = [self.init_response, self.response]
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.create_file(self.path, self.file_data)
def test_create_throws_exception_for_not_created(self):
self.init_response.status_code = httplib.TEMPORARY_REDIRECT
self.response.status_code = httplib.BAD_REQUEST
self.requests.put.side_effect = [self.init_response, self.response]
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.create_file(self.path, self.file_data)
def test_create_returns_file_location(self):
self.init_response.status_code = httplib.TEMPORARY_REDIRECT
self.response.status_code = httplib.CREATED
self.put_method = MagicMock(
side_effect=[self.init_response, self.response])
self.requests.put = self.put_method
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.create_file(self.path, self.file_data)
self.assertTrue(result)
self.put_method.assert_called_with(
self.location, headers=self.expected_headers, data=self.file_data)
开发者ID:ProjectMeniscus,项目名称:pywebhdfs,代码行数:49,代码来源:test_webhdfs.py
示例18: load
def load(self, job, task, fifo):
self.job = job
self.task = task
self.fifo = fifo
self.key = None
self.script_proc = None
self.decompress_obj = None
self.pycurl_callback_exception = None
if task.data['scheme'] == 's3':
self.is_anonymous = job.spec.source.aws_access_key is None or job.spec.source.aws_secret_key is None
if self.is_anonymous:
s3_conn = S3Connection(anon=True)
else:
s3_conn = S3Connection(job.spec.source.aws_access_key, job.spec.source.aws_secret_key)
bucket = s3_conn.get_bucket(task.data['bucket'])
try:
self.key = bucket.get_key(task.data['key_name'])
except S3ResponseError as e:
raise WorkerException("Received %s %s accessing `%s`, aborting" % (e.status, e.reason, task.data['key_name']))
elif task.data['scheme'] == 'hdfs':
fname = task.data['key_name']
client = PyWebHdfsClient(
job.spec.source.hdfs_host,
job.spec.source.webhdfs_port,
user_name=job.spec.source.hdfs_user)
try:
filesize = client.get_file_dir_status(fname)['FileStatus']['length']
except pywebhdfs.errors.FileNotFound:
raise WorkerException("File '%s' does not exist on HDFS" % fname)
self.key = AttrDict({'name': fname, 'size': filesize})
elif task.data['scheme'] == 'file':
globber = glob2.Globber()
fname = globber._normalize_string(task.data['key_name'])
if not os.path.exists(fname):
raise WorkerException("File '%s' does not exist on this filesystem" % fname)
elif not os.path.isfile(fname):
raise WorkerException("File '%s' exists, but is not a file" % fname)
self.key = AttrDict({'name': fname, 'size': os.path.getsize(fname)})
else:
raise WorkerException('Unsupported job with paths: %s' % [ str(p) for p in self.job.paths ])
if self.key is None:
raise WorkerException('Failed to find key associated with task ID %s' % task.task_id)
self.metrics = DownloadMetrics(self.key.size)
开发者ID:Vlad777,项目名称:memsql-loader,代码行数:49,代码来源:downloader.py
示例19: create_data_from_station_data
def create_data_from_station_data(first, second):
"""this function creates the data analyzing the two stations in comparison"""
global hdfs; #global hdfs object
global hbase; #global hbase object
if(hdfs is None):
from pywebhdfs.webhdfs import PyWebHdfsClient;
hdfs = PyWebHdfsClient(host='cshadoop.boisestate.edu',port='50070', user_name='uacharya');
if(hbase is None):
import happybase;
hbase = happybase.ConnectionPool(size=1,host='cshadoop.boisestate.edu');
date_for_comparision = first["Date"].strip();
# creating directory for each date
try:
hdfs.get_file_dir_status('user/uacharya/simulation/'+date_for_comparision);
except Exception:
# directory to hold dataset in csv file for reach node in wall display starting from 1 to 9
for index in range(1, 10):
content = 'Date,ID,Source,Destination,S_Lat,S_Lon,D_Lat,D_Lon,Wind_Lat,Wind_Lon,Wind_Velocity\n';
try:
hdfs.create_file('user/uacharya/simulation/'+date_for_comparision+'/node'+str(index)+'/output.csv',content,replication=1);
except Exception:
continue;
dataset = {'node_1':[],'node_2':[],'node_3':[],'node_4':[],'node_5':[],'node_6':[],'node_7':[],'node_8':[],'node_9':[]};
for data in broadcast_variable.value:
compare_data_between(date_for_comparision, first, data,dataset);
# for key in dataset:
# if(len(dataset[key])!=0):
# content = "\n".join(dataset[key]);
# content +="\n";
# while(True):
# try:
# hdfs.append_file('user/uacharya/simulation/'+date+'/'+key+'/output.csv',content,buffersize=4096);
# break;
# except Exception:
# time.sleep(0.2);
# continue;
dataset.clear(); #clearing the dictionary
# append over here after all the global variable has been made
return second;
开发者ID:uacharya,项目名称:WebServer,代码行数:49,代码来源:TestingSpark.py
示例20: setUp
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.path = 'user/hdfs'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
开发者ID:waliaashish85,项目名称:pywebhdfs,代码行数:7,代码来源:test_webhdfs.py
注:本文中的pywebhdfs.webhdfs.PyWebHdfsClient类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论