def POST(self, id=None, outputs=None, **kwargs):
		if id is None:
			metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body))
			for inp in metadata.get("inputs",[]):
				if '_id' in inp: inp['_id'] = self.parse_value(inp['_id'])

			id = Transformations.insert_one(metadata).inserted_id

			# for inp in metadata.get("inputs",[]):
			# 	Products.update_one({'_id':inp['_id']},{'$push':{'transformations':{'_id':id}}})

			return self.build_response('ok', **dict(transformation={'_id':id}))

		else:
			if outputs == 'outputs':
				metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body))
				resp = simplejson.loads(ProductsController().POST(id=None, metadata=metadata))
				if resp['status'] != 'ok':
					return self.build_response('error',
						message='error creating output product: %s' % resp['message'])
				Transformations.update_one({'_id':ObjectId(id)},{'$push':{'outputs':{'_id':ObjectId(resp['data']['product']['_id'])}}})
				return self.build_response('ok', **dict(product=resp['data']['product']))

			else:
				return self.build_response('error',message='cannot POST metadata for an existing transformation. Use PUT instead.')

def index(request):
    conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20desc&fq=raceStart:[*%20TO%20NOW]&rows=5')
    results = simplejson.load(conn)
    conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20asc&fq=raceStart:[NOW%20TO%20*]&rows=5')
    events = simplejson.load(conn)

    return render_to_response('runapp/index2.html', {'results': results['grouped']['eventName']['groups'], 'events': events['grouped']['eventName']['groups'], 'view':'index'}, context_instance=RequestContext(request) )

def generate(mapping, schema, filename):
  mapping = json.load(open(mapping))
  schema = json.load(open(schema))
  schema.pop('$schema', 0)

  # Index for which keys need to be marked as having exact counterparts
  exact_index = get_exact_fields(mapping)
  # Index to lookup a non-array version of key name (so it maps to ES)
  es_items_index = generate_es_to_schema_index(schema)
  # All the fields to process from the schema
  schema_fields = get_schema_fields(schema)

  for field in schema_fields:
    type_field = field + '.' + 'type'
    # No need expand these complex types with DEFAULT attributes
    if get_deep(schema, type_field) in ['object', 'array']:
      continue
    if not field:
      continue

    es_field = es_items_index.get(field, None)
    is_exact = True if es_field in exact_index else False

    # Expand each of the keys to match DEFAULTS.
    # set_deep() will not overwrite data if it does exist, but will add if it
    # does not
    for key, value in DEFAULTS.items():
      full_key = field + '.' + key
      if is_exact and key == 'is_exact':
        value = True

      set_deep(schema, full_key, value)

  make_yaml(schema, filename)

def run_compare():
    with open('url-content-read.json') as f:
        readability_text = simplejson.load(f)

    with open('url-content-fetcher.json') as f:
        fetcher_text = simplejson.load(f)

    cnt = 0
    z_cnt = 0
    dmp = diff_match_patch()
    rets = []
    for key, value in readability_text.items():
        if key in fetcher_text:
            cnt += 1
            rc = re.sub(r' ', '', value)
            fc = re.sub(r' ', '', fetcher_text[key])
            l_len = len(rc)
            r_len = len(fc)
            retval = dif_content(rc, fc)
            retval_ground = 0
            results = dmp.diff_main(rc, fc)
            for res in results:
                if res[0] == 0:
                    retval_ground += len(res[1])
            print cnt, ': ', l_len, r_len, retval, retval_ground
            real_ret = max(retval, retval_ground)
            rets.append((cnt, l_len, r_len, real_ret))

    with open('diff_result_1', 'w') as f:
        for res in rets:
            print >> f, res[0], ': ', res[1], res[2], res[3]

def _resume(results_dir):
    """Loads a partially completed json results directory."""
    # Pylint can't infer that the json being loaded is a dict
    # pylint: disable=maybe-no-member
    assert os.path.isdir(results_dir), \
        "TestrunResult.resume() requires a directory"

    # Load the metadata
    with open(os.path.join(results_dir, 'metadata.json'), 'r') as f:
        meta = json.load(f)
    assert meta['results_version'] == CURRENT_JSON_VERSION, \
        "Old results version, resume impossible"

    meta['tests'] = collections.OrderedDict()

    # Load all of the test names and added them to the test list
    tests_dir = os.path.join(results_dir, 'tests')
    file_list = sorted(
        (l for l in os.listdir(tests_dir) if l.endswith('.json')),
        key=lambda p: int(os.path.splitext(p)[0]))

    for file_ in file_list:
        with open(os.path.join(tests_dir, file_), 'r') as f:
            try:
                meta['tests'].update(json.load(f))
            except ValueError:
                continue

    return results.TestrunResult.from_dict(meta)

    def read_config(self):
        try:
            logger.info("going to read the sites config file ")
            with open(NewsCrawlerConfig.SITE_FILE, "rbU") as config:
                try:
                    logger.info("going to json load the sites config")
                    json.load(config, object_hook = self.as_newscrawler)
                    self.is_config_read = True
                except Exception as load_e:
                    logger.error(load_e)
                    raise load_e
        except IOError as file_e:
            logger.error(file_e)
            raise file_e

        try:
            logger.info("going to read the crawler configurations")
            with open(NewsCrawlerConfig.CRAWL_FILE, "rbU") as config:
                try:
                    logger.info("going to json load the crawler configs")
                    json.load(config, object_hook = self.as_crawloptions)
                except Exception as load_e:
                    logger.error(load_e)
                    raise load_e
        except IOError as file_e:
            logger.error(file_e)
            raise file_e

    def query(self, object_id=None,type=None,connection_type=None, metadata=False):
      response = {}
      import urllib
      import simplejson
      if type == 'fb':
        url = 'https://graph.facebook.com/%s' % (object_id)
        if connection_type:
            url += '/%s' % (connection_type)

        params = {'access_token': self.access_token}
        if metadata:
            params['metadata'] = 1

        url += '?' + urllib.urlencode(params)
        response = simplejson.load(urllib.urlopen(url))
        if 'error' in response:
            error = response['error']
            raise FacebookSessionError(error['type'], error['message'])
      if type == 'github':
        url = 'https://api.github.com/user'
        params  = {'access_token': self.access_token}
        url += '?' + urllib.urlencode(params)
        print "DAY LA URL GITHUB %s " % url
        response = simplejson.load(urllib.urlopen(url))
      return response

	def __init__(self, authtype = "OAuth", username = None, password = None, oauth_keys = None, headers = None):
		self.authtype = authtype
		self.authenticated = False
		self.username = username
		self.password = password
		self.oauth_keys = oauth_keys
		if self.username is not None and self.password is not None:
			if self.authtype == "OAuth":
				self.request_token_url = 'https://twitter.com/oauth/request_token'
				self.access_token_url = 'https://twitter.com/oauth/access_token'
				self.authorization_url = 'http://twitter.com/oauth/authorize'
				self.signin_url = 'http://twitter.com/oauth/authenticate'
				# Do OAuth type stuff here - how should this be handled? Seems like a framework question...
			elif self.authtype == "Basic":
				self.auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
				self.auth_manager.add_password(None, "http://twitter.com", self.username, self.password)
				self.handler = urllib2.HTTPBasicAuthHandler(self.auth_manager)
				self.opener = urllib2.build_opener(self.handler)
				if headers is not None:
					self.opener.addheaders = [('User-agent', headers)]
				try:
					simplejson.load(self.opener.open("http://twitter.com/account/verify_credentials.json"))
					self.authenticated = True
				except HTTPError, e:
					raise TangoError("Authentication failed with your provided credentials. Try again? (%s failure)" % `e.code`, e.code)

    def __init__(self, resultfile=None):
        self.serialized_keys = ['options',
                                'name',
                                'tests',
                                'uname',
                                'wglinfo',
                                'glxinfo',
                                'lspci',
                                'time_elapsed']
        self.name = None
        self.uname = None
        self.options = None
        self.glxinfo = None
        self.lspci = None
        self.time_elapsed = None
        self.tests = {}

        if resultfile:
            # Attempt to open the json file normally, if it fails then attempt
            # to repair it.
            try:
                raw_dict = json.load(resultfile)
            except ValueError:
                raw_dict = json.load(self.__repairFile(resultfile))

            # Check that only expected keys were unserialized.
            for key in raw_dict:
                if key not in self.serialized_keys:
                    raise Exception('unexpected key in results file: ', str(key))

            self.__dict__.update(raw_dict)

            # Replace each raw dict in self.tests with a TestResult.
            for (path, result) in self.tests.items():
                self.tests[path] = TestResult(result)

def BestMatch(sname, api_code):

    url = "https://opencorporates.com/reconcile/?query="+urllib.quote(sname)
    entities = simplejson.load(urllib.urlopen(url))

    try:
        bm = entities['result'][0]
        url = "https://api.opencorporates.com" + bm['id'] + '?api_token='+ api_code

        match = simplejson.load(urllib.urlopen(url))
        m = match['results']['company']

        if m['controlling_entity'] == None:
            cen = ''
            cec = ''
            ceu = ''
        else:
            cen = m['controlling_entity']['name']
            cec = m['controlling_entity']['jurisdiction_code']
            ceu = m['controlling_entity']['opencorporates_url']
    

        print sname, m['name'], m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url']
        reconciled.writerow([sname, m['name'].encode('utf-8'), m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url']])
        
        return match['results']['company']['name']

    except IndexError:
        reconciled.writerow([sname, 'nomatch'])

        return "nomatch"

def tz(phenny, input):
	"""Displays time in different time zones"""
	with open('nickloc.csv', 'rU') as f:
			z = csv.reader(f)
			nickdict = {}
			for key, val in z:
				nickdict[key] = val
	try:
		locinput = input.group(2)
		locinput1 = locinput.strip().lower()
		if locinput1 in nickdict:
			htmlinput = urllib.quote(nickdict[locinput1])
		else:
			htmlinput = urllib.quote(locinput1)
		url2 = 'http://nominatim.openstreetmap.org/search?q=' + htmlinput + '&format=json'
		jsonResponse = simplejson.load(urllib.urlopen(url2))
		lati = jsonResponse[0]['lat']
		longi = jsonResponse[0]['lon']
		loca = jsonResponse[0]['display_name']
		url3 = 'http://api.geonames.org/timezoneJSON?lat=' + lati + '&lng=' + longi + '&username=jinpark'
		jsonResponse1 = simplejson.load(urllib.urlopen(url3))
		if jsonResponse1['dstOffset'] == 0:
			timezone = ''
		elif jsonResponse1['dstOffset'] > 0:
			timezone = '+' + str(jsonResponse1['dstOffset'])
		else:
			timezone = str(jsonResponse1['dstOffset'])
		phennyout = loca + ": " + str(jsonResponse1['time']) + ' UTC' + timezone
		phenny.say(phennyout)
	except:
		phenny.say('Something went wrong')

def search(query, resultnum=None, lang=None, since_id=None):
    output = []
    
    q = []
    rpp = 100
    q.append(urlencode({'q': query}))
    if not since_id is None:
        q.append(urlencode({'since_id': since_id}))
    if not lang is None:
        q.append(urlencode({'lang': lang}))
    if not resultnum is None:
        rpp = resultnum
    q.append(urlencode({'rpp': rpp}))
    baseurl = 'http://search.twitter.com/search.json'
    
    
    url = baseurl + '?' + '&'.join(q)
    print url
       
    response = urllib2.urlopen(url)
    data = simplejson.load(response)
    output.append(data)
    
    while 'next_page' in data:
        url = baseurl + data['next_page']
        print url
        response = urllib2.urlopen(url)
        data = simplejson.load(response)
        output.append(data)
    
    return output

def apply_array_location_correction(phot_groups_filepath):

    """
	Reads the phot_groups.json files and applies the array
	location correction to all measurements of ALL sources.
	Writes the result to disk in work_dir as
	'phot_groups_arrayloc.json'
	"""

    work_dir = "/".join(phot_groups_filepath.split("/")[:-1])
    meta = json.load(open(work_dir + "/metadata.json"))

    # read in the array location correction values
    root = os.path.abspath("../bcdphot")
    if meta["channel"] is "1":
        fp = os.path.join(root, "cal", "ch1_photcorr_ap_5.fits")
    elif meta["channel"] is "2":
        fp = os.path.join(root, "cal", "ch2_photcorr_ap_5.fits")
    arrloc = pyfits.open(fp)[0].data

    # read in the photometry JSON files
    ch = json.load(open(phot_groups_filepath))

    # apply correction
    for key in ch:
        for obs in ch[key]:
            x, y = obs[4:6]
            obs[6:] = [i * arrloc[x, y] for i in obs[6:]]

            # write to disk
    out_path = work_dir + "/phot_groups_arrayloc.json"
    with open(out_path, "w") as w:
        json.dump(ch, w, indent=4 * " ")
    print("created file: " + out_path)

def get_manifest(tarfile_path):
    """Extract and return manifest in tarball.

    Given a path to a tarfile, which is itself the product of "docker save",
    this discovers the manifest for the collection of images, which provides
    version information.
    """
    manifest = []
    configjson = None
    with contextlib.closing(tarfile.open(tarfile_path)) as archive:
        for member in archive.getmembers():
            # find the "manifest.json" file, which points to metadata json file
            if os.path.basename(member.path) == 'manifest.json':
                initial_manifest = json.load(archive.extractfile(member))
                configjson = initial_manifest[0]['Config']
        for member in archive.getmembers():
            # get manifest from shasum json file, docker ver > 1.10
            if configjson and os.path.basename(member.path) == configjson:
                image_data = json.load(archive.extractfile(member))
                manifest.append(image_data)
            # find the "json" files, which contain all image metadata
            # legacy code for docker ver < 1.10
            elif not configjson and os.path.basename(member.path) == 'json':
                image_data = json.load(archive.extractfile(member))
                manifest.append(image_data)
    return manifest

def getPages():
    """
    For each page of the diary: get the json and save in ./Diaries/diaryID/pageID.json
  """
    path = "Diaries/"
    urlBase = "http://transcripts.sl.nsw.gov.au/api/node/"
    errorsList = []
    for diaryUrl in diariesUrls:
        print diaryUrl
        response = urllib2.urlopen(diaryUrl)
        # print response.info()
        myjson = simplejson.load(response)
        pagesList = myjson["field_transcript_pages"]["und"]
        ## Create folder if it doesn't exist
        directory = path + diaryUrl[42:-5]
        if not os.path.exists(directory):
            os.makedirs(directory)
        ## Get each page in json
        for page in pagesList:
            print page["nid"]
            mypage = urlBase + page["nid"] + ".json"
            try:
                response = urllib2.urlopen(mypage)
                # print response.info()
                myjson = simplejson.load(response)
                file = open(directory + "/" + page["nid"] + ".json", "w+")
                file.write(str(myjson))
                file.close()
            except urllib2.HTTPError, e:
                print "We failed with error code - %s." % e.code
                print mypage
                print "$$$$$$$$$$$$"
                errorsList.append("diary:" + diaryUrl[42:-5] + ": page " + mypage)

def load_data(dump_file):
    client = pymongo.MongoClient("localhost", 27017)
    db = client['courses']
    coursesd = db['coursesd']
    sectionsd = db['sectionsd']

    doc_queue = []
    with open(dump_file) as f:
        for course in json.load(f):
            pairs = [(name, _typify(course.get(name), data_type)) for (name,
                    data_type) in course_schema if name not in special_fields]
            pairs += _special_treatment(course, course_schema)
            doc = dict(pairs)
            coursesd.update({'CourseFull': doc['CourseFull']},
                doc, True)
    print '%d courses in db.' % coursesd.count()
    with open(dump_file) as f:
        for course in json.load(f):
            pairs = [(name, _typify(course[name], data_type)) for (name,
                    data_type) in section_schema if name not in special_fields]
            pairs += _special_treatment(course, section_schema)
            doc = dict(pairs)
            doc_queue.append(doc)
            if len(doc_queue) == 100:
                inserted = sectionsd.insert(doc_queue)
                print '%d documents inserted' % len(inserted)
                doc_queue = []
        if doc_queue:
            inserted = sectionsd.insert(doc_queue)
            print '%d documents inserted' % len(inserted)
            doc_queue = []
    print '%d sections in db.' % sectionsd.count()

    def update(self):
        request = self.request
        id = request.model_id
        fs = self.get_fieldset(suffix="Edit", id=id)

        event = events.BeforeRenderEvent(self.request.model_instance, self.request, fs=fs)
        alsoProvides(event, events.IBeforeEditRenderEvent)
        zope.component.event.objectEventNotify(event)

        if request.format == "json" and request.method == "PUT":
            data = json.load(request.body_file)
        elif request.content_type == "application/json":
            data = json.load(request.body_file)
        else:
            data = request.POST

        with_prefix = True
        if request.format == "json":
            with_prefix = bool(request.params.get("with_prefix"))

        fs = fs.bind(request=request, with_prefix=with_prefix)
        if self.validate(fs):
            fs.sync()
            self.sync(fs, id)
            self.session.flush()
            if request.format in ("html", "xhr"):
                if request.is_xhr or request.format == "xhr":
                    return Response(content_type="text/plain")
                return exc.HTTPFound(location=request.fa_url(request.model_name, _pk(fs.model)))
            else:
                return self.render(fs=fs, status=0)
        if request.format == "html":
            return self.render(fs=fs, id=id)
        else:
            return self.render(fs=fs, status=1)

def showkeys(d,root=''):

    import simplejson
    import urllib

    if d == '-':
        d = simplejson.load(sys.stdin)
    elif isinstance(d,str):
        d = simplejson.load(urllib.urlopen(d))


    for k,v in d.items():
        if isinstance(v,str):
            print "%s[%s]=%r" % (root,k,v)
        if isinstance(v,int):
            print "%s[%s]=%r" % (root,k,v)
        if isinstance(v,float):
            print "%s[%s]=%r" % (root,k,v)

        if isinstance(v,dict):
            showkeys(v,root = root + '['+k+']')

        if isinstance(v,list):
            for i,v in enumerate(v):
                showkeys(v,root = root + '['+k+']' + '['+str(i)+']')

def test_jsonsink():
    import simplejson
    import os

    ds = nio.JSONFileSink()
    yield assert_equal, ds.inputs._outputs, {}
    ds = nio.JSONFileSink(in_dict={'foo': 'var'})
    yield assert_equal, ds.inputs.in_dict, {'foo': 'var'}
    ds = nio.JSONFileSink(infields=['test'])
    yield assert_true, 'test' in ds.inputs.copyable_trait_names()

    curdir = os.getcwd()
    outdir = mkdtemp()
    os.chdir(outdir)
    js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'})
    js.inputs.new_entry = 'someValue'
    setattr(js.inputs, 'contrasts.alt', 'someNestedValue')
    res = js.run()

    with open(res.outputs.out_file, 'r') as f:
        data = simplejson.load(f)
    yield assert_true, data == {"contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"}

    js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'})
    js.inputs.new_entry = 'someValue'
    js.inputs.test = 'testInfields'
    setattr(js.inputs, 'contrasts.alt', 'someNestedValue')
    res = js.run()

    with open(res.outputs.out_file, 'r') as f:
        data = simplejson.load(f)
    yield assert_true, data == {"test": "testInfields", "contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"}

    os.chdir(curdir)
    shutil.rmtree(outdir)

def _resume(results_dir):
    """Loads a partially completed json results directory."""
    # TODO: could probably use TestrunResult.from_dict here
    # Pylint can't infer that the json being loaded is a dict
    # pylint: disable=maybe-no-member
    assert os.path.isdir(results_dir), \
        "TestrunResult.resume() requires a directory"

    # Load the metadata
    with open(os.path.join(results_dir, 'metadata.json'), 'r') as f:
        meta = json.load(f)
    assert meta['results_version'] == CURRENT_JSON_VERSION, \
        "Old results version, resume impossible"

    meta['tests'] = {}

    # Load all of the test names and added them to the test list
    for file_ in os.listdir(os.path.join(results_dir, 'tests')):
        with open(os.path.join(results_dir, 'tests', file_), 'r') as f:
            try:
                meta['tests'].update(json.load(f, object_hook=piglit_decoder))
            except ValueError:
                continue

    return results.TestrunResult.from_dict(meta)