def _httpConfigAndServices(self, apiVersion, arguments, serviceIdentifier=None, prettyPrint=False, **ignored):
     result = {}
     additionalConfigDict = result
     fullServiceInfo = arguments.get('allServiceInfo', ['False'])[0] == 'True'
     useVpn = arguments.get('useVpn', ['False'])[0] == 'True'
     retrieveAll = arguments.get('__all__', ['False'])[0] == 'True'
     keys = self._allKeys() if retrieveAll else self._keysFromArgs(arguments)
     for key in _requestedKeys(keys):
         try:
             if key == 'services':
                 additionalConfigDict[key] = self.call.listServices(activeOnly=not fullServiceInfo, includeState=fullServiceInfo, convertIpsToVpn=useVpn)
             elif key == 'config':
                 additionalConfigDict[key] = self.call.getConfig()
             else:
                 additionalConfigDict[key] = self.call[key].getConfiguration(allConfiguration=retrieveAll)
         except NoneOfTheObserversRespond:
             result.setdefault('errors', []).append("Key '%s' not found." % key)
     if serviceIdentifier:
         this_service = self.call.getService(identifier=serviceIdentifier)
         if this_service is not None:
             result['this_service'] = this_service
             result['this_service']['state'] = self.call.getPrivateStateFor(identifier=serviceIdentifier)
     result = JsonDict(api_version=apiVersion, domain=self.call.getDomain(), **result)
     if self._softwareVersion is not None:
         result['software_version'] = self._softwareVersion
     yield okJson
     yield result.pretty_print() if prettyPrint else str(result)

 def testLoadEmptyFile(self):
     tempfile = join(self.tempdir, 'json.json')
     open(tempfile, 'w').close()
     self.assertRaises(JSONDecodeError, lambda: JsonDict.load(tempfile))
     self.assertEquals({}, JsonDict.load(tempfile, emptyOnError=True))
     self.assertRaises(JSONDecodeError, lambda: JsonList.load(tempfile))
     self.assertEquals([], JsonList.load(tempfile, emptyOnError=True))

 def _configure(self):
     configuration = JsonDict({
         "path": {
             "data": ensureDir(self.stateDir, 'data'),
             "logs": ensureDir(self.stateDir, 'logs'),
             "work": ensureDir(self.stateDir, 'work'), # temporary files
             "conf": self.configDir,
             "plugins": ensureDir(self.stateDir, 'plugins'),
         },
         "cluster":{
             "name": self.name,
         },
         "http":{
             "port": self.port,
         },
         "transport": {
             "tcp": {
                 "port": self.transportPort
             }
         }
     })
     self._configureIndex(configuration)
     if self.identifier:
         configuration.setdefault("node", dict())['name'] = self.identifier
     with open(self.configFile, 'w') as f:
         configuration.dump(f, indent=4, sort_keys=True)

def dna(reactor, port, dataPath, logPath, statePath, harvesterStatusUrl, **ignored):
    harvesterData = HarvesterData(dataPath)
    repositoryStatus = be((RepositoryStatus(logPath, statePath),
            (harvesterData,)
        ))
    configDict = JsonDict(
            logPath=logPath,
            statePath=statePath,
            harvesterStatusUrl=harvesterStatusUrl,
            dataPath=dataPath,
        )

    return \
        (Observable(),
            (ObservableHttpServer(reactor, port),
                (ApacheLogger(stdout),
                    (PathFilter("/info/version"),
                        (StringServer(VERSION_STRING, ContentTypePlainText), )
                    ),
                    (PathFilter("/info/config"),
                        (StringServer(configDict.dumps(), ContentTypeJson), )
                    ),
                    (PathFilter("/static"),
                        (PathRename(lambda name: name[len('/static/'):]),
                            (FileServer(seecrWebLibPath),)
                        )
                    ),
                    (PathFilter('/', excluding=['/info/version', '/info/config', '/static', '/action', '/get']),
                        (DynamicHtml(
                                [dynamicHtmlPath],
                                reactor=reactor,
                                additionalGlobals = {
                                    'time': time,
                                    'harvesterStatusUrl': harvesterStatusUrl,
                                    'escapeXml': escapeXml,
                                    'compose': compose,
                                },
                                indexPage="/index.html",
                            ),
                            (harvesterData,),
                            (repositoryStatus,),
                        )
                    ),
                    (PathFilter('/action'),
                        (HarvesterDataActions(),
                            (harvesterData,)
                        ),
                    ),
                    (PathFilter('/get'),
                        (HarvesterDataRetrieve(),
                            (harvesterData,),
                            (repositoryStatus,),
                        )
                    )
                )
            )
        )

 def asPostDict(self):
     drilldownFields = []
     for fieldname, options in self.fieldRegistry.drilldownFieldNames.items():
         drilldownFields.append({
             "dim": fieldname,
             "hierarchical": options["hierarchical"],
             "multiValued": options["multiValued"],
             "fieldname": options["indexFieldName"]
         })
     result = JsonDict(drilldownFields=drilldownFields)
     result.update((k[1:], v) for k, v in self.__dict__.iteritems() if k[1:] in SETTING_NAMES)
     return result

 def validate(self):
     for core in self.cores:
         if core == self.resultsFrom:
             continue
         try:
             self._matchCoreSpecs(self.resultsFrom, core)
         except KeyError:
             raise ValueError("No match set for cores %s" % str((self.resultsFrom, core)))
     if self.relationalFilterJson:
         try:
             JsonDict.loads(self.relationalFilterJson)
         except JSONDecodeError:
             raise ValueError("Value '%s' for 'relationalFilterJson' can not be parsed as JSON." % self.relationalFilterJson)

 def testGetStatusForDomain(self):
     self.controlHelper(action='allInvalid')
     self.startHarvester(repository=REPOSITORY)
     header, result = getRequest(self.harvesterInternalServerPortNumber, '/get', {'verb': 'GetStatus', 'domainId': 'adomain'}, parse=False)
     data = JsonDict.loads(result)
     self.assertEquals(2, len(data['response']['GetStatus']))
     self.assertEquals("adomain", data['request']['domainId'])

 def getRepositoryIds(self, domainId, repositoryGroupId=None):
     result = JsonList()
     allIds = self.getRepositoryGroupIds(domainId) if repositoryGroupId is None else [repositoryGroupId]
     for repositoryGroupId in allIds:
         jsonData = JsonDict.load(open(join(self._dataPath, '%s.%s.repositoryGroup' % (domainId, repositoryGroupId))))
         result.extend(jsonData.get('repositoryIds', []))
     return result

 def _read(self):
     result = JsonDict.load(self._filename)
     assert result['version'] == self.version, 'Expected database version %s' % self.version
     groups = set(self._groups)
     groups.update(set(result['data']['groups']))
     self._groups = list(groups)
     self._users.update(result['data']['users'])

 def urlJsonDict(self, **kwargs):
     arguments = dict((k ,v) for k, v in kwargs.items() if v)
     result = JsonDict.load(
             self._urlopen("{}/get?{}".format(self._internalurl, urlencode(arguments)))
         )
     if 'error' in result:
         raise ValueError(result['error']['message'])
     return result

    def testClear(self):
        self.startHarvester(repository=REPOSITORY)
        self.assertEquals(BATCHSIZE, self.sizeDumpDir())

        header, result = getRequest(self.harvesterInternalServerPortNumber, '/get', {'verb': 'GetStatus', 'domainId': DOMAIN, 'repositoryId': REPOSITORY}, parse=False)
        data = JsonDict.loads(result)
        self.assertEquals(8, data['response']['GetStatus'][0]['total'])

        self.saveRepository(DOMAIN, REPOSITORY, REPOSITORYGROUP, action='clear')

        self.startHarvester(repository=REPOSITORY)
        self.assertEquals(18, self.sizeDumpDir())
        for filename in sorted(listdir(self.dumpDir))[-8:]:
            self.assertTrue('_delete.updateRequest' in filename, filename)

        header, result = getRequest(self.harvesterInternalServerPortNumber, '/get', {'verb': 'GetStatus', 'domainId': DOMAIN, 'repositoryId': REPOSITORY}, parse=False)
        self.assertEqual(0, JsonDict.loads(result)['response']['GetStatus'][0]['total'])

 def testKeysAll(self):
     result = asString(self.dna.all.handleRequest(
         path='/service/v2/list',
         Method='GET',
         arguments={'__all__':['True']}
     ))
     header, body = httpSplit(result)
     dictBodyV2 = JsonDict.loads(body)
     self.assertEquals(['api_version', 'collections', 'config', 'domain', 'other', 'services', 'software_version'], sorted(dictBodyV2.keys()))

 def getRepositories(self, domainId, repositoryGroupId=None):
     try:
         repositoryIds = self.getRepositoryIds(domainId=domainId, repositoryGroupId=repositoryGroupId)
     except IOError:
         raise ValueError("idDoesNotExist")
     return JsonList([
             JsonDict.load(open(join(self._dataPath, '%s.%s.repository' % (domainId, repositoryId))))
             for repositoryId in repositoryIds
         ])

 def testRemovingNotListedKeys(self):
     result = asString(self.dna.all.handleRequest(
         path='/service/v2/list',
         Method='GET',
         arguments={'keys':['-no']}
     ))
     header, body = httpSplit(result)
     dictBodyV2 = JsonDict.loads(body)
     self.assertEquals(['api_version', 'config', 'domain', 'services', 'software_version'], sorted(dictBodyV2.keys()))

 def load(cls, filePath):
     state = cls(filePath=filePath)
     if isfile(filePath):
         d = JsonDict.load(filePath)
         state.datetime = d.get('datetime')
         state.harvestingReady = d.get('harvestingReady', False)
         state.error = d.get('error')
         state.resumptionAttributes = d.get('resumptionAttributes')
     return state

 def testGetStatusForDomainAndRepositoryId(self):
     self.controlHelper(action='allInvalid')
     self.startHarvester(repository=REPOSITORY)
     header, result = getRequest(self.harvesterInternalServerPortNumber, '/get', {'verb': 'GetStatus', 'domainId': 'adomain', 'repositoryId': 'integrationtest'}, parse=False)
     data = JsonDict.loads(result)
     self.assertEquals("GetStatus", data['request']['verb'])
     self.assertEquals("adomain", data['request']['domainId'])
     self.assertEquals("integrationtest", data['request']['repositoryId'])
     self.assertEquals("IntegrationTest", data['response']['GetStatus'][0]['repositoryGroupId'])
     self.assertEquals(6, data['response']['GetStatus'][0]['invalid'])

 def testShouldHaveGlobalConfigForVersion2(self):
     result = asString(self.dna.all.handleRequest(
         path='/service/v2/list',
         arguments={},
         Method='GET',
     ))
     header, body = httpSplit(result)
     dictBodyV2 = JsonDict.loads(body)
     self.assertEquals(['host', 'port'], sorted(dictBodyV2['config'].keys()))
     self.assertEquals(['api_version', 'config', 'domain', 'services', 'software_version'], sorted(dictBodyV2.keys()))

 def testNonexistingKeys(self):
     result = asString(self.dna.all.handleRequest(
         path='/service/v2/list',
         Method='GET',
         arguments={'keys':['no']}
     ))
     header, body = httpSplit(result)
     dictBodyV2 = JsonDict.loads(body)
     self.assertEquals(['api_version', 'config', 'domain', 'errors', 'services', 'software_version'], sorted(dictBodyV2.keys()))
     self.assertEquals(["Key 'no' not found."], dictBodyV2['errors'])

 def testServiceRegistryOldFormat(self):
     uuid1 = str(uuid4())
     uuid2 = str(uuid4())
     with open(join(self.tempdir, 'serviceregistry.json'), 'w') as f:
         d = JsonDict({
                 uuid1: {
                     "ipAddress": "5.153.228.85",
                     "readable": True,
                     "number": 1,
                     "data": {
                         "uptime": 366867,
                         "VERSION": "1.5.12.3"
                     },
                     "writable": True,
                     "lastseen": 1423494771.904539,
                     "type": "holding",
                     "infoport": 35609,
                 },
                 uuid2: {
                     "ipAddress": "5.153.228.85",
                     "readable": True,
                     "number": 1,
                     "data": {
                         "uptime": 366867,
                         "VERSION": "1.5.12.3"
                     },
                     "writable": True,
                     "lastseen": 1423494771.904539,
                     "type": "plein",
                     "infoport": 41609,
                 }
             })
         d.dump(f)
     registry = ServiceRegistry(
         stateDir=self.tempdir,
         domainname='zp.example.org',
         reactor=CallTrace(),
     )
     self.assertEquals(set([uuid1, uuid2]), set(registry.listServices(activeOnly=False).keys()))

 def testAllKeys(self):
     result = asString(self.dna.all.handleRequest(
         path='/service/v2/list',
         Method='GET',
         arguments={'keys':['collections,other']}
     ))
     header, body = httpSplit(result)
     dictBodyV2 = JsonDict.loads(body)
     self.assertEquals(['api_version', 'collections', 'config', 'domain', 'other', 'services', 'software_version'], sorted(dictBodyV2.keys()))
     self.assertEquals({'collection': {'provenanceSource': 'collection_source', 'enabled': True, 'name': 'collection'}}, dictBodyV2['collections'])
     self.assertEquals(['other'], dictBodyV2['other'])
     self.assertEquals({'host': 'localhost', 'port': 8000}, dictBodyV2['config'])
     self.assertEquals({}, dictBodyV2['services'])