import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
public static void exportToNif(A2WDataset ds, String baseUri, WikipediaInterface wikiApi, OutputStream outputStream) {
	List<org.aksw.gerbil.transfer.nif.Document> documents = new Vector<>();

	for (int i = 0; i < ds.getSize(); i++) {
		String text = ds.getTextInstanceList().get(i);
		org.aksw.gerbil.transfer.nif.Document d = new DocumentImpl(text, baseUri + "/doc" + i);

		for (it.unipi.di.acube.batframework.data.Annotation a : ds.getA2WGoldStandardList().get(i)) {
			String title;
			try {
				title = wikiApi.getTitlebyId(a.getConcept());
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
			d.addMarking(new NamedEntity(a.getPosition(), a.getLength(), getDBPediaURI(title)));
		}
		documents.add(d);
	}
	NIFWriter writer = new TurtleNIFWriter();
	writer.writeNIF(documents, outputStream);
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
private boolean annotateDocument(final Document document, final Resource resource) {
	final String label = getEnglishLabel(resource.getURI());
	if (label == null) {
		LOGGER.info("Couldn't find an English label for " + resource.toString() + ". Returning null.");
		return false;
	}
	final String text = document.getText();
	final int pos = text.indexOf(label);
	if (pos < 0) {
		LOGGER.info("Couldn't find the label \"{}\" inside the given text \"{}\". Returning false.", label, text);
		// return false;
	} else {
		document.addMarking(new NamedEntity(pos, label.length(), resource.getURI()));
	}
	return true;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
public Document performTypeExtraction(Document document) {
    // extract the surface forms
    // List<List<NamedEntity>> types =
    // surfaceFormExtractor.extractTypeSurfaceForms(document);
    // TypingInfoImpl typingInfo = new TypingInfoImpl();
    // // TODO find a type for every type surface form that has been found
    // List<NamedEntity> typesFound = linkTypes(types, typingInfo);
    // return new TypeExtractionResult(typesFound, typingInfo);
    Map<NamedEntity, List<ExtendedTypedNamedEntity>> extractedTypes = surfaceFormExtractor
            .extractTypeSurfaceForms(document);
    Document resultDoc = new DocumentImpl(document.getText(), document.getDocumentURI());
    List<ExtendedTypedNamedEntity> surfaceForms;
    for (NamedEntity ne : extractedTypes.keySet()) {
        surfaceForms = extractedTypes.get(ne);
        resultDoc.addMarking(typeSearcher.getAllTypes(document, ne, surfaceForms));
        resultDoc.getMarkings().addAll(surfaceForms);
    }
    return resultDoc;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Override
public void endElement(String uri, String localName, String qName)
		throws SAXException {
	if (qName.equalsIgnoreCase(SENTENCE_ELEMENT)) {
		i++;
		documents.add(new DocumentImpl(sentence.toString(),
				"http://senseval" + i, markings));
		sentence = new StringBuilder();
	} else if (qName.equalsIgnoreCase(INSTANCE_ELEMENT)) {
		markings.add(new NamedEntity(start, length, instanceUri));
		start = sentence.length();
	} else if (qName.equalsIgnoreCase(WF_ELEMENT)) {
		start = sentence.length();

	}
	this.field = 0;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
private List<MeaningSpan> parseJsonStream(InputStream in) throws IOException, ParseException {
    List<MeaningSpan> annotations = new ArrayList<>();

    JSONArray namedEntities = (JSONArray) this.jsonParser.parse(new InputStreamReader(in, "UTF-8"));
    JSONObject namedEntity;
    String url;
    long start, length;
    for (Object obj : namedEntities) {
        namedEntity = (JSONObject) obj;

        start = (long) namedEntity.get("start");
        length = (long) namedEntity.get("offset");

        url = (String) namedEntity.get("disambiguatedURL");
        if (url == null) {
            annotations.add(new NamedEntity((int) start, (int) length, new HashSet<String>()));
        } else {
            annotations.add(new NamedEntity((int) start, (int) length, URLDecoder.decode(url, "UTF-8")));
        }
    }

    return annotations;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Test
public void test() throws GerbilException, IOException {
	SensevalDataset data = new SensevalDataset(this.file);
	data.init();
	List<Document> documents = data.getInstances();
	Document doc = documents.get(docIndex);
	assertEquals(expectedSentence, doc.getText());
	List<Marking> markings = doc.getMarkings();
	String[] marks = new String[markings.size()];
	for(int i=0; i<markings.size();i++){
		NamedEntity entity = ((NamedEntity)markings.get(i));
		marks[i]=doc.getText().substring(entity.getStartPosition(), 
				entity.getStartPosition()+entity.getLength());
	}
	assertArrayEquals(expectedMarkings, 
			marks);
	data.close();

}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Test
public void test() {
    ConfidenceScoreEvaluatorDecorator<NamedEntity> decorator = new ConfidenceScoreEvaluatorDecorator<NamedEntity>(
            this, EVALUTION_RESULT_NAME, this);
    EvaluationResultContainer results = new EvaluationResultContainer();
    decorator.evaluate(annotatorResults, new ArrayList<List<NamedEntity>>(), results);
    boolean evalationResultFound = false;
    boolean scoreThresholdFound = false;
    for (EvaluationResult result : results.getResults()) {
        if (EVALUTION_RESULT_NAME.equals(result.getName())) {
            evalationResultFound = true;
            Assert.assertEquals(expectedScore, result.getValue());
        }
        if (ConfidenceScoreEvaluatorDecorator.CONFIDENCE_SCORE_THRESHOLD_RESULT_NAME.equals(result.getName())) {
            scoreThresholdFound = true;
            Assert.assertEquals(expectedThreshold, result.getValue());
        }
    }
    Assert.assertTrue(evalationResultFound);
    if (expectedThreshold != null) {
        Assert.assertTrue(scoreThresholdFound);
    }
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Override
public void evaluate(List<List<NamedEntity>> annotatorResults, List<List<NamedEntity>> goldStandard,
        EvaluationResultContainer results) {
    // all gold standards in this test are empty
    Assert.assertEquals(0, goldStandard.size());
    // simply count all correct named entities
    int score = 0, sum = 0;
    for (List<NamedEntity> nes : annotatorResults) {
        for (NamedEntity ne : nes) {
            if (ne.containsUri(CORRECT_MARKING)) {
                ++score;
            }
            ++sum;
        }
    }
    if (sum == 0) {
        results.addResult(new DoubleEvaluationResult(EVALUTION_RESULT_NAME, 0));
    } else {
        results.addResult(new DoubleEvaluationResult(EVALUTION_RESULT_NAME, (double) score / (double) sum));
    }
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Override
public int compare(final NamedEntity n1, final NamedEntity n2) {
	final int diff = n1.getLength() - n2.getLength();
	if (diff < 0) {
		// n1 is shorter
		return 1;
	} else if (diff > 0) {
		return -1;
	} else {
		return 0;
	}
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Override
public int compare(NamedEntity n1, NamedEntity n2) {
	int diff = n1.getLength() - n2.getLength();
	if (diff < 0) {
		// n1 is shorter
		return 1;
	} else if (diff > 0) {
		return -1;
	} else {
		return 0;
	}
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
/**
 * Sorts Named entities descending by their length.
 */
@Override
public int compare(NamedEntity n1, NamedEntity n2) {
	int diff = n1.getLength() - n2.getLength();
	if (diff < 0) {
		// n1 is shorter
		return 1;
	} else if (diff > 0) {
		return -1;
	} else {
		return 0;
	}
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Test
public void test() {
	ParaphraserImpl paraphraser = new ParaphraserImpl(this);
	Document newDocument = paraphraser.getParaphrase(document);

	Assert.assertEquals(expectedDocument.getDocumentURI(), newDocument.getDocumentURI());
	Assert.assertEquals(expectedDocument.getText(), newDocument.getText());
	StartPosBasedComparator comparator = new StartPosBasedComparator();
	List<NamedEntity> expectedNes = expectedDocument.getMarkings(NamedEntity.class);
	List<NamedEntity> nes = newDocument.getMarkings(NamedEntity.class);
	Collections.sort(expectedNes, comparator);
	Collections.sort(nes, comparator);
	Assert.assertArrayEquals(nes.toArray(), expectedNes.toArray());
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Override
public TypedNamedEntity getAllTypes(Document document, NamedEntity ne, List<ExtendedTypedNamedEntity> surfaceForms) {
    TypedNamedEntity tne = new TypedNamedEntity(ne.getStartPosition(), ne.getLength(), ne.getUris(),
            new HashSet<String>());
    for(ExtendedTypedNamedEntity surfaceForm : surfaceForms) {
        tne.getTypes().addAll(surfaceForm.getUris());
    }

    try {
        // request FOX
        Response response = Request
                .Post(FOX_SERVICE)
                .addHeader("Content-type", "application/json")
                .addHeader("Accept-Charset", "UTF-8")
                .body(new StringEntity(new JSONObject().put("input", document.getText()).put("type", "text")
                        .put("task", "ner").put("output", "JSON-LD").toString(), ContentType.APPLICATION_JSON))
                .execute();

        HttpResponse httpResponse = response.returnResponse();
        HttpEntity entry = httpResponse.getEntity();

        String content = IOUtils.toString(entry.getContent(), "UTF-8");
        EntityUtils.consume(entry);

        // parse results
        JSONObject outObj = new JSONObject(content);
        if (outObj.has("@graph")) {

            JSONArray graph = outObj.getJSONArray("@graph");
            for (int i = 0; i < graph.length(); i++) {
                parseType(graph.getJSONObject(i), tne, surfaceForms);
            }
        } else {
            parseType(outObj, tne, surfaceForms);
        }
    } catch (Exception e) {
        LOGGER.error("Got an exception while communicating with the FOX web service.", e);
    }
    return tne;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
/**
 * 
 * 
 * @param document
 * @return
 */
public synchronized Map<NamedEntity, List<ExtendedTypedNamedEntity>> extractTypeSurfaceForms(Document nifDocument) {
    // Document tmDocument = createDocument(nifDocument);
    // tmDocument = preprocessor.processDocument(tmDocument);
    // return tmDocument.getProperty(TypeSurfaceForms.class).get();
    Annotation document = new Annotation(nifDocument.getText());
    preprocessDocument(document);
    return getTypeSurfaceForms(document, nifDocument.getMarkings(NamedEntity.class));
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
private Map<NamedEntity, List<ExtendedTypedNamedEntity>> getTypeSurfaceForms(Annotation document,
        List<NamedEntity> list) {
    Map<NamedEntity, List<ExtendedTypedNamedEntity>> results = new HashMap<NamedEntity, List<ExtendedTypedNamedEntity>>();
    for (NamedEntity ne : list) {
        results.put(ne, getTypeSurfaceForms(document, ne));
    }
    return results;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Test
public void test() {
    Map<NamedEntity, List<ExtendedTypedNamedEntity>> result = EXTRACTOR.extractTypeSurfaceForms(nifDocument);
    List<ExtendedTypedNamedEntity> resultTypes;
    for (NamedEntity ne : nifDocument.getMarkings(NamedEntity.class)) {
        Assert.assertTrue(result.containsKey(ne));
        resultTypes = result.get(ne);
        for (ExtendedTypedNamedEntity types : resultTypes) {
            Assert.assertTrue("Couldn't find " + types + " inside the list of expected types "
                    + expectedTypes.toString() + ".", expectedTypes.contains(types));
        }
        Assert.assertEquals(expectedTypes.size(), resultTypes.size());
    }
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
@Test
    public void test() {
        Map<NamedEntity, List<ExtendedTypedNamedEntity>> result = EXTRACTOR.extractTypeSurfaceForms(nifDocument);
        List<ExtendedTypedNamedEntity> resultTypes;
        ExtendedTypedNamedEntity matchingEntity;
        int expectedStartPos, expectedEndPos, startPos, endPos;
        for (NamedEntity ne : nifDocument.getMarkings(NamedEntity.class)) {
            Assert.assertTrue(result.containsKey(ne));
            resultTypes = result.get(ne);
            for (TypedNamedEntity expectedType : expectedTypes) {
                matchingEntity = null;
                expectedStartPos = expectedType.getStartPosition();
                expectedEndPos = expectedStartPos + expectedType.getLength();
                // search for a matching entity
                for (ExtendedTypedNamedEntity type : resultTypes) {
                    startPos = type.getStartPosition();
                    endPos = startPos + type.getLength();
                    if ((expectedStartPos >= startPos) && (expectedEndPos <= endPos)) {
                        matchingEntity = type;
                    }
                }
                Assert.assertNotNull("Couldn't find " + expectedType + " inside the list of expected types.",
                        matchingEntity);
            }
//            if (resultTypes.size() > 0) {
//                LOGGER.warn("Got more types than expected: " + Arrays.toString(resultTypes.toArray()));
//            }
        }
    }
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
public static Document reduceToTextAndEntities(Document document) {
    MarkingFilter<TypedNamedEntity> filter = new TypeBasedMarkingFilter<TypedNamedEntity>(false,
            RDFS.Class.getURI(), OWL.Class.getURI());
    List<TypedNamedEntity> namedEntities = document.getMarkings(TypedNamedEntity.class);
    List<Marking> markings = new ArrayList<Marking>(namedEntities.size());
    for (TypedNamedEntity tne : namedEntities) {
        if (filter.isMarkingGood(tne)) {
            markings.add(new NamedEntity(tne.getStartPosition(), tne.getLength(), tne.getUris()));
        }
    }
    return new DocumentImpl(document.getText(), document.getDocumentURI(), markings);
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
public ExtendedYagoBasedTypeSearcherTest(Document nifDocument, NamedEntity namedEntity,
        List<ExtendedTypedNamedEntity> surfaceForms, Set<String> expectedTypes) {
    this.nifDocument = nifDocument;
    this.namedEntity = namedEntity;
    this.surfaceForms = surfaceForms;
    this.expectedTypes = expectedTypes;
}
 

import org.aksw.gerbil.transfer.nif.data.NamedEntity; //导入依赖的package包/类
public static NamedEntity extractNamedEntity(Document document) {
    MarkingFilter<TypedNamedEntity> filter = new TypeBasedMarkingFilter<TypedNamedEntity>(false,
            RDFS.Class.getURI(), OWL.Class.getURI());
    List<TypedNamedEntity> namedEntities = document.getMarkings(TypedNamedEntity.class);
    for (TypedNamedEntity tne : namedEntities) {
        if (filter.isMarkingGood(tne)) {
            return new NamedEntity(tne.getStartPosition(), tne.getLength(), tne.getUris());
        }
    }
    return null;
}