in ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfBothEEAndETRelations.java [360:531]
protected AnnotationStatistics<String> test(CollectionReader collectionReader, File directory)
throws Exception {
this.useClosure=false;
AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();
aggregateBuilder.add( AnalysisEngineFactory.createEngineDescription(
ViewCreatorAnnotator.class,
ViewCreatorAnnotator.PARAM_VIEW_NAME,
PROB_VIEW_NAME ) );
aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class));
aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(
RemoveCrossSentenceRelations.class,
RemoveCrossSentenceRelations.PARAM_SENTENCE_VIEW,
CAS.NAME_DEFAULT_SOFA,
RemoveCrossSentenceRelations.PARAM_RELATION_VIEW,
GOLD_VIEW_NAME));
//keep event event tlinks, remove the other relations
// aggregateBuilder.add(
// AnalysisEngineFactory.createEngineDescription(PreserveEventEventRelations.class),
// CAS.NAME_DEFAULT_SOFA,
// GOLD_VIEW_NAME);
//remove non-tlink relations, such as alinks
aggregateBuilder.add(
AnalysisEngineFactory.createEngineDescription(RemoveNonTLINKRelations.class),
CAS.NAME_DEFAULT_SOFA,
GOLD_VIEW_NAME);
// aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveNonUMLSEvents.class));
if (!recallModeEvaluation && this.useClosure) { //closure for gold
aggregateBuilder.add(
AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
CAS.NAME_DEFAULT_SOFA,
GOLD_VIEW_NAME);
}
aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveNonContainsRelations.class),
CAS.NAME_DEFAULT_SOFA,
GOLD_VIEW_NAME);
aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveRelations.class));
AnalysisEngineDescription aed = this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory) :
EventEventRelationAnnotator.createAnnotatorDescription((new File(directory,"event-event/model.jar")).getAbsolutePath());
if(this.writeProbabilities){
ConfigurationParameterFactory.addConfigurationParameter(aed,
TemporalRelationExtractorAnnotator.PARAM_PROB_VIEW,
PROB_VIEW_NAME);
}
aggregateBuilder.add(aed);
aed = EventTimeSelfRelationAnnotator.createEngineDescription(new File(directory,"event-time/model.jar").getAbsolutePath());
if(this.writeProbabilities){
ConfigurationParameterFactory.addConfigurationParameter(aed,
TemporalRelationExtractorAnnotator.PARAM_PROB_VIEW,
PROB_VIEW_NAME);
}
aggregateBuilder.add(aed);
aed = DocTimeRelAnnotator.createAnnotatorDescription(new File("target/eval/event-properties/train_and_test/docTimeRel/model.jar").getAbsolutePath());
if(this.writeProbabilities){
ConfigurationParameterFactory.addConfigurationParameters(
aed,
DocTimeRelAnnotator.PARAM_PROB_VIEW,
PROB_VIEW_NAME);
}
aggregateBuilder.add(aed);
aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(CrossSentenceTemporalRelationAnnotator.class));
// aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(WithinSentenceBeforeRelationAnnotator.class));
if(this.anaforaOutput != null){
aed = AnalysisEngineFactory.createEngineDescription(WriteAnaforaXML.class, WriteAnaforaXML.PARAM_OUTPUT_DIR, this.anaforaOutput);
if(this.writeProbabilities){
ConfigurationParameterFactory.addConfigurationParameters(
aed,
WriteAnaforaXML.PARAM_PROB_VIEW,
PROB_VIEW_NAME);
}
aggregateBuilder.add(aed, "TimexView", CAS.NAME_DEFAULT_SOFA);
}
File outf = null;
if (recallModeEvaluation && this.useClosure) {//add closure for system output
aggregateBuilder.add(
AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
GOLD_VIEW_NAME,
CAS.NAME_DEFAULT_SOFA
);
outf = new File("target/eval/thyme/SystemError_eventEvent_recall_test.txt");
}else if (!recallModeEvaluation && this.useClosure){
outf = new File("target/eval/thyme/SystemError_eventEvent_precision_test.txt");
}else{
outf = new File("target/eval/thyme/SystemError_eventEvent_plain_test.txt");
}
PrintWriter outDrop =null;
outDrop = new PrintWriter(new BufferedWriter(new FileWriter(outf, false)));
Function<BinaryTextRelation, ?> getSpan = new Function<BinaryTextRelation, HashableArguments>() {
public HashableArguments apply(BinaryTextRelation relation) {
return new HashableArguments(relation);
}
};
Function<BinaryTextRelation, String> getOutcome = AnnotationStatistics.annotationToFeatureValue("category");
AnnotationStatistics<String> stats = new AnnotationStatistics<>();
JCasIterator jcasIter =new JCasIterator(collectionReader, aggregateBuilder.createAggregate());
JCas jCas = null;
while(jcasIter.hasNext()) {
jCas = jcasIter.next();
JCas goldView = jCas.getView(GOLD_VIEW_NAME);
JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA);
Collection<BinaryTextRelation> goldRelations = JCasUtil.select(
goldView,
BinaryTextRelation.class);
Collection<BinaryTextRelation> systemRelations = JCasUtil.select(
systemView,
BinaryTextRelation.class);
//newly add
// systemRelations = removeNonGoldRelations(systemRelations, goldRelations, getSpan);//for removing non-gold pairs
// systemRelations = correctArgOrder(systemRelations, goldRelations);//change the argument order of "OVERLAP" relation, if the order is flipped
//find duplicates in gold relations:
// Collection<BinaryTextRelation> duplicateGoldRelations = getDuplicateRelations(goldRelations, getSpan);
// if(!duplicateGoldRelations.isEmpty()){
// System.err.println("******Duplicate gold relations in : " + ViewURIUtil.getURI(jCas).toString());
// for (BinaryTextRelation rel : duplicateGoldRelations){
// System.err.println("Duplicate : "+ formatRelation(rel));
// }
// }
//end newly add
stats.add(goldRelations, systemRelations, getSpan, getOutcome);
if(this.printRelations){
URI uri = ViewUriUtil.getURI(jCas);
String[] path = uri.getPath().split("/");
printRelationAnnotations(path[path.length - 1], systemRelations);
}
if(this.printErrors){
Map<HashableArguments, BinaryTextRelation> goldMap = Maps.newHashMap();
for (BinaryTextRelation relation : goldRelations) {
goldMap.put(new HashableArguments(relation), relation);
}
Map<HashableArguments, BinaryTextRelation> systemMap = Maps.newHashMap();
for (BinaryTextRelation relation : systemRelations) {
systemMap.put(new HashableArguments(relation), relation);
}
Set<HashableArguments> all = Sets.union(goldMap.keySet(), systemMap.keySet());
List<HashableArguments> sorted = Lists.newArrayList(all);
Collections.sort(sorted);
outDrop.println("Doc id: " + ViewUriUtil.getURI(jCas).toString());
for (HashableArguments key : sorted) {
BinaryTextRelation goldRelation = goldMap.get(key);
BinaryTextRelation systemRelation = systemMap.get(key);
if (goldRelation == null) {
outDrop.println("System added: " + formatRelation(systemRelation));
} else if (systemRelation == null) {
outDrop.println("System dropped: " + formatRelation(goldRelation));
} else if (!systemRelation.getCategory().equals(goldRelation.getCategory())) {
String label = systemRelation.getCategory();
outDrop.printf("System labeled %s for %s\n", label, formatRelation(goldRelation));
} else{
outDrop.println("Nailed it! " + formatRelation(systemRelation));
}
}
}
}
outDrop.close();
return stats;
}