Skip to content

Commit 6e43013

Browse files
committed
fix http header type as expected for string contents
1 parent abb7b03 commit 6e43013

File tree

2 files changed

+6
-8
lines changed

2 files changed

+6
-8
lines changed

src/main/scala/org/archive/webservices/archivespark/functions/Entities.scala

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,14 @@
2424

2525
package org.archive.webservices.archivespark.functions
2626

27-
import java.util.Properties
28-
import edu.stanford.nlp.ling.CoreAnnotations.{NamedEntityTagAnnotation, SentencesAnnotation, TextAnnotation, TokensAnnotation}
29-
import edu.stanford.nlp.ling.CoreLabel
30-
import edu.stanford.nlp.pipeline.{Annotation, CoreDocument, StanfordCoreNLP}
31-
import edu.stanford.nlp.util.CoreMap
27+
import edu.stanford.nlp.pipeline.{CoreDocument, StanfordCoreNLP}
3228
import org.archive.webservices.archivespark.model._
3329
import org.archive.webservices.archivespark.model.dataloads.ByteLoad
3430
import org.archive.webservices.archivespark.model.pointers.DependentFieldPointer
3531
import org.archive.webservices.sparkling.ars.WANE
3632

33+
import java.util.Properties
3734
import scala.collection.JavaConverters._
38-
import scala.collection.mutable
3935

4036
object EntitiesNamespace {
4137
def get: DependentFieldPointer[ByteLoad.Root, String] = HtmlText.mapIdentity("entities").get[String]("entities")
@@ -66,7 +62,9 @@ class Entities private (properties: Properties, tagFieldMapping: Seq[(String, St
6662
case _: Exception => Map.empty[String, Set[String]]
6763
}
6864

69-
for ((tag, _) <- tagFieldMapping) derivatives.setNext(MultiValueEnrichable(mentions.getOrElse(tag, Set.empty).toSeq))
65+
for ((tag, _) <- tagFieldMapping) {
66+
derivatives.setNext(MultiValueEnrichable(mentions.getOrElse(tag, Set.empty).toSeq))
67+
}
7068
}
7169
}
7270

src/main/scala/org/archive/webservices/archivespark/specific/warc/functions/WarcPayload.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ class WarcPayload private (http: Boolean = true)
4949
if (http) {
5050
for (msg <- record.http) {
5151
derivatives << msg.statusLine
52-
derivatives << msg.headers.toMap
52+
derivatives << msg.headers
5353
derivatives << IOUtil.bytes(msg.payload)
5454
}
5555
} else {

0 commit comments

Comments
 (0)