set hierarchy in getters, leads to big gains in performance, raise version to 2.7

Author: helgeho

build.sbt

@@ -4,7 +4,7 @@ import sbt.Keys._
 lazy val commonSettings = Seq(
   name := "archivespark",
   organization := "com.github.helgeho",
-  version := "2.6.2",
+  version := "2.7",
   scalaVersion := "2.11.7",
   fork := true
 )

src/main/scala/de/l3s/archivespark/enrich/EnrichRoot.scala

@@ -29,16 +29,10 @@ import de.l3s.archivespark.utils.Json._
 trait TypedEnrichRoot[+Meta] extends EnrichRoot with TypedEnrichable[Meta]
 
 trait EnrichRoot extends Enrichable { this: TypedEnrichRoot[_] =>
-  setHierarchy(null, null, this)
-
-  override def copy() = {
-    val copy = super.copy().asInstanceOf[EnrichRoot]
-    copy.setHierarchy(null, null, copy)
-    copy
-  }
-
   def metaKey: String = "record"
 
+  override def root[A]: TypedEnrichRoot[A] = this.asInstanceOf[TypedEnrichRoot[A]]
+
   def toJson: Map[String, Any] = Map(
     metaKey -> json(this.get)
   ) ++ enrichments.map{e => (e, mapToJsonValue(enrichment(e).get.toJson))}.filter{ case (_, field) => field != null }

src/main/scala/de/l3s/archivespark/enrich/Enrichable.scala

@@ -37,18 +37,6 @@ trait Enrichable extends Serializable with Copyable[Enrichable] with JsonConvert
   def get: Any
   def typed[T]: TypedEnrichable[T] = this.asInstanceOf[TypedEnrichable[T]]
 
-  override def copy(): Enrichable = copy(Map.empty)
-
-  protected[enrich] def copy(cloned: Map[String, Enrichable]): Enrichable = {
-    val copy = super.copy()
-    copy._enrichments = (_enrichments.keySet ++ cloned.keySet).map{field =>
-      val enrichable = cloned.getOrElse(field, _enrichments(field).copy())
-      enrichable.setHierarchy(copy, field, _root)
-      (field, enrichable)
-    }.toMap
-    copy
-  }
-
   private var excludeFromOutput: Option[Boolean] = None
   def isExcludedFromOutput: Boolean = excludeFromOutput match {
     case Some(value) => value
@@ -63,19 +51,19 @@ trait Enrichable extends Serializable with Copyable[Enrichable] with JsonConvert
     case None => excludeFromOutput = Some(value)
   }
 
-  private var _field: String = _
+  @transient private var _field: String = _
   def field: String = _field
 
-  private var _parent: Enrichable = _
+  @transient private var _parent: Enrichable = _
   def parent[A]: TypedEnrichable[A] = _parent.asInstanceOf[TypedEnrichable[A]]
 
-  private var _root: EnrichRoot = _
+  @transient private var _root: EnrichRoot = _
   def root[A]: TypedEnrichRoot[A] = _root.asInstanceOf[TypedEnrichRoot[A]]
 
   def path: Seq[String] = if (_parent == null) Seq.empty else _parent.path :+ _field
   def chain: Seq[Enrichable] = if (_parent == null) Seq(this) else _parent.chain :+ this
 
-  protected[enrich] def setHierarchy(parent: Enrichable, field: String, root: EnrichRoot = null): Unit = {
+  protected[enrich] def setHierarchy(parent: Enrichable, field: String, root: EnrichRoot): Unit = {
     _field = field
     _parent = parent
     _root = root
@@ -92,21 +80,29 @@ trait Enrichable extends Serializable with Copyable[Enrichable] with JsonConvert
     clone
   }
 
-  def enrichment[D : ClassTag](key: String): Option[TypedEnrichable[D]] = _enrichments.get(field(key)).map(_.asInstanceOf[TypedEnrichable[D]])
+  def enrichment[D : ClassTag](key: String): Option[TypedEnrichable[D]] = {
+    val fieldname = field(key)
+    _enrichments.get(fieldname) match {
+      case Some(enrichable) =>
+        enrichable.setHierarchy(this, fieldname, root)
+        Some(enrichable.asInstanceOf[TypedEnrichable[D]])
+      case None => None
+    }
+  }
 
   def enrich(fieldName: String, enrichment: Enrichable): Enrichable = {
-    val clone = copy(Map(fieldName -> enrichment))
+    val clone = copy()
+    clone._enrichments = clone._enrichments.updated(fieldName, enrichment)
     clone._lastException = enrichment._lastException
     clone._aliases -= fieldName
     clone
   }
 
   def enrichValue[Value](fieldName: String, value: Value): Enrichable = {
-    val enrichable = SingleValueEnrichable[Value](value, this, fieldName, _root)
-    enrich(fieldName, enrichable)
+    enrich(fieldName, SingleValueEnrichable[Value](value))
   }
 
-  private[enrich] def enrich[D](func: EnrichFunc[_, D], excludeFromOutput: Boolean = false): Enrichable = {
+  private def enrich[D](func: EnrichFunc[_, D], excludeFromOutput: Boolean = false): Enrichable = {
     if (!func.exists(this)) {
       val derivatives = new Derivatives(func.fields, func.aliases)
       var lastException: Option[Exception] = None
@@ -118,20 +114,28 @@ trait Enrichable extends Serializable with Copyable[Enrichable] with JsonConvert
 //          if (ArchiveSpark.conf.catchExceptions) lastException = Some(exception)
 //          else throw exception
       }
-      val clone = copy(derivatives.get)
+      val clone = copy()
       clone._lastException = lastException
       clone._aliases ++= derivatives.aliases
       for ((field, enrichment) <- derivatives.get) {
         enrichment.excludeFromOutput(excludeFromOutput, overwrite = false)
+        clone._enrichments = clone._enrichments.updated(field, enrichment)
         clone._aliases -= field
       }
       clone
-    } else if (!excludeFromOutput && func.fields.exists(f => enrichment(f).get.isExcludedFromOutput)) {
-      val clone = copy()
-      for (field <- func.fields if enrichment(field).get.isExcludedFromOutput) {
-        clone.enrichment(field).get.excludeFromOutput(value = false)
+    } else if (!excludeFromOutput) {
+      val excluded = func.fields.map(enrichment).filter(_.isDefined).map(_.get).filter(_.isExcludedFromOutput)
+      if (excluded.nonEmpty) {
+        val clone = copy()
+        for (enrichment <- excluded) {
+          val enrichmentClone = enrichment.copy()
+          enrichmentClone.excludeFromOutput(value = false)
+          clone._enrichments = clone._enrichments.updated(enrichment.field, enrichmentClone)
+        }
+        clone
+      } else {
+        this
       }
-      clone
     } else {
       this
     }
@@ -158,10 +162,11 @@ trait Enrichable extends Serializable with Copyable[Enrichable] with JsonConvert
         val remaining = path.tail
         enrichment(remaining.head) match {
           case Some(child) => child(remaining.tail)
-          case None => for (child <- _enrichments.values) {
-            val target: Option[TypedEnrichable[D]] = child[D](path)
-            if (target.isDefined) return target
-          }
+          case None =>
+            for (child <- _enrichments.values) {
+              val target: Option[TypedEnrichable[D]] = child[D](path)
+              if (target.isDefined) return target
+            }
             None
         }
       } else if (path.head.matches("\\[\\d+\\]")) {

src/main/scala/de/l3s/archivespark/enrich/MultiValueEnrichable.scala

@@ -32,28 +32,9 @@ class MultiValueEnrichable[T] private (private var _children: Seq[TypedEnrichabl
   def children: Seq[TypedEnrichable[T]] = _children
   def get: Seq[T] = children.map(e => e.get)
 
-  override protected[enrich] def copy(cloned: Map[String, Enrichable]): Enrichable = {
-    val copy = super.copy(cloned).asInstanceOf[MultiValueEnrichable[T]]
-    copy._children = _children.zipWithIndex.map{case (c, i) =>
-      val child = c.copy().asInstanceOf[TypedEnrichable[T]]
-      child.setHierarchy(copy, s"[$i]", root)
-      child
-    }
-    copy
-  }
-
-  protected[enrich] def copy(children: Seq[TypedEnrichable[T]]): Enrichable = {
-    val copy = super.copy().asInstanceOf[MultiValueEnrichable[T]]
-    copy._children = children.zipWithIndex.map{case (c, i) =>
-      c.setHierarchy(copy, s"[$i]", root)
-      c
-    }
-    copy
-  }
-
   override protected[enrich] def setHierarchy(parent: Enrichable, field: String, root: EnrichRoot): Unit = {
     super.setHierarchy(parent, field, root)
-    for ((child, i) <- _children.zipWithIndex) child.setHierarchy(this, s"[$i]", root)
+    for ((c, i) <- _children.zipWithIndex) c.setHierarchy(this, s"[$i]", root)
   }
 
   override protected[archivespark] def excludeFromOutput(value: Boolean, overwrite: Boolean): Unit = {
@@ -70,13 +51,12 @@ class MultiValueEnrichable[T] private (private var _children: Seq[TypedEnrichabl
         if (enriched != c) {
           hasEnriched = true
           lastException = enriched._lastException.orElse(lastException)
-          (enriched.asInstanceOf[TypedEnrichable[T]], true)
-        } else {
-          (c, false)
-        }
+          enriched.asInstanceOf[TypedEnrichable[T]]
+        } else c
       }
       if (hasEnriched) {
-        val clone = copy(enriched.map{case (enriched, cloned) => if (cloned) enriched else enriched.copy().asInstanceOf[TypedEnrichable[T]]})
+        val clone = copy().asInstanceOf[MultiValueEnrichable[T]]
+        clone._children = enriched
         clone._lastException = lastException
         clone
       } else this
@@ -86,7 +66,8 @@ class MultiValueEnrichable[T] private (private var _children: Seq[TypedEnrichabl
         val enriched = children(index).enrich(path.tail, func, excludeFromOutput).asInstanceOf[TypedEnrichable[T]]
         if (children(index) == enriched) this
         else {
-          val clone = copy(children.zipWithIndex.map{case (c, i) => if (i == index) enriched else c.copy().asInstanceOf[TypedEnrichable[T]]})
+          val clone = copy().asInstanceOf[MultiValueEnrichable[T]]
+          clone._children = children.zipWithIndex.map{case (c, i) => if (i == index) enriched else c}
           clone._lastException = enriched.lastException
           clone
         }

src/main/scala/de/l3s/archivespark/enrich/SingleValueEnrichable.scala

@@ -26,14 +26,12 @@ package de.l3s.archivespark.enrich
 
 import de.l3s.archivespark.utils.Json._
 
-class SingleValueEnrichable[T] private (override val get: T, parentEnrichable: Enrichable = null, field: String = null, enrichRoot: EnrichRoot = null) extends TypedEnrichable[T] {
-  setHierarchy(parentEnrichable, field, enrichRoot)
-
+class SingleValueEnrichable[T] private (override val get: T) extends TypedEnrichable[T] {
   def toJson: Map[String, Any] = (if (isExcludedFromOutput) Map() else Map(
     null.asInstanceOf[String] -> json(this.get)
   )) ++ enrichments.map{e => (e, mapToJsonValue(enrichment(e).get.toJson))}.filter{ case (_, field) => field != null }
 }
 
 object SingleValueEnrichable {
-  def apply[T](value: T, parent: Enrichable = null, field: String = null, root: EnrichRoot = null): SingleValueEnrichable[T] = new SingleValueEnrichable[T](value, parent, field, root)
+  def apply[T](value: T): SingleValueEnrichable[T] = new SingleValueEnrichable[T](value)
 }

src/main/scala/de/l3s/archivespark/utils/Json.scala

@@ -55,7 +55,7 @@ object Json extends Serializable {
 
   def mapToEnrichable(jsonMap: Map[String, Any], parent: Enrichable, field: String): Enrichable = {
     val json = de.l3s.archivespark.utils.Json.mapToJson(jsonMap)
-    var enrichable: Enrichable = SingleValueEnrichable[String](json, parent, field, if (parent != null) parent.root else null)
+    var enrichable: Enrichable = SingleValueEnrichable[String](json)
     enrichable.excludeFromOutput()
     for ((key, value) <- jsonMap) {
       enrichable = Try{value.asInstanceOf[Map[String, Any]]} match {