diff --git a/README.rst b/README.rst index 0b0267c5ce..ca3704e9e3 100644 --- a/README.rst +++ b/README.rst @@ -18,8 +18,9 @@ Getting Started User Documentation ~~~~~~~~~~~~~~~~~~ -`Read the documentation for Spark 3.4 `__ (or -`3.3 `__ , +`Read the documentation for Spark 3.5 `__ (or +`3.4 `__ , +`3.3 `__ , `3.2 `__ , `3.1 `__, `3.0 `__, @@ -29,7 +30,8 @@ User Documentation Download Binaries ~~~~~~~~~~~~~~~~~ -`Download the latest version for Spark 3.4 `__ (or +`Download the latest version for Spark 3.5 `__ (or +`3.4 `__, `3.3 `__, `3.2 `__, `3.1 `__, @@ -95,20 +97,20 @@ Use Sparkling Water with PySpark ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sparkling Water can be also used directly from PySpark and the integration is called PySparkling. -See `PySparkling README `__ to learn about PySparkling. +See `PySparkling README `__ to learn about PySparkling. Use Sparkling Water via Spark Packages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To see how Sparkling Water can be used as Spark package, please see `Use as Spark Package `__. +To see how Sparkling Water can be used as Spark package, please see `Use as Spark Package `__. Use Sparkling Water in Windows environments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -See `Windows Tutorial `__ to learn how to use Sparkling Water in Windows environments. +See `Windows Tutorial `__ to learn how to use Sparkling Water in Windows environments. Sparkling Water examples ~~~~~~~~~~~~~~~~~~~~~~~~ -To see how to run examples for Sparkling Water, please see `Running Examples `__. +To see how to run examples for Sparkling Water, please see `Running Examples `__. Maven packages ~~~~~~~~~~~~~~ @@ -140,26 +142,26 @@ backend. The backend can be specified before creation of the ``H2OContext``. For more details regarding the internal or external backend, please see -`Backends `__. +`Backends `__. -------------- FAQ --- -List of all Frequently Asked Questions is available at `FAQ `__. +List of all Frequently Asked Questions is available at `FAQ `__. -------------- Development ----------- -Complete development documentation is available at `Development Documentation `__. +Complete development documentation is available at `Development Documentation `__. Build Sparkling Water ~~~~~~~~~~~~~~~~~~~~~ -To see how to build Sparkling Water, please see `Build Sparkling Water `__. +To see how to build Sparkling Water, please see `Build Sparkling Water `__. Develop applications with Sparkling Water ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -187,7 +189,7 @@ We also respond to questions tagged with sparkling-water and h2o tags on the `St Change Logs ~~~~~~~~~~~ -Change logs are available at `Change Logs `__. +Change logs are available at `Change Logs `__. --------------- diff --git a/core/src/test/scala/ai/h2o/sparkling/TestUtils.scala b/core/src/test/scala/ai/h2o/sparkling/TestUtils.scala index feca5a8d02..ec13a37769 100644 --- a/core/src/test/scala/ai/h2o/sparkling/TestUtils.scala +++ b/core/src/test/scala/ai/h2o/sparkling/TestUtils.scala @@ -16,13 +16,11 @@ */ package ai.h2o.sparkling +import ai.h2o.sparkling.sql.catalyst.encoders.RowEncoder import java.io.File -import java.nio.file.Files import java.sql.Timestamp - import org.apache.spark.mllib import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.functions.{lit, rand} import org.apache.spark.sql.types._ @@ -130,7 +128,7 @@ object TestUtils extends Matchers { spark: SparkSession, schemaHolder: SchemaHolder, settings: GenerateDataFrameSettings): DataFrame = { - implicit val encoder: ExpressionEncoder[Row] = RowEncoder(schemaHolder.schema) + implicit val encoder = RowEncoder(schemaHolder.schema) val numberOfPartitions = Math.max(1, settings.numberOfRows / settings.rowsPerPartition) spark .range(settings.numberOfRows) diff --git a/gradle-spark3.3.properties b/gradle-spark3.3.properties index c96eadfb8c..f5ae200b5c 100644 --- a/gradle-spark3.3.properties +++ b/gradle-spark3.3.properties @@ -1,6 +1,6 @@ sparkVersion=3.3.2 minSupportedJavaVersion=1.8 -supportedEmrVersion=emr-6.10.0 +supportedEmrVersion=emr-6.11.1 unsupportedMinorSparkVersions= scalaVersion=2.12.15 databricksVersion=11.0.x-cpu-ml-scala2.12 diff --git a/gradle-spark3.4.properties b/gradle-spark3.4.properties index f75f26d742..cdac42a1ba 100644 --- a/gradle-spark3.4.properties +++ b/gradle-spark3.4.properties @@ -1,6 +1,6 @@ sparkVersion=3.4.1 minSupportedJavaVersion=1.8 -supportedEmrVersion=emr-6.10.0 +supportedEmrVersion=emr-6.13.0 unsupportedMinorSparkVersions= scalaVersion=2.12.17 databricksVersion=13.0.x-cpu-ml-scala2.12 diff --git a/gradle-spark3.5.properties b/gradle-spark3.5.properties new file mode 100644 index 0000000000..c66118a346 --- /dev/null +++ b/gradle-spark3.5.properties @@ -0,0 +1,10 @@ +sparkVersion=3.5.0 +minSupportedJavaVersion=1.8 +supportedEmrVersion=emr-6.10.0 +unsupportedMinorSparkVersions= +scalaVersion=2.12.18 +databricksVersion=14.0.x-cpu-ml-scala2.12 +fabricK8sClientVersion=6.4.1 +executorOverheadMemoryOption=spark.executor.memoryOverhead +driverOverheadMemoryOption=spark.driver.memoryOverhead +supportedPythonVersions=3.8 3.9 3.10 3.11 diff --git a/gradle.properties b/gradle.properties index 814b2733b5..ceaa481a84 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,15 +19,15 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true # Version of Terraform used in the script creating the docker image terraformVersion=0.12.25 # Version of docker image used in Jenkins tests -dockerImageVersion=83 +dockerImageVersion=84 # Is this build nightly build isNightlyBuild=false # Supported Major Spark Versions -supportedSparkVersions=2.3 2.4 3.0 3.1 3.2 3.3 3.4 +supportedSparkVersions=2.3 2.4 3.0 3.1 3.2 3.3 3.4 3.5 # The list of python environments used in automated tests pythonEnvironments=3.6 3.7 3.8 3.9 3.10 3.11 # Select for which Spark version is Sparkling Water built by default -spark=3.4 +spark=3.5 # Sparkling Water Version version=3.44.0.1-1-SNAPSHOT # Spark version from which is Kubernetes Supported diff --git a/py-scoring/README.rst b/py-scoring/README.rst index f16331f036..0b9c7b6a86 100644 --- a/py-scoring/README.rst +++ b/py-scoring/README.rst @@ -7,6 +7,7 @@ This package contains just functionality for scoring with Sparkling Water, H2O-3 Documentation describing scoring with H2O-3 MOJO models is located at: +- For Spark 3.5 - https://docs.h2o.ai/sparkling-water/3.5/latest-stable/doc/deployment/load_mojo.html - For Spark 3.4 - https://docs.h2o.ai/sparkling-water/3.4/latest-stable/doc/deployment/load_mojo.html - For Spark 3.3 - https://docs.h2o.ai/sparkling-water/3.3/latest-stable/doc/deployment/load_mojo.html - For Spark 3.2 - https://docs.h2o.ai/sparkling-water/3.2/latest-stable/doc/deployment/load_mojo.html diff --git a/py/README.rst b/py/README.rst index cdfd1108cd..2b6d70183a 100644 --- a/py/README.rst +++ b/py/README.rst @@ -8,6 +8,7 @@ to use this package for scoring with Driverless AI MOJO models. PySparkling Documentation is hosted at our documentation page: +- For Spark 3.5 - http://docs.h2o.ai/sparkling-water/3.5/latest-stable/doc/pysparkling.html - For Spark 3.4 - http://docs.h2o.ai/sparkling-water/3.4/latest-stable/doc/pysparkling.html - For Spark 3.3 - http://docs.h2o.ai/sparkling-water/3.3/latest-stable/doc/pysparkling.html - For Spark 3.2 - http://docs.h2o.ai/sparkling-water/3.2/latest-stable/doc/pysparkling.html diff --git a/scoring/src/main/scala/ai/h2o/sparkling/ml/models/H2OMOJOPipelineModel.scala b/scoring/src/main/scala/ai/h2o/sparkling/ml/models/H2OMOJOPipelineModel.scala index 7fa155d6c0..d6e9f7a874 100644 --- a/scoring/src/main/scala/ai/h2o/sparkling/ml/models/H2OMOJOPipelineModel.scala +++ b/scoring/src/main/scala/ai/h2o/sparkling/ml/models/H2OMOJOPipelineModel.scala @@ -17,21 +17,21 @@ package ai.h2o.sparkling.ml.models -import java.io._ import ai.h2o.mojos.runtime.MojoPipeline import ai.h2o.mojos.runtime.api.{MojoPipelineService, PipelineConfig} import ai.h2o.mojos.runtime.frame.MojoColumn.Type import ai.h2o.mojos.runtime.frame.MojoFrame import ai.h2o.sparkling.ml.params.{H2OAlgorithmMOJOParams, H2OBaseMOJOParams, HasFeatureTypesOnMOJO} -import org.apache.spark.ml.param._ -import org.apache.spark.sql._ +import ai.h2o.sparkling.sql.catalyst.encoders.RowEncoder import com.google.common.collect.Iterators import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.Model -import org.apache.spark.sql.catalyst.encoders.RowEncoder +import org.apache.spark.ml.param._ +import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ +import java.io._ import scala.collection.JavaConverters._ class H2OMOJOPipelineModel(override val uid: String) diff --git a/utils/src/main/scala/ai/h2o/sparkling/ml/utils/SchemaUtils.scala b/utils/src/main/scala/ai/h2o/sparkling/ml/utils/SchemaUtils.scala index 9028e8a309..87815bdd3d 100644 --- a/utils/src/main/scala/ai/h2o/sparkling/ml/utils/SchemaUtils.scala +++ b/utils/src/main/scala/ai/h2o/sparkling/ml/utils/SchemaUtils.scala @@ -17,9 +17,9 @@ package ai.h2o.sparkling.ml.utils +import ai.h2o.sparkling.sql.catalyst.encoders.RowEncoder import org.apache.spark.ml.attribute.AttributeGroup import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ diff --git a/utils/src/main/scala_spark_3.0/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.0/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_3.0/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +} diff --git a/utils/src/main/scala_spark_3.1/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.1/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_3.1/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +} diff --git a/utils/src/main/scala_spark_3.2/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.2/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_3.2/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +} diff --git a/utils/src/main/scala_spark_3.3/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.3/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_3.3/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +} diff --git a/utils/src/main/scala_spark_3.4/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.4/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_3.4/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +} diff --git a/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..151a2dc8b5 --- /dev/null +++ b/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.catalyst.encoders.RowEncoder.encoderFor +import org.apache.spark.sql.types.StructType + +/** + * Spark 3.5 removed apply method for RowEncoder, forcing the user to use encoderFor method + * But some older versions do not have it, hence we use different code for older/newer Spark versions. + */ +object RowEncoder { + def apply(schema: StructType) = ExpressionEncoder(encoderFor(schema)) + +} diff --git a/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/functions.scala b/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/functions.scala new file mode 100644 index 0000000000..44442efc4e --- /dev/null +++ b/utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/functions.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql + +import org.apache.spark.sql.expose.SparkUserDefinedFunction +import org.apache.spark.sql.expressions.UserDefinedFunction +import org.apache.spark.sql.types.DataType + +/** + * This objects contains functions from org.apache.spark.sql.functions which are not compatible across all Spark + * versions supported by Sparkling Water. + */ +object functions { + def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = { + SparkUserDefinedFunction(f, dataType, inputEncoders = Nil, outputEncoder = None) + } +} diff --git a/utils/src/main/scala_spark_3.5/org/apache/spark/sql/expose/SparkUserDefinedFunction.scala b/utils/src/main/scala_spark_3.5/org/apache/spark/sql/expose/SparkUserDefinedFunction.scala new file mode 100644 index 0000000000..3a7ce735db --- /dev/null +++ b/utils/src/main/scala_spark_3.5/org/apache/spark/sql/expose/SparkUserDefinedFunction.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.expose + +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.expressions +import org.apache.spark.sql.types.DataType + +object SparkUserDefinedFunction { + def apply( + f: AnyRef, + dataType: DataType, + inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil, + outputEncoder: Option[ExpressionEncoder[_]] = None, + name: Option[String] = None, + nullable: Boolean = true, + deterministic: Boolean = true): expressions.SparkUserDefinedFunction = { + expressions.SparkUserDefinedFunction(f, dataType, inputEncoders, outputEncoder, name, nullable, deterministic) + } +} diff --git a/utils/src/main/scala_spark_others/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala b/utils/src/main/scala_spark_others/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala new file mode 100644 index 0000000000..d6b2554615 --- /dev/null +++ b/utils/src/main/scala_spark_others/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.h2o.sparkling.sql.catalyst.encoders + +import org.apache.spark.sql.types.StructType + +/** + * for explanation see utils/src/main/scala_spark_3.5/ai/h2o/sparkling/sql/catalyst/encoders/RowEncoder.scala + */ +object RowEncoder { + def apply(schema: StructType) = org.apache.spark.sql.catalyst.encoders.RowEncoder(schema) +}