# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from functools import total_ordering import itertools import os import re all_modules = [] @total_ordering class Module(object): """ A module is the basic abstraction in our test runner script. Each module consists of a set of source files, a set of test commands, and a set of dependencies on other modules. We use modules to define a dependency graph that let us determine which tests to run based on which files have changed. """ def __init__( self, name, dependencies, source_file_regexes, build_profile_flags=(), environ=None, sbt_test_goals=(), python_test_goals=(), excluded_python_implementations=(), test_tags=(), should_run_r_tests=False, should_run_build_tests=False, ): """ Define a new module. :param name: A short module name, for display in logging and error messages. :param dependencies: A set of dependencies for this module. This should only include direct dependencies; transitive dependencies are resolved automatically. :param source_file_regexes: a set of regexes that match source files belonging to this module. These regexes are applied by attempting to match at the beginning of the filename strings. :param build_profile_flags: A set of profile flags that should be passed to Maven or SBT in order to build and test this module (e.g. '-PprofileName'). :param environ: A dict of environment variables that should be set when files in this module are changed. :param sbt_test_goals: A set of SBT test goals for testing this module. :param python_test_goals: A set of Python test goals for testing this module. :param excluded_python_implementations: A set of Python implementations that are not supported by this module's Python components. The values in this set should match strings returned by Python's `platform.python_implementation()`. :param test_tags A set of tags that will be excluded when running unit tests if the module is not explicitly changed. :param should_run_r_tests: If true, changes in this module will trigger all R tests. :param should_run_build_tests: If true, changes in this module will trigger build tests. """ self.name = name self.dependencies = dependencies self.source_file_prefixes = source_file_regexes self.sbt_test_goals = sbt_test_goals self.build_profile_flags = build_profile_flags self.environ = environ or {} self.python_test_goals = python_test_goals self.excluded_python_implementations = excluded_python_implementations self.test_tags = test_tags self.should_run_r_tests = should_run_r_tests self.should_run_build_tests = should_run_build_tests self.dependent_modules = set() for dep in dependencies: dep.dependent_modules.add(self) all_modules.append(self) def contains_file(self, filename): return any(re.match(p, filename) for p in self.source_file_prefixes) def __repr__(self): return "Module<%s>" % self.name def __lt__(self, other): return self.name < other.name def __eq__(self, other): return self.name == other.name def __ne__(self, other): return not (self.name == other.name) def __hash__(self): return hash(self.name) tags = Module( name="tags", dependencies=[], source_file_regexes=[ "common/tags/", ], ) utils = Module( name="utils", dependencies=[tags], source_file_regexes=[ "common/utils/", ], sbt_test_goals=[ "common-utils/test", ], ) kvstore = Module( name="kvstore", dependencies=[tags], source_file_regexes=[ "common/kvstore/", ], sbt_test_goals=[ "kvstore/test", ], ) network_common = Module( name="network-common", dependencies=[tags, utils], source_file_regexes=[ "common/network-common/", ], sbt_test_goals=[ "network-common/test", ], ) network_shuffle = Module( name="network-shuffle", dependencies=[tags], source_file_regexes=[ "common/network-shuffle/", ], sbt_test_goals=[ "network-shuffle/test", ], ) unsafe = Module( name="unsafe", dependencies=[tags, utils], source_file_regexes=[ "common/unsafe", ], sbt_test_goals=[ "unsafe/test", ], ) launcher = Module( name="launcher", dependencies=[tags], source_file_regexes=[ "launcher/", ], sbt_test_goals=[ "launcher/test", ], ) sketch = Module( name="sketch", dependencies=[tags], source_file_regexes=[ "common/sketch/", ], sbt_test_goals=["sketch/test"], ) variant = Module( name="variant", dependencies=[tags], source_file_regexes=[ "common/variant/", ], sbt_test_goals=["variant/test"], ) core = Module( name="core", dependencies=[kvstore, network_common, network_shuffle, unsafe, launcher, utils], source_file_regexes=[ "core/", ], sbt_test_goals=[ "core/test", ], ) api = Module( name="api", dependencies=[utils, unsafe], source_file_regexes=[ "sql/api/", ], ) catalyst = Module( name="catalyst", dependencies=[tags, sketch, variant, core, api], source_file_regexes=[ "sql/catalyst/", ], sbt_test_goals=[ "catalyst/test", ], environ=None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}, ) sql = Module( name="sql", dependencies=[catalyst], source_file_regexes=[ "sql/core/", "python/pyspark/sql/worker/", # analyze_udtf is invoked and tested in JVM ], sbt_test_goals=[ "sql/test", ], environ=None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}, ) hive = Module( name="hive", dependencies=[sql], source_file_regexes=[ "sql/hive/", "bin/spark-sql", ], build_profile_flags=[ "-Phive", ], sbt_test_goals=[ "hive/test", ], test_tags=["org.apache.spark.tags.ExtendedHiveTest"], ) repl = Module( name="repl", dependencies=[hive], source_file_regexes=[ "repl/", ], sbt_test_goals=[ "repl/test", ], ) hive_thriftserver = Module( name="hive-thriftserver", dependencies=[hive], source_file_regexes=[ "sql/hive-thriftserver", "sbin/start-thriftserver.sh", ], build_profile_flags=[ "-Phive-thriftserver", ], sbt_test_goals=[ "hive-thriftserver/test", ], ) avro = Module( name="avro", dependencies=[sql], source_file_regexes=[ "connector/avro", ], sbt_test_goals=[ "avro/test", ], ) sql_kafka = Module( name="sql-kafka-0-10", dependencies=[sql], source_file_regexes=[ "connector/kafka-0-10-sql", ], sbt_test_goals=[ "sql-kafka-0-10/test", ], ) profiler = Module( name="profiler", dependencies=[], build_profile_flags=["-Pjvm-profiler"], source_file_regexes=[ "connector/profiler", ], ) protobuf = Module( name="protobuf", dependencies=[sql], source_file_regexes=[ "connector/protobuf", ], sbt_test_goals=[ "protobuf/test", ], ) graphx = Module( name="graphx", dependencies=[tags, core], source_file_regexes=[ "graphx/", ], sbt_test_goals=["graphx/test"], ) streaming = Module( name="streaming", dependencies=[tags, core], source_file_regexes=[ "streaming", ], sbt_test_goals=[ "streaming/test", ], ) # Don't set the dependencies because changes in other modules should not trigger Kinesis tests. # Kinesis tests depends on external Amazon kinesis service. We should run these tests only when # files in streaming_kinesis_asl are changed, so that if Kinesis experiences an outage, we don't # fail other PRs. streaming_kinesis_asl = Module( name="streaming-kinesis-asl", dependencies=[tags, core], source_file_regexes=[ "connector/kinesis-asl/", "connector/kinesis-asl-assembly/", ], build_profile_flags=[ "-Pkinesis-asl", ], environ={"ENABLE_KINESIS_TESTS": "0"}, sbt_test_goals=[ "streaming-kinesis-asl/test", ], ) streaming_kafka_0_10 = Module( name="streaming-kafka-0-10", dependencies=[streaming, core], source_file_regexes=[ # The ending "/" is necessary otherwise it will include "sql-kafka" codes "connector/kafka-0-10/", "connector/kafka-0-10-assembly", "connector/kafka-0-10-token-provider", ], sbt_test_goals=["streaming-kafka-0-10/test", "token-provider-kafka-0-10/test"], ) mllib_local = Module( name="mllib-local", dependencies=[tags, core], source_file_regexes=[ "mllib-local", ], sbt_test_goals=[ "mllib-local/test", ], ) mllib = Module( name="mllib", dependencies=[mllib_local, streaming, sql], source_file_regexes=[ "data/mllib/", "mllib/", ], sbt_test_goals=[ "mllib/test", ], ) connect = Module( name="connect", dependencies=[hive, avro, protobuf, mllib], source_file_regexes=[ "sql/connect", ], sbt_test_goals=[ "connect/test", "connect-client-jvm/test", ], ) examples = Module( name="examples", dependencies=[graphx, mllib, streaming, hive], source_file_regexes=[ "examples/", ], sbt_test_goals=[ "examples/test", ], ) pyspark_core = Module( name="pyspark-core", dependencies=[core], source_file_regexes=["python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))"], python_test_goals=[ # doctests "pyspark.conf", "pyspark.core.rdd", "pyspark.core.context", "pyspark.core.broadcast", "pyspark.accumulators", "pyspark.core.files", "pyspark.serializers", "pyspark.profiler", "pyspark.shuffle", "pyspark.taskcontext", "pyspark.util", # unittests "pyspark.tests.test_appsubmit", "pyspark.tests.test_broadcast", "pyspark.tests.test_conf", "pyspark.tests.test_context", "pyspark.tests.test_daemon", "pyspark.tests.test_install_spark", "pyspark.tests.test_join", "pyspark.tests.test_memory_profiler", "pyspark.tests.test_profiler", "pyspark.tests.test_rdd", "pyspark.tests.test_rddbarrier", "pyspark.tests.test_rddsampler", "pyspark.tests.test_readwrite", "pyspark.tests.test_serializers", "pyspark.tests.test_shuffle", "pyspark.tests.test_statcounter", "pyspark.tests.test_taskcontext", "pyspark.tests.test_util", "pyspark.tests.test_worker", "pyspark.tests.test_stage_sched", ], ) pyspark_sql = Module( name="pyspark-sql", dependencies=[pyspark_core, hive, avro, protobuf], source_file_regexes=["python/pyspark/sql"], python_test_goals=[ # doctests "pyspark.sql.types", "pyspark.sql.context", "pyspark.sql.session", "pyspark.sql.conf", "pyspark.sql.catalog", "pyspark.sql.classic.column", "pyspark.sql.classic.dataframe", "pyspark.sql.classic.window", "pyspark.sql.datasource", "pyspark.sql.group", "pyspark.sql.functions.builtin", "pyspark.sql.functions.partitioning", "pyspark.sql.merge", "pyspark.sql.readwriter", "pyspark.sql.streaming.query", "pyspark.sql.streaming.readwriter", "pyspark.sql.streaming.listener", "pyspark.sql.udf", "pyspark.sql.udtf", "pyspark.sql.avro.functions", "pyspark.sql.protobuf.functions", "pyspark.sql.pandas.conversion", "pyspark.sql.pandas.map_ops", "pyspark.sql.pandas.group_ops", "pyspark.sql.pandas.types", "pyspark.sql.pandas.serializers", "pyspark.sql.pandas.typehints", "pyspark.sql.pandas.utils", "pyspark.sql.observation", "pyspark.sql.tvf", # unittests "pyspark.sql.tests.test_artifact", "pyspark.sql.tests.test_catalog", "pyspark.sql.tests.test_column", "pyspark.sql.tests.test_conf", "pyspark.sql.tests.test_context", "pyspark.sql.tests.test_dataframe", "pyspark.sql.tests.test_collection", "pyspark.sql.tests.test_creation", "pyspark.sql.tests.test_listener", "pyspark.sql.tests.test_observation", "pyspark.sql.tests.test_repartition", "pyspark.sql.tests.test_stat", "pyspark.sql.tests.test_datasources", "pyspark.sql.tests.test_errors", "pyspark.sql.tests.test_functions", "pyspark.sql.tests.test_group", "pyspark.sql.tests.test_sql", "pyspark.sql.tests.arrow.test_arrow", "pyspark.sql.tests.arrow.test_arrow_map", "pyspark.sql.tests.arrow.test_arrow_cogrouped_map", "pyspark.sql.tests.arrow.test_arrow_grouped_map", "pyspark.sql.tests.arrow.test_arrow_python_udf", "pyspark.sql.tests.pandas.test_pandas_cogrouped_map", "pyspark.sql.tests.pandas.test_pandas_grouped_map", "pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state", "pyspark.sql.tests.pandas.test_pandas_map", "pyspark.sql.tests.pandas.test_pandas_transform_with_state", "pyspark.sql.tests.pandas.test_pandas_udf", "pyspark.sql.tests.pandas.test_pandas_udf_grouped_agg", "pyspark.sql.tests.pandas.test_pandas_udf_scalar", "pyspark.sql.tests.pandas.test_pandas_udf_typehints", "pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations", "pyspark.sql.tests.pandas.test_pandas_udf_window", "pyspark.sql.tests.pandas.test_pandas_sqlmetrics", "pyspark.sql.tests.pandas.test_converter", "pyspark.sql.tests.test_python_datasource", "pyspark.sql.tests.test_python_streaming_datasource", "pyspark.sql.tests.test_readwriter", "pyspark.sql.tests.test_serde", "pyspark.sql.tests.test_session", "pyspark.sql.tests.streaming.test_streaming", "pyspark.sql.tests.streaming.test_streaming_foreach", "pyspark.sql.tests.streaming.test_streaming_foreach_batch", "pyspark.sql.tests.streaming.test_streaming_listener", "pyspark.sql.tests.test_subquery", "pyspark.sql.tests.test_types", "pyspark.sql.tests.test_udf", "pyspark.sql.tests.test_udf_profiler", "pyspark.sql.tests.test_udtf", "pyspark.sql.tests.test_tvf", "pyspark.sql.tests.test_utils", "pyspark.sql.tests.test_resources", "pyspark.sql.tests.plot.test_frame_plot", "pyspark.sql.tests.plot.test_frame_plot_plotly", "pyspark.sql.tests.test_connect_compatibility", ], ) pyspark_testing = Module( name="pyspark-testing", dependencies=[pyspark_core, pyspark_sql], source_file_regexes=["python/pyspark/testing"], python_test_goals=[ # doctests "pyspark.testing.utils", "pyspark.testing.pandasutils", ], ) pyspark_resource = Module( name="pyspark-resource", dependencies=[pyspark_core], source_file_regexes=["python/pyspark/resource"], python_test_goals=[ # doctests "pyspark.resource.profile", # unittests "pyspark.resource.tests.test_resources", "pyspark.resource.tests.test_connect_resources", ], ) pyspark_streaming = Module( name="pyspark-streaming", dependencies=[pyspark_core, streaming, streaming_kinesis_asl], source_file_regexes=["python/pyspark/streaming"], python_test_goals=[ # doctests "pyspark.streaming.util", # unittests "pyspark.streaming.tests.test_context", "pyspark.streaming.tests.test_dstream", "pyspark.streaming.tests.test_kinesis", "pyspark.streaming.tests.test_listener", ], ) pyspark_mllib = Module( name="pyspark-mllib", dependencies=[pyspark_core, pyspark_streaming, pyspark_sql, mllib], source_file_regexes=["python/pyspark/mllib"], python_test_goals=[ # doctests "pyspark.mllib.classification", "pyspark.mllib.clustering", "pyspark.mllib.evaluation", "pyspark.mllib.feature", "pyspark.mllib.fpm", "pyspark.mllib.linalg.__init__", "pyspark.mllib.linalg.distributed", "pyspark.mllib.random", "pyspark.mllib.recommendation", "pyspark.mllib.regression", "pyspark.mllib.stat._statistics", "pyspark.mllib.stat.KernelDensity", "pyspark.mllib.tree", "pyspark.mllib.util", # unittests "pyspark.mllib.tests.test_algorithms", "pyspark.mllib.tests.test_feature", "pyspark.mllib.tests.test_linalg", "pyspark.mllib.tests.test_stat", "pyspark.mllib.tests.test_streaming_algorithms", "pyspark.mllib.tests.test_util", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there ], ) pyspark_ml = Module( name="pyspark-ml", dependencies=[pyspark_core, pyspark_mllib], source_file_regexes=["python/pyspark/ml/"], python_test_goals=[ # doctests "pyspark.ml.classification", "pyspark.ml.clustering", "pyspark.ml.evaluation", "pyspark.ml.feature", "pyspark.ml.fpm", "pyspark.ml.functions", "pyspark.ml.image", "pyspark.ml.linalg.__init__", "pyspark.ml.recommendation", "pyspark.ml.regression", "pyspark.ml.stat", "pyspark.ml.tuning", # unittests "pyspark.ml.tests.test_algorithms", "pyspark.ml.tests.test_als", "pyspark.ml.tests.test_fpm", "pyspark.ml.tests.test_base", "pyspark.ml.tests.test_evaluation", "pyspark.ml.tests.test_feature", "pyspark.ml.tests.test_functions", "pyspark.ml.tests.test_image", "pyspark.ml.tests.test_linalg", "pyspark.ml.tests.test_model_cache", "pyspark.ml.tests.test_param", "pyspark.ml.tests.test_persistence", "pyspark.ml.tests.test_pipeline", "pyspark.ml.tests.test_tuning", "pyspark.ml.tests.test_ovr", "pyspark.ml.tests.test_stat", "pyspark.ml.tests.test_training_summary", "pyspark.ml.tests.tuning.test_tuning", "pyspark.ml.tests.tuning.test_cv_io_basic", "pyspark.ml.tests.tuning.test_cv_io_nested", "pyspark.ml.tests.tuning.test_cv_io_pipeline", "pyspark.ml.tests.tuning.test_tvs_io_basic", "pyspark.ml.tests.tuning.test_tvs_io_nested", "pyspark.ml.tests.tuning.test_tvs_io_pipeline", "pyspark.ml.tests.test_util", "pyspark.ml.tests.test_wrapper", "pyspark.ml.torch.tests.test_distributor", "pyspark.ml.torch.tests.test_log_communication", "pyspark.ml.torch.tests.test_data_loader", "pyspark.ml.deepspeed.tests.test_deepspeed_distributor", "pyspark.ml.tests.connect.test_legacy_mode_summarizer", "pyspark.ml.tests.connect.test_legacy_mode_evaluation", "pyspark.ml.tests.connect.test_legacy_mode_feature", "pyspark.ml.tests.connect.test_legacy_mode_classification", "pyspark.ml.tests.connect.test_legacy_mode_pipeline", "pyspark.ml.tests.connect.test_legacy_mode_tuning", "pyspark.ml.tests.test_classification", "pyspark.ml.tests.test_regression", "pyspark.ml.tests.test_clustering", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there ], ) pyspark_pandas = Module( name="pyspark-pandas", dependencies=[pyspark_core, pyspark_sql], source_file_regexes=["python/pyspark/pandas/"], python_test_goals=[ # doctests "pyspark.pandas.accessors", "pyspark.pandas.base", "pyspark.pandas.categorical", "pyspark.pandas.config", "pyspark.pandas.datetimes", "pyspark.pandas.exceptions", "pyspark.pandas.extensions", "pyspark.pandas.groupby", "pyspark.pandas.indexing", "pyspark.pandas.internal", "pyspark.pandas.mlflow", "pyspark.pandas.namespace", "pyspark.pandas.numpy_compat", "pyspark.pandas.sql_processor", "pyspark.pandas.sql_formatter", "pyspark.pandas.strings", "pyspark.pandas.supported_api_gen", "pyspark.pandas.utils", "pyspark.pandas.window", "pyspark.pandas.indexes.base", "pyspark.pandas.indexes.category", "pyspark.pandas.indexes.datetimes", "pyspark.pandas.indexes.timedelta", "pyspark.pandas.indexes.multi", "pyspark.pandas.spark.accessors", "pyspark.pandas.spark.utils", "pyspark.pandas.typedef.typehints", # unittests "pyspark.pandas.tests.test_categorical", "pyspark.pandas.tests.test_config", "pyspark.pandas.tests.test_extension", "pyspark.pandas.tests.test_frame_spark", "pyspark.pandas.tests.test_generic_functions", "pyspark.pandas.tests.test_indexops_spark", "pyspark.pandas.tests.test_internal", "pyspark.pandas.tests.test_namespace", "pyspark.pandas.tests.test_numpy_compat", "pyspark.pandas.tests.test_repr", "pyspark.pandas.tests.test_spark_functions", "pyspark.pandas.tests.test_scalars", "pyspark.pandas.tests.test_sql", "pyspark.pandas.tests.test_typedef", "pyspark.pandas.tests.test_utils", "pyspark.pandas.tests.computation.test_any_all", "pyspark.pandas.tests.computation.test_apply_func", "pyspark.pandas.tests.computation.test_binary_ops", "pyspark.pandas.tests.computation.test_combine", "pyspark.pandas.tests.computation.test_compute", "pyspark.pandas.tests.computation.test_corr", "pyspark.pandas.tests.computation.test_corrwith", "pyspark.pandas.tests.computation.test_cov", "pyspark.pandas.tests.computation.test_cumulative", "pyspark.pandas.tests.computation.test_describe", "pyspark.pandas.tests.computation.test_eval", "pyspark.pandas.tests.computation.test_melt", "pyspark.pandas.tests.computation.test_missing_data", "pyspark.pandas.tests.computation.test_pivot", "pyspark.pandas.tests.computation.test_pivot_table", "pyspark.pandas.tests.computation.test_pivot_table_adv", "pyspark.pandas.tests.computation.test_pivot_table_multi_idx", "pyspark.pandas.tests.computation.test_pivot_table_multi_idx_adv", "pyspark.pandas.tests.computation.test_stats", "pyspark.pandas.tests.data_type_ops.test_as_type", "pyspark.pandas.tests.data_type_ops.test_base", "pyspark.pandas.tests.data_type_ops.test_binary_ops", "pyspark.pandas.tests.data_type_ops.test_boolean_ops", "pyspark.pandas.tests.data_type_ops.test_categorical_ops", "pyspark.pandas.tests.data_type_ops.test_complex_ops", "pyspark.pandas.tests.data_type_ops.test_date_ops", "pyspark.pandas.tests.data_type_ops.test_datetime_ops", "pyspark.pandas.tests.data_type_ops.test_null_ops", "pyspark.pandas.tests.data_type_ops.test_num_ops", "pyspark.pandas.tests.data_type_ops.test_num_arithmetic", "pyspark.pandas.tests.data_type_ops.test_num_mod", "pyspark.pandas.tests.data_type_ops.test_num_mul_div", "pyspark.pandas.tests.data_type_ops.test_num_pow", "pyspark.pandas.tests.data_type_ops.test_num_reverse", "pyspark.pandas.tests.data_type_ops.test_string_ops", "pyspark.pandas.tests.data_type_ops.test_udt_ops", "pyspark.pandas.tests.data_type_ops.test_timedelta_ops", "pyspark.pandas.tests.plot.test_frame_plot", "pyspark.pandas.tests.plot.test_frame_plot_matplotlib", "pyspark.pandas.tests.plot.test_frame_plot_plotly", "pyspark.pandas.tests.plot.test_series_plot", "pyspark.pandas.tests.plot.test_series_plot_matplotlib", "pyspark.pandas.tests.plot.test_series_plot_plotly", "pyspark.pandas.tests.frame.test_interpolate", "pyspark.pandas.tests.frame.test_interpolate_error", "pyspark.pandas.tests.frame.test_attrs", "pyspark.pandas.tests.frame.test_axis", "pyspark.pandas.tests.frame.test_constructor", "pyspark.pandas.tests.frame.test_conversion", "pyspark.pandas.tests.frame.test_reindexing", "pyspark.pandas.tests.frame.test_reshaping", "pyspark.pandas.tests.frame.test_spark", "pyspark.pandas.tests.frame.test_take", "pyspark.pandas.tests.frame.test_take_adv", "pyspark.pandas.tests.frame.test_time_series", "pyspark.pandas.tests.frame.test_truncate", "pyspark.pandas.tests.series.test_interpolate", "pyspark.pandas.tests.resample.test_on", "pyspark.pandas.tests.resample.test_error", "pyspark.pandas.tests.resample.test_frame", "pyspark.pandas.tests.resample.test_missing", "pyspark.pandas.tests.resample.test_series", "pyspark.pandas.tests.resample.test_timezone", "pyspark.pandas.tests.reshape.test_get_dummies", "pyspark.pandas.tests.reshape.test_get_dummies_kwargs", "pyspark.pandas.tests.reshape.test_get_dummies_multiindex", "pyspark.pandas.tests.reshape.test_get_dummies_object", "pyspark.pandas.tests.reshape.test_get_dummies_prefix", "pyspark.pandas.tests.reshape.test_merge_asof", "pyspark.pandas.tests.window.test_expanding", "pyspark.pandas.tests.window.test_expanding_adv", "pyspark.pandas.tests.window.test_expanding_error", "pyspark.pandas.tests.window.test_groupby_expanding", "pyspark.pandas.tests.window.test_groupby_expanding_adv", "pyspark.pandas.tests.window.test_ewm_error", "pyspark.pandas.tests.window.test_ewm_mean", "pyspark.pandas.tests.window.test_groupby_ewm_mean", "pyspark.pandas.tests.window.test_missing", "pyspark.pandas.tests.window.test_rolling", "pyspark.pandas.tests.window.test_rolling_adv", "pyspark.pandas.tests.window.test_rolling_count", "pyspark.pandas.tests.window.test_rolling_error", "pyspark.pandas.tests.window.test_groupby_rolling", "pyspark.pandas.tests.window.test_groupby_rolling_adv", "pyspark.pandas.tests.window.test_groupby_rolling_count", "pyspark.pandas.tests.series.test_datetime", "pyspark.pandas.tests.series.test_string_ops_adv", "pyspark.pandas.tests.series.test_string_ops_basic", "pyspark.pandas.tests.series.test_all_any", "pyspark.pandas.tests.series.test_arg_ops", "pyspark.pandas.tests.series.test_as_of", "pyspark.pandas.tests.series.test_as_type", "pyspark.pandas.tests.series.test_compute", "pyspark.pandas.tests.series.test_conversion", "pyspark.pandas.tests.series.test_cumulative", "pyspark.pandas.tests.series.test_index", "pyspark.pandas.tests.series.test_missing_data", "pyspark.pandas.tests.series.test_series", "pyspark.pandas.tests.series.test_sort", "pyspark.pandas.tests.series.test_stat", "pyspark.pandas.tests.io.test_io", "pyspark.pandas.tests.io.test_csv", "pyspark.pandas.tests.io.test_feather", "pyspark.pandas.tests.io.test_stata", "pyspark.pandas.tests.io.test_dataframe_conversion", "pyspark.pandas.tests.io.test_dataframe_spark_io", "pyspark.pandas.tests.io.test_series_conversion", # fallback "pyspark.pandas.tests.frame.test_asfreq", "pyspark.pandas.tests.frame.test_asof", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_pandas_slow = Module( name="pyspark-pandas-slow", dependencies=[pyspark_core, pyspark_sql], source_file_regexes=["python/pyspark/pandas/"], python_test_goals=[ # doctests "pyspark.pandas.frame", "pyspark.pandas.generic", "pyspark.pandas.series", # unittests "pyspark.pandas.tests.indexes.test_default", "pyspark.pandas.tests.indexes.test_category", "pyspark.pandas.tests.indexes.test_timedelta", "pyspark.pandas.tests.indexes.test_basic", "pyspark.pandas.tests.indexes.test_getattr", "pyspark.pandas.tests.indexes.test_name", "pyspark.pandas.tests.indexes.test_conversion", "pyspark.pandas.tests.indexes.test_drop", "pyspark.pandas.tests.indexes.test_level", "pyspark.pandas.tests.indexes.test_missing", "pyspark.pandas.tests.indexes.test_repeat", "pyspark.pandas.tests.indexes.test_sort", "pyspark.pandas.tests.indexes.test_stat", "pyspark.pandas.tests.indexes.test_symmetric_diff", "pyspark.pandas.tests.indexes.test_take", "pyspark.pandas.tests.indexes.test_unique", "pyspark.pandas.tests.indexes.test_asof", "pyspark.pandas.tests.indexes.test_astype", "pyspark.pandas.tests.indexes.test_delete", "pyspark.pandas.tests.indexes.test_diff", "pyspark.pandas.tests.indexes.test_insert", "pyspark.pandas.tests.indexes.test_map", "pyspark.pandas.tests.indexes.test_append", "pyspark.pandas.tests.indexes.test_intersection", "pyspark.pandas.tests.indexes.test_monotonic", "pyspark.pandas.tests.indexes.test_union", "pyspark.pandas.tests.indexes.test_datetime", "pyspark.pandas.tests.indexes.test_datetime_at", "pyspark.pandas.tests.indexes.test_datetime_between", "pyspark.pandas.tests.indexes.test_datetime_ceil", "pyspark.pandas.tests.indexes.test_datetime_floor", "pyspark.pandas.tests.indexes.test_datetime_iso", "pyspark.pandas.tests.indexes.test_datetime_map", "pyspark.pandas.tests.indexes.test_datetime_property", "pyspark.pandas.tests.indexes.test_datetime_round", "pyspark.pandas.tests.indexes.test_align", "pyspark.pandas.tests.indexes.test_indexing", "pyspark.pandas.tests.indexes.test_indexing_adv", "pyspark.pandas.tests.indexes.test_indexing_basic", "pyspark.pandas.tests.indexes.test_indexing_iloc", "pyspark.pandas.tests.indexes.test_indexing_loc", "pyspark.pandas.tests.indexes.test_indexing_loc_2d", "pyspark.pandas.tests.indexes.test_indexing_loc_multi_idx", "pyspark.pandas.tests.indexes.test_reindex", "pyspark.pandas.tests.indexes.test_rename", "pyspark.pandas.tests.indexes.test_reset_index", "pyspark.pandas.tests.groupby.test_aggregate", "pyspark.pandas.tests.groupby.test_apply_func", "pyspark.pandas.tests.groupby.test_corr", "pyspark.pandas.tests.groupby.test_cumulative", "pyspark.pandas.tests.groupby.test_describe", "pyspark.pandas.tests.groupby.test_groupby", "pyspark.pandas.tests.groupby.test_grouping", "pyspark.pandas.tests.groupby.test_head_tail", "pyspark.pandas.tests.groupby.test_index", "pyspark.pandas.tests.groupby.test_missing", "pyspark.pandas.tests.groupby.test_missing_data", "pyspark.pandas.tests.groupby.test_nlargest_nsmallest", "pyspark.pandas.tests.groupby.test_raises", "pyspark.pandas.tests.groupby.test_rank", "pyspark.pandas.tests.groupby.test_size", "pyspark.pandas.tests.groupby.test_split_apply", "pyspark.pandas.tests.groupby.test_split_apply_count", "pyspark.pandas.tests.groupby.test_split_apply_first", "pyspark.pandas.tests.groupby.test_split_apply_last", "pyspark.pandas.tests.groupby.test_split_apply_min_max", "pyspark.pandas.tests.groupby.test_split_apply_skew", "pyspark.pandas.tests.groupby.test_split_apply_std", "pyspark.pandas.tests.groupby.test_split_apply_var", "pyspark.pandas.tests.groupby.test_stat", "pyspark.pandas.tests.groupby.test_stat_adv", "pyspark.pandas.tests.groupby.test_stat_ddof", "pyspark.pandas.tests.groupby.test_stat_func", "pyspark.pandas.tests.groupby.test_stat_prod", "pyspark.pandas.tests.groupby.test_value_counts", "pyspark.pandas.tests.diff_frames_ops.test_align", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_ext", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_ext_float", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain_ext", "pyspark.pandas.tests.diff_frames_ops.test_arithmetic_chain_ext_float", "pyspark.pandas.tests.diff_frames_ops.test_assign_frame", "pyspark.pandas.tests.diff_frames_ops.test_assign_series", "pyspark.pandas.tests.diff_frames_ops.test_basic", "pyspark.pandas.tests.diff_frames_ops.test_bitwise", "pyspark.pandas.tests.diff_frames_ops.test_combine_first", "pyspark.pandas.tests.diff_frames_ops.test_compare_series", "pyspark.pandas.tests.diff_frames_ops.test_concat_inner", "pyspark.pandas.tests.diff_frames_ops.test_concat_outer", "pyspark.pandas.tests.diff_frames_ops.test_basic_slow", "pyspark.pandas.tests.diff_frames_ops.test_cov", "pyspark.pandas.tests.diff_frames_ops.test_corrwith", "pyspark.pandas.tests.diff_frames_ops.test_dot_frame", "pyspark.pandas.tests.diff_frames_ops.test_dot_series", "pyspark.pandas.tests.diff_frames_ops.test_error", "pyspark.pandas.tests.diff_frames_ops.test_index", "pyspark.pandas.tests.diff_frames_ops.test_series", "pyspark.pandas.tests.diff_frames_ops.test_setitem_frame", "pyspark.pandas.tests.diff_frames_ops.test_setitem_series", "pyspark.pandas.tests.diff_frames_ops.test_groupby", "pyspark.pandas.tests.diff_frames_ops.test_groupby_aggregate", "pyspark.pandas.tests.diff_frames_ops.test_groupby_apply", "pyspark.pandas.tests.diff_frames_ops.test_groupby_cumulative", "pyspark.pandas.tests.diff_frames_ops.test_groupby_diff", "pyspark.pandas.tests.diff_frames_ops.test_groupby_diff_len", "pyspark.pandas.tests.diff_frames_ops.test_groupby_fillna", "pyspark.pandas.tests.diff_frames_ops.test_groupby_filter", "pyspark.pandas.tests.diff_frames_ops.test_groupby_shift", "pyspark.pandas.tests.diff_frames_ops.test_groupby_split_apply_combine", "pyspark.pandas.tests.diff_frames_ops.test_groupby_transform", "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding", "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_adv", "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_count", "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling", "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_adv", "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_count", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_connect = Module( name="pyspark-connect", dependencies=[pyspark_sql, connect], source_file_regexes=[ "python/pyspark/sql/connect", ], python_test_goals=[ # sql doctests "pyspark.sql.connect.catalog", "pyspark.sql.connect.conf", "pyspark.sql.connect.group", "pyspark.sql.connect.session", "pyspark.sql.connect.window", "pyspark.sql.connect.column", "pyspark.sql.connect.merge", "pyspark.sql.connect.readwriter", "pyspark.sql.connect.dataframe", "pyspark.sql.connect.functions.builtin", "pyspark.sql.connect.functions.partitioning", "pyspark.sql.connect.observation", "pyspark.sql.connect.avro.functions", "pyspark.sql.connect.protobuf.functions", "pyspark.sql.connect.streaming.readwriter", "pyspark.sql.connect.streaming.query", "pyspark.sql.connect.tvf", # sql unittests "pyspark.sql.tests.connect.test_connect_plan", "pyspark.sql.tests.connect.test_connect_basic", "pyspark.sql.tests.connect.test_connect_dataframe_property", "pyspark.sql.tests.connect.test_connect_channel", "pyspark.sql.tests.connect.test_connect_error", "pyspark.sql.tests.connect.test_connect_function", "pyspark.sql.tests.connect.test_connect_collection", "pyspark.sql.tests.connect.test_connect_column", "pyspark.sql.tests.connect.test_connect_creation", "pyspark.sql.tests.connect.test_connect_readwriter", "pyspark.sql.tests.connect.test_connect_retry", "pyspark.sql.tests.connect.test_connect_session", "pyspark.sql.tests.connect.test_connect_stat", "pyspark.sql.tests.connect.test_parity_datasources", "pyspark.sql.tests.connect.test_parity_errors", "pyspark.sql.tests.connect.test_parity_catalog", "pyspark.sql.tests.connect.test_parity_conf", "pyspark.sql.tests.connect.test_parity_serde", "pyspark.sql.tests.connect.test_parity_functions", "pyspark.sql.tests.connect.test_parity_group", "pyspark.sql.tests.connect.test_parity_sql", "pyspark.sql.tests.connect.test_parity_dataframe", "pyspark.sql.tests.connect.test_parity_collection", "pyspark.sql.tests.connect.test_parity_creation", "pyspark.sql.tests.connect.test_parity_observation", "pyspark.sql.tests.connect.test_parity_repartition", "pyspark.sql.tests.connect.test_parity_stat", "pyspark.sql.tests.connect.test_parity_subquery", "pyspark.sql.tests.connect.test_parity_types", "pyspark.sql.tests.connect.test_parity_column", "pyspark.sql.tests.connect.test_parity_readwriter", "pyspark.sql.tests.connect.test_parity_udf", "pyspark.sql.tests.connect.test_parity_udf_profiler", "pyspark.sql.tests.connect.test_parity_memory_profiler", "pyspark.sql.tests.connect.test_parity_udtf", "pyspark.sql.tests.connect.test_parity_tvf", "pyspark.sql.tests.connect.test_parity_python_datasource", "pyspark.sql.tests.connect.test_parity_python_streaming_datasource", "pyspark.sql.tests.connect.test_parity_frame_plot", "pyspark.sql.tests.connect.test_parity_frame_plot_plotly", "pyspark.sql.tests.connect.test_utils", "pyspark.sql.tests.connect.client.test_artifact", "pyspark.sql.tests.connect.client.test_artifact_localcluster", "pyspark.sql.tests.connect.client.test_client", "pyspark.sql.tests.connect.client.test_reattach", "pyspark.sql.tests.connect.streaming.test_parity_streaming", "pyspark.sql.tests.connect.streaming.test_parity_listener", "pyspark.sql.tests.connect.streaming.test_parity_foreach", "pyspark.sql.tests.connect.streaming.test_parity_foreach_batch", "pyspark.sql.tests.connect.test_resources", "pyspark.sql.tests.connect.shell.test_progress", "pyspark.sql.tests.connect.test_df_debug", "pyspark.sql.tests.connect.arrow.test_parity_arrow", "pyspark.sql.tests.connect.arrow.test_parity_arrow_map", "pyspark.sql.tests.connect.arrow.test_parity_arrow_grouped_map", "pyspark.sql.tests.connect.arrow.test_parity_arrow_cogrouped_map", "pyspark.sql.tests.connect.arrow.test_parity_arrow_python_udf", "pyspark.sql.tests.connect.pandas.test_parity_pandas_map", "pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map", "pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state", "pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map", "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf", "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_scalar", "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg", "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window", "pyspark.sql.tests.connect.pandas.test_parity_pandas_transform_with_state", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_ml_connect = Module( name="pyspark-ml-connect", dependencies=[pyspark_connect, pyspark_ml], source_file_regexes=[ "python/pyspark/ml/connect", ], python_test_goals=[ # ml doctests "pyspark.ml.connect.functions", # ml unittests "pyspark.ml.tests.connect.test_connect_cache", "pyspark.ml.tests.connect.test_connect_function", "pyspark.ml.tests.connect.test_parity_torch_distributor", "pyspark.ml.tests.connect.test_parity_torch_data_loader", "pyspark.ml.tests.connect.test_connect_summarizer", "pyspark.ml.tests.connect.test_connect_evaluation", "pyspark.ml.tests.connect.test_connect_feature", "pyspark.ml.tests.connect.test_connect_classification", "pyspark.ml.tests.connect.test_connect_pipeline", "pyspark.ml.tests.connect.test_connect_tuning", "pyspark.ml.tests.connect.test_parity_als", "pyspark.ml.tests.connect.test_parity_fpm", "pyspark.ml.tests.connect.test_parity_classification", "pyspark.ml.tests.connect.test_parity_regression", "pyspark.ml.tests.connect.test_parity_clustering", "pyspark.ml.tests.connect.test_parity_evaluation", "pyspark.ml.tests.connect.test_parity_feature", "pyspark.ml.tests.connect.test_parity_functions", "pyspark.ml.tests.connect.test_parity_pipeline", "pyspark.ml.tests.connect.test_parity_tuning", "pyspark.ml.tests.connect.test_parity_ovr", "pyspark.ml.tests.connect.test_parity_stat", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_pandas_connect_part0 = Module( name="pyspark-pandas-connect-part0", dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], source_file_regexes=[ "python/pyspark/pandas", ], python_test_goals=[ # unittests dedicated for Spark Connect "pyspark.pandas.tests.connect.test_connect_plotting", # pandas-on-Spark unittests "pyspark.pandas.tests.connect.test_parity_categorical", "pyspark.pandas.tests.connect.test_parity_config", "pyspark.pandas.tests.connect.test_parity_extension", "pyspark.pandas.tests.connect.test_parity_frame_spark", "pyspark.pandas.tests.connect.test_parity_generic_functions", "pyspark.pandas.tests.connect.test_parity_indexops_spark", "pyspark.pandas.tests.connect.test_parity_internal", "pyspark.pandas.tests.connect.test_parity_namespace", "pyspark.pandas.tests.connect.test_parity_numpy_compat", "pyspark.pandas.tests.connect.test_parity_repr", "pyspark.pandas.tests.connect.test_parity_scalars", "pyspark.pandas.tests.connect.test_parity_spark_functions", "pyspark.pandas.tests.connect.test_parity_sql", "pyspark.pandas.tests.connect.test_parity_typedef", "pyspark.pandas.tests.connect.test_parity_utils", "pyspark.pandas.tests.connect.data_type_ops.test_parity_as_type", "pyspark.pandas.tests.connect.data_type_ops.test_parity_base", "pyspark.pandas.tests.connect.data_type_ops.test_parity_binary_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_boolean_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_categorical_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_complex_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_date_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_datetime_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_null_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_reverse", "pyspark.pandas.tests.connect.data_type_ops.test_parity_string_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_udt_ops", "pyspark.pandas.tests.connect.data_type_ops.test_parity_timedelta_ops", "pyspark.pandas.tests.connect.plot.test_parity_frame_plot", "pyspark.pandas.tests.connect.plot.test_parity_frame_plot_matplotlib", "pyspark.pandas.tests.connect.plot.test_parity_frame_plot_plotly", "pyspark.pandas.tests.connect.plot.test_parity_series_plot", "pyspark.pandas.tests.connect.plot.test_parity_series_plot_matplotlib", "pyspark.pandas.tests.connect.plot.test_parity_series_plot_plotly", "pyspark.pandas.tests.connect.indexes.test_parity_default", "pyspark.pandas.tests.connect.indexes.test_parity_category", "pyspark.pandas.tests.connect.indexes.test_parity_timedelta", "pyspark.pandas.tests.connect.indexes.test_parity_basic", "pyspark.pandas.tests.connect.indexes.test_parity_getattr", "pyspark.pandas.tests.connect.indexes.test_parity_name", "pyspark.pandas.tests.connect.indexes.test_parity_conversion", "pyspark.pandas.tests.connect.indexes.test_parity_drop", "pyspark.pandas.tests.connect.indexes.test_parity_level", "pyspark.pandas.tests.connect.indexes.test_parity_missing", "pyspark.pandas.tests.connect.indexes.test_parity_repeat", "pyspark.pandas.tests.connect.indexes.test_parity_sort", "pyspark.pandas.tests.connect.indexes.test_parity_stat", "pyspark.pandas.tests.connect.indexes.test_parity_symmetric_diff", "pyspark.pandas.tests.connect.indexes.test_parity_take", "pyspark.pandas.tests.connect.indexes.test_parity_unique", "pyspark.pandas.tests.connect.indexes.test_parity_asof", "pyspark.pandas.tests.connect.indexes.test_parity_astype", "pyspark.pandas.tests.connect.indexes.test_parity_delete", "pyspark.pandas.tests.connect.indexes.test_parity_diff", "pyspark.pandas.tests.connect.indexes.test_parity_insert", "pyspark.pandas.tests.connect.indexes.test_parity_map", "pyspark.pandas.tests.connect.indexes.test_parity_align", "pyspark.pandas.tests.connect.indexes.test_parity_indexing", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_adv", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_basic", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_iloc", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_loc", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_loc_2d", "pyspark.pandas.tests.connect.indexes.test_parity_indexing_loc_multi_idx", "pyspark.pandas.tests.connect.indexes.test_parity_reindex", "pyspark.pandas.tests.connect.indexes.test_parity_rename", "pyspark.pandas.tests.connect.indexes.test_parity_reset_index", "pyspark.pandas.tests.connect.indexes.test_parity_datetime", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_at", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_between", "pyspark.pandas.tests.connect.computation.test_parity_any_all", "pyspark.pandas.tests.connect.computation.test_parity_apply_func", "pyspark.pandas.tests.connect.computation.test_parity_binary_ops", "pyspark.pandas.tests.connect.computation.test_parity_combine", "pyspark.pandas.tests.connect.computation.test_parity_compute", "pyspark.pandas.tests.connect.computation.test_parity_cov", "pyspark.pandas.tests.connect.computation.test_parity_corr", "pyspark.pandas.tests.connect.computation.test_parity_corrwith", "pyspark.pandas.tests.connect.computation.test_parity_cumulative", "pyspark.pandas.tests.connect.computation.test_parity_describe", "pyspark.pandas.tests.connect.computation.test_parity_eval", "pyspark.pandas.tests.connect.computation.test_parity_melt", "pyspark.pandas.tests.connect.computation.test_parity_missing_data", "pyspark.pandas.tests.connect.groupby.test_parity_stat", "pyspark.pandas.tests.connect.groupby.test_parity_stat_adv", "pyspark.pandas.tests.connect.groupby.test_parity_stat_ddof", "pyspark.pandas.tests.connect.groupby.test_parity_stat_func", "pyspark.pandas.tests.connect.groupby.test_parity_stat_prod", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_pandas_connect_part1 = Module( name="pyspark-pandas-connect-part1", dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], source_file_regexes=[ "python/pyspark/pandas", ], python_test_goals=[ # pandas-on-Spark unittests "pyspark.pandas.tests.connect.frame.test_parity_attrs", "pyspark.pandas.tests.connect.frame.test_parity_axis", "pyspark.pandas.tests.connect.frame.test_parity_constructor", "pyspark.pandas.tests.connect.frame.test_parity_conversion", "pyspark.pandas.tests.connect.frame.test_parity_reindexing", "pyspark.pandas.tests.connect.frame.test_parity_reshaping", "pyspark.pandas.tests.connect.frame.test_parity_spark", "pyspark.pandas.tests.connect.frame.test_parity_take", "pyspark.pandas.tests.connect.frame.test_parity_take_adv", "pyspark.pandas.tests.connect.frame.test_parity_time_series", "pyspark.pandas.tests.connect.frame.test_parity_truncate", "pyspark.pandas.tests.connect.groupby.test_parity_aggregate", "pyspark.pandas.tests.connect.groupby.test_parity_apply_func", "pyspark.pandas.tests.connect.groupby.test_parity_corr", "pyspark.pandas.tests.connect.groupby.test_parity_cumulative", "pyspark.pandas.tests.connect.groupby.test_parity_missing_data", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_count", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_first", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_last", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_min_max", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_skew", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_std", "pyspark.pandas.tests.connect.groupby.test_parity_split_apply_var", "pyspark.pandas.tests.connect.series.test_parity_datetime", "pyspark.pandas.tests.connect.series.test_parity_string_ops_adv", "pyspark.pandas.tests.connect.series.test_parity_string_ops_basic", "pyspark.pandas.tests.connect.series.test_parity_all_any", "pyspark.pandas.tests.connect.series.test_parity_arg_ops", "pyspark.pandas.tests.connect.series.test_parity_as_of", "pyspark.pandas.tests.connect.series.test_parity_as_type", "pyspark.pandas.tests.connect.series.test_parity_compute", "pyspark.pandas.tests.connect.series.test_parity_conversion", "pyspark.pandas.tests.connect.series.test_parity_cumulative", "pyspark.pandas.tests.connect.series.test_parity_index", "pyspark.pandas.tests.connect.series.test_parity_missing_data", "pyspark.pandas.tests.connect.series.test_parity_series", "pyspark.pandas.tests.connect.series.test_parity_sort", "pyspark.pandas.tests.connect.series.test_parity_stat", "pyspark.pandas.tests.connect.series.test_parity_interpolate", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_arithmetic", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_mod", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_mul_div", "pyspark.pandas.tests.connect.data_type_ops.test_parity_num_pow", "pyspark.pandas.tests.connect.reshape.test_parity_get_dummies", "pyspark.pandas.tests.connect.reshape.test_parity_get_dummies_kwargs", "pyspark.pandas.tests.connect.reshape.test_parity_get_dummies_multiindex", "pyspark.pandas.tests.connect.reshape.test_parity_get_dummies_object", "pyspark.pandas.tests.connect.reshape.test_parity_get_dummies_prefix", "pyspark.pandas.tests.connect.reshape.test_parity_merge_asof", "pyspark.pandas.tests.connect.indexes.test_parity_append", "pyspark.pandas.tests.connect.indexes.test_parity_intersection", "pyspark.pandas.tests.connect.indexes.test_parity_monotonic", "pyspark.pandas.tests.connect.indexes.test_parity_union", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_ceil", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_floor", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_iso", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_map", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_property", "pyspark.pandas.tests.connect.indexes.test_parity_datetime_round", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_shift", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_transform", # fallback "pyspark.pandas.tests.connect.frame.test_parity_asfreq", "pyspark.pandas.tests.connect.frame.test_parity_asof", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_pandas_connect_part2 = Module( name="pyspark-pandas-connect-part2", dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], source_file_regexes=[ "python/pyspark/pandas", ], python_test_goals=[ # pandas-on-Spark unittests "pyspark.pandas.tests.connect.computation.test_parity_pivot", "pyspark.pandas.tests.connect.computation.test_parity_pivot_table", "pyspark.pandas.tests.connect.computation.test_parity_pivot_table_adv", "pyspark.pandas.tests.connect.computation.test_parity_pivot_table_multi_idx", "pyspark.pandas.tests.connect.computation.test_parity_pivot_table_multi_idx_adv", "pyspark.pandas.tests.connect.computation.test_parity_stats", "pyspark.pandas.tests.connect.frame.test_parity_interpolate", "pyspark.pandas.tests.connect.frame.test_parity_interpolate_error", "pyspark.pandas.tests.connect.resample.test_parity_frame", "pyspark.pandas.tests.connect.resample.test_parity_series", "pyspark.pandas.tests.connect.resample.test_parity_error", "pyspark.pandas.tests.connect.resample.test_parity_missing", "pyspark.pandas.tests.connect.resample.test_parity_on", "pyspark.pandas.tests.connect.resample.test_parity_timezone", "pyspark.pandas.tests.connect.window.test_parity_ewm_error", "pyspark.pandas.tests.connect.window.test_parity_ewm_mean", "pyspark.pandas.tests.connect.window.test_parity_groupby_ewm_mean", "pyspark.pandas.tests.connect.window.test_parity_missing", "pyspark.pandas.tests.connect.window.test_parity_rolling", "pyspark.pandas.tests.connect.window.test_parity_rolling_adv", "pyspark.pandas.tests.connect.window.test_parity_rolling_count", "pyspark.pandas.tests.connect.window.test_parity_rolling_error", "pyspark.pandas.tests.connect.window.test_parity_groupby_rolling", "pyspark.pandas.tests.connect.window.test_parity_groupby_rolling_adv", "pyspark.pandas.tests.connect.window.test_parity_groupby_rolling_count", "pyspark.pandas.tests.connect.window.test_parity_expanding", "pyspark.pandas.tests.connect.window.test_parity_expanding_adv", "pyspark.pandas.tests.connect.window.test_parity_expanding_error", "pyspark.pandas.tests.connect.window.test_parity_groupby_expanding", "pyspark.pandas.tests.connect.window.test_parity_groupby_expanding_adv", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_rolling", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_rolling_adv", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_rolling_count", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_dot_frame", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_dot_series", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_error", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_align", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_basic_slow", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_cov", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_corrwith", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_index", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_series", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_setitem_frame", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_setitem_series", "pyspark.pandas.tests.connect.groupby.test_parity_index", "pyspark.pandas.tests.connect.groupby.test_parity_describe", "pyspark.pandas.tests.connect.groupby.test_parity_head_tail", "pyspark.pandas.tests.connect.groupby.test_parity_groupby", "pyspark.pandas.tests.connect.groupby.test_parity_grouping", "pyspark.pandas.tests.connect.groupby.test_parity_missing", "pyspark.pandas.tests.connect.groupby.test_parity_nlargest_nsmallest", "pyspark.pandas.tests.connect.groupby.test_parity_raises", "pyspark.pandas.tests.connect.groupby.test_parity_rank", "pyspark.pandas.tests.connect.groupby.test_parity_size", "pyspark.pandas.tests.connect.groupby.test_parity_value_counts", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_pandas_connect_part3 = Module( name="pyspark-pandas-connect-part3", dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], source_file_regexes=[ "python/pyspark/pandas", ], python_test_goals=[ # pandas-on-Spark unittests "pyspark.pandas.tests.connect.io.test_parity_io", "pyspark.pandas.tests.connect.io.test_parity_csv", "pyspark.pandas.tests.connect.io.test_parity_feather", "pyspark.pandas.tests.connect.io.test_parity_stata", "pyspark.pandas.tests.connect.io.test_parity_dataframe_conversion", "pyspark.pandas.tests.connect.io.test_parity_dataframe_spark_io", "pyspark.pandas.tests.connect.io.test_parity_series_conversion", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_ext", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_ext_float", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain_ext", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_arithmetic_chain_ext_float", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_assign_frame", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_assign_series", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_basic", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_bitwise", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_combine_first", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_compare_series", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_concat_inner", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_concat_outer", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_aggregate", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_apply", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_cumulative", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_diff", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_diff_len", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_fillna", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_filter", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_split_apply_combine", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_expanding", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_expanding_adv", "pyspark.pandas.tests.connect.diff_frames_ops.test_parity_groupby_expanding_count", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and # they aren't available there ], ) pyspark_errors = Module( name="pyspark-errors", dependencies=[], source_file_regexes=[ # SPARK-44544: Force the execution of pyspark_errors when there are any changes # in PySpark, since the Python Packaging Tests is only enabled within this module. # This module is the smallest Python test module, it contains only 1 test file # and normally takes < 2 seconds, so the additional cost is small. "python/", "python/pyspark/errors", ], python_test_goals=[ # unittests "pyspark.errors.tests.test_connect_errors_conversion", "pyspark.errors.tests.test_errors", "pyspark.errors.tests.test_traceback", "pyspark.errors.tests.connect.test_parity_traceback", ], ) pyspark_logger = Module( name="pyspark-logger", dependencies=[], source_file_regexes=["python/pyspark/logger"], python_test_goals=[ # doctests "pyspark.logger.logger", # unittests "pyspark.logger.tests.test_logger", "pyspark.logger.tests.connect.test_parity_logger", ], ) sparkr = Module( name="sparkr", dependencies=[hive, mllib], source_file_regexes=[ "R/", ], should_run_r_tests=True, ) docs = Module( name="docs", dependencies=[], source_file_regexes=[ "docs/", ], ) build = Module( name="build", dependencies=[], source_file_regexes=[ ".*pom.xml", "dev/test-dependencies.sh", ], should_run_build_tests=True, ) yarn = Module( name="yarn", dependencies=[], source_file_regexes=[ "resource-managers/yarn/", "common/network-yarn/", ], build_profile_flags=["-Pyarn"], sbt_test_goals=[ "yarn/test", "network-yarn/test", ], test_tags=["org.apache.spark.tags.ExtendedYarnTest"], ) kubernetes = Module( name="kubernetes", dependencies=[], source_file_regexes=["resource-managers/kubernetes"], build_profile_flags=["-Pkubernetes", "-Pvolcano"], sbt_test_goals=["kubernetes/test"], ) hadoop_cloud = Module( name="hadoop-cloud", dependencies=[], source_file_regexes=["hadoop-cloud"], build_profile_flags=["-Phadoop-cloud"], sbt_test_goals=["hadoop-cloud/test"], ) spark_ganglia_lgpl = Module( name="spark-ganglia-lgpl", dependencies=[], build_profile_flags=["-Pspark-ganglia-lgpl"], source_file_regexes=[ "connector/spark-ganglia-lgpl", ], ) docker_integration_tests = Module( name="docker-integration-tests", dependencies=[sql], build_profile_flags=["-Pdocker-integration-tests"], source_file_regexes=["connector/docker-integration-tests"], sbt_test_goals=["docker-integration-tests/test"], environ=None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}, test_tags=["org.apache.spark.tags.DockerTest"], ) # The root module is a dummy module which is used to run all of the tests. # No other modules should directly depend on this module. root = Module( name="root", dependencies=[build, core], # Changes to build should trigger all tests. source_file_regexes=[], # In order to run all of the tests, enable every test profile: build_profile_flags=list( set(itertools.chain.from_iterable(m.build_profile_flags for m in all_modules)) ), sbt_test_goals=[ "test", ], python_test_goals=list(itertools.chain.from_iterable(m.python_test_goals for m in all_modules)), should_run_r_tests=True, should_run_build_tests=True, )