Merge pull request #135 from Yelp/d/manpreet/limit_spark_app_name

Limiting spark appname to 63 characters

service_configuration_lib/spark_config.py

@@ -1099,6 +1099,9 @@ def get_spark_conf(
  raw_app_id = f'{paasta_service}__{paasta_instance}__{random_postfix}'
  app_id = re.sub(r'[\.,-]', '_', _get_k8s_resource_name_limit_size_with_hash(raw_app_id))
 
+ # Starting Spark 3.4+, spark-app-name label has been added. Limiting to 63 characters
+ app_name = _get_k8s_resource_name_limit_size_with_hash(app_name)
+
  spark_conf.update({
  'spark.app.name': app_name,
  'spark.app.id': app_id,

setup.py

@@ -17,7 +17,7 @@
 
 setup(
  name='service-configuration-lib',
- version='2.18.10',
+ version='2.18.11',
  provides=['service_configuration_lib'],
  description='Start, stop, and inspect Yelp SOA services',
  url='https://github.com/Yelp/service_configuration_lib',

tests/spark_config_test.py

@@ -1144,6 +1144,7 @@ def assert_app_name(
  def verify(output):
  key = 'spark.app.name'
  assert output[key] == expected_output
+ assert len(output[key]) <= 63
  return [key]
  return verify