[spark] Add "Spark Conf" param to Spark session config dropdown
It allows the user to submit key-value pairs of Spark configuration that will be added to the properties file submitted with the session create.
@@ -46,6 +46,16 @@ class SparkConfiguration(object):
APP_NAME = 'spark'
PROPERTIES = [
+ {
+ "name": "conf",
+ "nice_name": _("Spark Conf"),
+ "help_text": _("Add one or more Spark conf properties to the session."),
+ "type": "settings",
+ "is_yarn": False,
+ "multiple": True,
+ "defaultValue": [],
+ "value": [],
+ },
{
"name": "jars",
"nice_name": _("Jars"),
@@ -186,6 +196,17 @@ class SparkApi(Api):
props['pyFiles'] = tmppyFiles.split(",")
LOG.debug("Check List type: pyFiles was not a list")
+ # Convert the conf list to a dict for Livy
+ listitems = props['conf']
+ LOG.debug("Property Spark Conf kvp list from UI is: " + str(listitems))
+ confDict = {}
+ for i in range(len(listitems)):
+ kvp = listitems[i]
+ LOG.debug("Property Spark Conf key " + str(i) + " = " + str(kvp.get('key')))
+ LOG.debug("Property Spark Conf value " + str(i) + " = " + str(kvp.get('value')))
+ confDict[kvp.get('key')] = kvp.get('value')
+ props['conf'] = confDict
+ LOG.debug("Property Spark Conf dictionary is: " + str(confDict))
props['kind'] = lang
