| spark.pyspark.python | /usr/bin/python3 |
| spark.shuffle.spill.compress | true |
| spark.glue.enable-continuous-cloudwatch-log | true |
| spark.sql.parquet.fs.optimized.committer.optimization-enabled | true |
| spark.hadoop.hive.metastore.client.factory.class | com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory |
| spark.metrics.conf.*.sink.GlueCloudwatch.jobRunId | jr_136ac36931df1615aad250a305858b20e9a170a47f0dbaa4aec77ebf1b5132d1 |
| spark.executor.extraJavaOptions | -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -XX:+UseG1GC |
| spark.driver.host | 172.36.83.239 |
| spark.serializer.objectStreamReset | 100 |
| spark.glue.enable-continuous-log-filter | true |
| spark.sql.autoBroadcastJoinThreshold | 1 |
| spark.io.compression.codec | lz4 |
| spark.eventLog.enabled | true |
| spark.hadoop.lakeformation.credentials.url | http://localhost:9998/lakeformationcredentials |
| spark.driver.maxResultSize | 4g |
| spark.hadoop.fs.s3.impl | com.amazon.ws.emr.hadoop.fs.EmrFileSystem |
| spark.hadoop.glue.michiganCredentialsProviderProxy | com.amazonaws.services.glue.remote.LakeformationCredentialsProvider |
| spark.glue.JOB_RUN_ID | jr_136ac36931df1615aad250a305858b20e9a170a47f0dbaa4aec77ebf1b5132d1 |
| spark.driver.port | 39519 |
| spark.shuffle.service.enabled | false |
| spark.extraListeners | com.amazonaws.services.glueexceptionanalysis.GlueExceptionAnalysisListener |
| spark.hadoop.parquet.enable.summary-metadata | false |
| spark.rdd.compress | True |
| spark.glue.enable-job-insights | true |
| spark.glue.USE_PROXY | false |
| spark.shuffle.compress | true |
| spark.metrics.conf.*.source.jvm.class | org.apache.spark.metrics.source.JvmSource |
| spark.glueAppInsightsLog.dir | /tmp/glue-app-insights-logs/ |
| spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version | 2 |
| spark.ui.enabled | false |
| spark.app.name | mpu-usecase4 |
| spark.hadoop.aws.glue.endpoint | https://glue.us-west-2.amazonaws.com |
| spark.network.timeout | 600 |
| spark.hadoop.fs.s3.buffer.dir | /tmp/hadoop-spark/s3 |
| spark.pyFiles | |
| spark.glueExceptionAnalysisEventLog.dir | /tmp/glue-exception-analysis-logs/ |
| spark.scheduler.mode | FIFO |
| spark.unsafe.sorter.spill.read.ahead.enabled | false |
| spark.glue.additional-python-modules | boto3==1.16.52 |
| spark.rpc.askTimeout | 600 |
| spark.shuffle.mapStatus.compression.codec | lz4 |
| spark.driver.memory | 20g |
| spark.executor.instances | 49 |
| spark.glue.GLUE_VERSION | 4.0 |
| spark.glue.GLUE_TASK_GROUP_ID | 43d8d7bf-2a17-48ff-8a20-799e8bb32946 |
| spark.sql.files.maxRecordsPerFile | 30000 |
| spark.hadoop.mapred.output.direct.EmrFileSystem | true |
| spark.default.parallelism | 392 |
| spark.dynamicAllocation.maxExecutors | 49 |
| spark.metrics.conf.*.source.system.class | org.apache.spark.metrics.source.SystemMetricsSource |
| spark.app.startTime | 1683260189309 |
| spark.executor.id | driver |
| spark.glueJobInsights.enabled | true |
| spark.glue.GLUE_COMMAND_CRITERIA | glueetl |
| spark.driver.cores | 8 |
| spark.driver.extraJavaOptions | -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED |
| spark.sql.sources.partitionOverwriteMode | dynamic |
| spark.cloudwatch.logging.conf.jobRunId | jr_136ac36931df1615aad250a305858b20e9a170a47f0dbaa4aec77ebf1b5132d1 |
| spark.sql.broadcastTimeout | 1000 |
| spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs | false |
| spark.master | jes |
| spark.metrics.conf.driver.source.aggregate.class | org.apache.spark.metrics.source.AggregateMetricsSource |
| spark.sql.parquet.output.committer.class | com.amazon.emr.committer.EmrOptimizedSparkSqlParquetOutputCommitter |
| spark.authenticate | true |
| spark.files.overwrite | true |
| spark.metrics.conf.*.sink.GlueCloudwatch.jobName | jag-pac-dp-mpu-usage-usecase4-dev-etl |
| spark.authenticate.secret | *********(redacted) |
| spark.script.location | s3://jagadish-test-flight/pac-dp-paxus-datapip/BuildArtifacts/20230504205932/jag-pac-dp-mpu-usage-usecase4-dev-etl.py |
| spark.executor.memory | 20g |
| spark.driver.extraClassPath | /tmp:/opt/amazon/conf:/opt/amazon/glue-manifest.jar |
| spark.eventLog.dir | /tmp/spark-event-logs/ |
| spark.dynamicAllocation.enabled | false |
| spark.executor.extraClassPath | /tmp:/opt/amazon/conf:/opt/amazon/glue-manifest.jar |
| spark.sql.catalogImplementation | hive |
| spark.glue.endpoint | https://glue-jes.us-west-2.amazonaws.com |
| spark.executor.cores | 6 |
| spark.metrics.conf.*.source.s3.class | org.apache.spark.metrics.source.S3FileSystemSource |
| spark.glue.JOB_NAME | jag-pac-dp-mpu-usage-usecase4-dev-etl |
| spark.metrics.conf.*.sink.GlueCloudwatch.namespace | Glue |
| spark.cloudwatch.logging.ui.showConsoleProgress | true |
| spark.metrics.conf.*.sink.GlueCloudwatch.class | org.apache.spark.metrics.sink.GlueCloudwatchSink |
| spark.hadoop.mapred.output.committer.class | org.apache.hadoop.mapred.DirectOutputCommitter |
| spark.app.id | spark-application-1683260190067 |
| spark.glue.jobLanguage | python |
| spark.hadoop.mapred.output.direct.NativeS3FileSystem | true |
| spark.sql.shuffle.partitions | 600 |
