| spark.pyspark.python | /usr/bin/python3 |
| spark.shuffle.spill.compress | true |
| spark.glue.enable-continuous-cloudwatch-log | true |
| spark.sql.parquet.fs.optimized.committer.optimization-enabled | true |
| spark.hadoop.hive.metastore.client.factory.class | com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory |
| spark.metrics.conf.*.sink.GlueCloudwatch.jobRunId | jr_c666ade2220467120d073f7218e97a9f886256942619da046836105e448571d3 |
| spark.executor.extraJavaOptions | -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED |
| spark.driver.host | 172.36.110.64 |
| spark.serializer.objectStreamReset | 100 |
| spark.glue.enable-continuous-log-filter | true |
| spark.sql.autoBroadcastJoinThreshold | -1 |
| spark.eventLog.enabled | true |
| spark.hadoop.lakeformation.credentials.url | http://localhost:9998/lakeformationcredentials |
| spark.driver.maxResultSize | 10g |
| spark.hadoop.fs.s3.impl | com.amazon.ws.emr.hadoop.fs.EmrFileSystem |
| spark.hadoop.glue.michiganCredentialsProviderProxy | com.amazonaws.services.glue.remote.LakeformationCredentialsProvider |
| spark.glue.JOB_RUN_ID | jr_c666ade2220467120d073f7218e97a9f886256942619da046836105e448571d3 |
| spark.driver.port | 37127 |
| spark.shuffle.service.enabled | false |
| spark.extraListeners | com.amazonaws.services.glueexceptionanalysis.GlueExceptionAnalysisListener |
| spark.hadoop.parquet.enable.summary-metadata | false |
| spark.rdd.compress | True |
| spark.glue.enable-job-insights | true |
| spark.glue.USE_PROXY | false |
| spark.metrics.conf.*.source.jvm.class | org.apache.spark.metrics.source.JvmSource |
| spark.glueAppInsightsLog.dir | /tmp/glue-app-insights-logs/ |
| spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version | 2 |
| spark.sql.crossJoin.enabled | true |
| spark.ui.enabled | false |
| spark.app.name | Section_A |
| spark.hadoop.aws.glue.endpoint | https://glue.us-west-2.amazonaws.com |
| spark.network.timeout | 600 |
| spark.hadoop.fs.s3.buffer.dir | /tmp/hadoop-spark/s3 |
| spark.pyFiles | |
| spark.glueExceptionAnalysisEventLog.dir | /tmp/glue-exception-analysis-logs/ |
| spark.scheduler.mode | FIFO |
| spark.unsafe.sorter.spill.read.ahead.enabled | false |
| spark.glue.additional-python-modules | boto3==1.16.52 |
| spark.rpc.askTimeout | 600 |
| spark.shuffle.mapStatus.compression.codec | lz4 |
| spark.driver.memory | 20g |
| spark.executor.instances | 79 |
| spark.glue.GLUE_VERSION | 4.0 |
| spark.glue.GLUE_TASK_GROUP_ID | 0c0377ce-d08f-488c-8b56-5f7fb7446ae0 |
| spark.hadoop.mapred.output.direct.EmrFileSystem | true |
| spark.default.parallelism | 632 |
| spark.dynamicAllocation.maxExecutors | 79 |
| spark.metrics.conf.*.source.system.class | org.apache.spark.metrics.source.SystemMetricsSource |
| spark.app.startTime | 1683129981241 |
| spark.executor.id | driver |
| spark.glueJobInsights.enabled | true |
| spark.glue.GLUE_COMMAND_CRITERIA | glueetl |
| spark.driver.cores | 8 |
| spark.driver.extraJavaOptions | -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED |
| spark.sql.hive.manageFilesourcePartitions | false |
| spark.sql.sources.partitionOverwriteMode | dynamic |
| spark.cloudwatch.logging.conf.jobRunId | jr_c666ade2220467120d073f7218e97a9f886256942619da046836105e448571d3 |
| spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs | false |
| spark.master | jes |
| spark.metrics.conf.driver.source.aggregate.class | org.apache.spark.metrics.source.AggregateMetricsSource |
| spark.sql.parquet.output.committer.class | com.amazon.emr.committer.EmrOptimizedSparkSqlParquetOutputCommitter |
| spark.authenticate | true |
| spark.files.overwrite | true |
| spark.metrics.conf.*.sink.GlueCloudwatch.jobName | pac-dp-offload-normalized-usage-dev-etl |
| spark.authenticate.secret | *********(redacted) |
| spark.script.location | s3://jagadish-test-flight/scripts/etl/usage_normalized/usage.py |
| spark.executor.memory | 20g |
| spark.driver.extraClassPath | /tmp:/opt/amazon/conf:/opt/amazon/glue-manifest.jar |
| spark.eventLog.dir | /tmp/spark-event-logs/ |
| spark.dynamicAllocation.enabled | false |
| spark.executor.extraClassPath | /tmp:/opt/amazon/conf:/opt/amazon/glue-manifest.jar |
| spark.sql.catalogImplementation | hive |
| spark.glue.endpoint | https://glue-jes.us-west-2.amazonaws.com |
| spark.executor.cores | 10 |
| spark.metrics.conf.*.source.s3.class | org.apache.spark.metrics.source.S3FileSystemSource |
| spark.glue.JOB_NAME | pac-dp-offload-normalized-usage-dev-etl |
| spark.metrics.conf.*.sink.GlueCloudwatch.namespace | Glue |
| spark.cloudwatch.logging.ui.showConsoleProgress | true |
| spark.metrics.conf.*.sink.GlueCloudwatch.class | org.apache.spark.metrics.sink.GlueCloudwatchSink |
| spark.hadoop.mapred.output.committer.class | org.apache.hadoop.mapred.DirectOutputCommitter |
| spark.app.id | spark-application-1683129982102 |
| spark.glue.jobLanguage | python |
| spark.hadoop.mapred.output.direct.NativeS3FileSystem | true |
| spark.sql.shuffle.partitions | 1000 |
