resource "aws_s3_bucket_object" "job_script" { bucket = var.s3_bucket_name key = "script/${basename(var.script_file_path)}" source = var.script_file_path etag = filemd5(var.script_file_path) } resource "aws_glue_job" "job" { name = var.name role_arn = var.role_arn command { python_version = 3 script_location = "s3://${var.s3_bucket_name}/script/${basename(var.script_file_path)}" } glue_version = "3.0" number_of_workers = 2 worker_type = "Standard" default_arguments = merge ( { "--TempDir" : "s3://${var.s3_bucket_name}/temp_dir", "--conf" : "spark.spline.producer.url=${var.lineage_endpoint} --conf spark.sql.queryExecutionListeners=za.co.absa.spline.harvester.listener.SplineQueryExecutionListener", "--extra-jars" : var.jars, "--job-language" : "python", "--user-jars-first" : "true", "--enable-glue-datacatalog": "" }, var.job_parameters ) }