diff --git a/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml b/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml index e8d9cdd..65070a4 100755 --- a/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml +++ b/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml @@ -200,9 +200,13 @@ Resources: - s3:GetBucketLocation - s3:PutBucketAcl - s3:GetBucketAcl + Resource: '*' + - Effect: Allow + Action: - s3:GetBucketLogging - s3:PutBucketLogging - Resource: '*' + Resource: + - !Sub arn:${AWS::Partition}:s3:::${BucketName} EnableNewAWSResourcesLambda: Type: 'AWS::Serverless::Function' diff --git a/cloudwatchevents/guardduty/cloudwatchevents.json b/cloudwatchevents/guardduty/cloudwatchevents.json index d8d78b3..3fb8b5f 100644 --- a/cloudwatchevents/guardduty/cloudwatchevents.json +++ b/cloudwatchevents/guardduty/cloudwatchevents.json @@ -25,6 +25,8 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, @@ -32,7 +34,16 @@ "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"} + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, + "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, + "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, + "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, + "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, + "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json index a935406..15ce53e 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json @@ -45,22 +45,25 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { @@ -228,7 +231,7 @@ } }, "Handler": "cloudwatchlogs_lambda.handler", - "Runtime": "nodejs22.x", + "Runtime": "nodejs24.x", "MemorySize": 128, "Environment": { "Variables": { @@ -290,7 +293,7 @@ ] } }, - "Runtime": "nodejs22.x", + "Runtime": "nodejs24.x", "MemorySize": 128, "Environment": { "Variables": { diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json index 4200d9b..83ff96c 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json @@ -45,22 +45,25 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { @@ -256,7 +259,7 @@ } }, "Handler": "cloudwatchlogs_lambda.handler", - "Runtime": "nodejs22.x", + "Runtime": "nodejs24.x", "MemorySize": 128, "Environment": { "Variables": { @@ -317,7 +320,7 @@ ] } }, - "Runtime": "nodejs22.x", + "Runtime": "nodejs24.x", "MemorySize": 128, "Environment": { "Variables": { diff --git a/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py b/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py index be0744d..5ebd11f 100644 --- a/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py +++ b/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py @@ -14,6 +14,8 @@ "ap-northeast-2": "appdevzipfiles-ap-northeast-2", "ap-southeast-1": "appdevzipfiles-ap-southeast-1", "ap-southeast-2": "appdevzipfiles-ap-southeast-2", + "ap-southeast-4": "appdevzipfiles-ap-southeast-4s", + "ap-southeast-6": "appdevzipfiles-ap-southeast-6ss", "ap-northeast-1": "appdevzipfiles-ap-northeast-1", "ca-central-1": "appdevzipfiles-ca-central-1", "eu-central-1": "appdevzipfiles-eu-central-1", @@ -29,7 +31,8 @@ "me-central-1": "appdevzipfiles-me-central-1", "eu-central-2": "appdevzipfiles-eu-central-2ss", "ap-northeast-3": "appdevzipfiles-ap-northeast-3s", - "ap-southeast-3": "appdevzipfiles-ap-southeast-3" + "ap-southeast-3": "appdevzipfiles-ap-southeast-3", + "il-central-1": "appdevzipfiles-il-central-1" } def get_bucket_name(region): diff --git a/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml new file mode 100644 index 0000000..80c9692 --- /dev/null +++ b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml @@ -0,0 +1,359 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Sumo Logic CloudWatch log collector +Parameters: + SumoEndPointURL: + Type: String + Default: + Description: Enter SUMO_ENDPOINT created while configuring HTTP Source + EmailID: + Type: String + Default: test@gmail.com + Description: Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts. + NumOfWorkers: + Type: Number + Default: 4 + Description: Enter the number of lambda function invocations for faster Dead Letter Queue processing. + LogFormat: + Type: String + Default: Others + AllowedValues: + - VPC-RAW + - VPC-JSON + - Others + Description: Choose the Service + IncludeLogGroupInfo: + Type: String + Default: 'false' + AllowedValues: + - 'true' + - 'false' + Description: Select true to get loggroup/logstream values in logs + LogStreamPrefix: + Type: String + Description: (Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value. + Default: '' +Mappings: + RegionMap: + us-east-1: + bucketname: appdevzipfiles-us-east-1 + us-east-2: + bucketname: appdevzipfiles-us-east-2 + us-west-1: + bucketname: appdevzipfiles-us-west-1 + us-west-2: + bucketname: appdevzipfiles-us-west-2 + ap-south-1: + bucketname: appdevzipfiles-ap-south-1 + ap-northeast-2: + bucketname: appdevzipfiles-ap-northeast-2 + ap-southeast-1: + bucketname: appdevzipfiles-ap-southeast-1 + ap-southeast-2: + bucketname: appdevzipfiles-ap-southeast-2 + ap-southeast-4: + bucketname: appdevzipfiles-ap-southeast-4s + ap-southeast-6: + bucketname: appdevzipfiles-ap-southeast-6ss + ap-northeast-1: + bucketname: appdevzipfiles-ap-northeast-1 + ca-central-1: + bucketname: appdevzipfiles-ca-central-1 + eu-central-1: + bucketname: appdevzipfiles-eu-central-1 + eu-west-1: + bucketname: appdevzipfiles-eu-west-1 + eu-west-2: + bucketname: appdevzipfiles-eu-west-2 + eu-west-3: + bucketname: appdevzipfiles-eu-west-3 + eu-north-1: + bucketname: appdevzipfiles-eu-north-1s + sa-east-1: + bucketname: appdevzipfiles-sa-east-1 + ap-east-1: + bucketname: appdevzipfiles-ap-east-1s + af-south-1: + bucketname: appdevzipfiles-af-south-1s + eu-south-1: + bucketname: appdevzipfiles-eu-south-1 + me-south-1: + bucketname: appdevzipfiles-me-south-1s + me-central-1: + bucketname: appdevzipfiles-me-central-1 + eu-central-2: + bucketname: appdevzipfiles-eu-central-2ss + ap-northeast-3: + bucketname: appdevzipfiles-ap-northeast-3s + ap-southeast-3: + bucketname: appdevzipfiles-ap-southeast-3 + il-central-1: + bucketname: appdevzipfiles-il-central-1 +Resources: + SumoCWLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Join + - '-' + - - SumoCWLogGroup + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + RetentionInDays: 7 + SumoCWLogSubsriptionFilter: + Type: AWS::Logs::SubscriptionFilter + Properties: + LogGroupName: !Ref SumoCWLogGroup + DestinationArn: !GetAtt SumoCWLogsLambda.Arn + FilterPattern: '' + DependsOn: + - SumoCWLogGroup + - SumoCWLambdaPermission + - SumoCWLogsLambda + SumoCWLambdaPermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !GetAtt SumoCWLogsLambda.Arn + Action: lambda:InvokeFunction + Principal: !Join + - . + - - logs + - !Ref AWS::Region + - amazonaws.com + SourceAccount: !Ref AWS::AccountId + SumoCWDeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Join + - '-' + - - SumoCWDeadLetterQueue + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + SumoCWLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Policies: + - PolicyName: !Join + - '-' + - - SQSCreateLogsRolePolicy + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - sqs:DeleteMessage + - sqs:GetQueueUrl + - sqs:ListQueues + - sqs:ChangeMessageVisibility + - sqs:SendMessageBatch + - sqs:ReceiveMessage + - sqs:SendMessage + - sqs:GetQueueAttributes + - sqs:ListQueueTags + - sqs:ListDeadLetterSourceQueues + - sqs:DeleteMessageBatch + - sqs:PurgeQueue + - sqs:DeleteQueue + - sqs:CreateQueue + - sqs:ChangeMessageVisibilityBatch + - sqs:SetQueueAttributes + Resource: + - !GetAtt SumoCWDeadLetterQueue.Arn + - PolicyName: !Join + - '-' + - - CloudWatchCreateLogsRolePolicy + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DescribeLogStreams + Resource: + - !Join + - ':' + - - arn + - !Ref AWS::Partition + - logs + - !Ref AWS::Region + - !Ref AWS::AccountId + - log-group + - '*' + - PolicyName: InvokeLambdaRolePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:InvokeFunction + Resource: + - !Join + - ':' + - - arn + - !Ref AWS::Partition + - lambda + - !Ref AWS::Region + - !Ref AWS::AccountId + - function + - !Join + - '-' + - - SumoCWProcessDLQLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + SumoCWLogsLambda: + Type: AWS::Lambda::Function + DependsOn: + - SumoCWLambdaExecutionRole + - SumoCWDeadLetterQueue + Properties: + FunctionName: !Join + - '-' + - - SumoCWLogsLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + Code: + S3Bucket: !FindInMap + - RegionMap + - !Ref AWS::Region + - bucketname + S3Key: sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip + Role: !GetAtt SumoCWLambdaExecutionRole.Arn + Timeout: 300 + DeadLetterConfig: + TargetArn: !GetAtt SumoCWDeadLetterQueue.Arn + Handler: cloudwatchlogs_lambda.handler + Runtime: nodejs22.x + MemorySize: 128 + Environment: + Variables: + SUMO_ENDPOINT: !Ref SumoEndPointURL + LOG_FORMAT: !Ref LogFormat + INCLUDE_LOG_INFO: !Ref IncludeLogGroupInfo + LOG_STREAM_PREFIX: !Ref LogStreamPrefix + SumoCWEventsInvokeLambdaPermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref SumoCWProcessDLQLambda + Action: lambda:InvokeFunction + Principal: events.amazonaws.com + SourceArn: !GetAtt SumoCWProcessDLQScheduleRule.Arn + SumoCWProcessDLQScheduleRule: + Type: AWS::Events::Rule + Properties: + Description: Events rule for Cron + ScheduleExpression: rate(5 minutes) + State: ENABLED + Targets: + - Arn: !GetAtt SumoCWProcessDLQLambda.Arn + Id: TargetFunctionV1 + SumoCWProcessDLQLambda: + Type: AWS::Lambda::Function + DependsOn: + - SumoCWLambdaExecutionRole + - SumoCWDeadLetterQueue + Properties: + FunctionName: !Join + - '-' + - - SumoCWProcessDLQLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + Code: + S3Bucket: !FindInMap + - RegionMap + - !Ref AWS::Region + - bucketname + S3Key: sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip + Role: !GetAtt SumoCWLambdaExecutionRole.Arn + Timeout: 300 + Handler: DLQProcessor.handler + DeadLetterConfig: + TargetArn: !GetAtt SumoCWDeadLetterQueue.Arn + Runtime: nodejs22.x + MemorySize: 128 + Environment: + Variables: + SUMO_ENDPOINT: !Ref SumoEndPointURL + TASK_QUEUE_URL: !Join + - '' + - - https://sqs. + - !Ref AWS::Region + - .amazonaws.com/ + - !Ref AWS::AccountId + - / + - !GetAtt SumoCWDeadLetterQueue.QueueName + NUM_OF_WORKERS: !Ref NumOfWorkers + LOG_FORMAT: !Ref LogFormat + INCLUDE_LOG_INFO: !Ref IncludeLogGroupInfo + LOG_STREAM_PREFIX: !Ref LogStreamPrefix + SumoCWEmailSNSTopic: + Type: AWS::SNS::Topic + Properties: + Subscription: + - Endpoint: !Ref EmailID + Protocol: email + SumoCWSpilloverAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmActions: + - !Ref SumoCWEmailSNSTopic + AlarmDescription: Notify via email if number of messages in DeadLetterQueue exceeds threshold + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: QueueName + Value: !GetAtt SumoCWDeadLetterQueue.QueueName + EvaluationPeriods: '1' + MetricName: ApproximateNumberOfMessagesVisible + Namespace: AWS/SQS + Period: '3600' + Statistic: Sum + Threshold: '100000' + DependsOn: + - SumoCWEmailSNSTopic +Outputs: + SumoCWLogsLambdaArn: + Description: The ARN of the sumologic cloudwatch logs lambda + Value: !GetAtt SumoCWLogsLambda.Arn + Export: + Name: !Join + - '-' + - - SumoCWLogsLambdaArn + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId diff --git a/sumologic-app-utils/build.sh b/sumologic-app-utils/build.sh index 965629f..74f613c 100755 --- a/sumologic-app-utils/build.sh +++ b/sumologic-app-utils/build.sh @@ -16,10 +16,10 @@ docker exec -it sumologic-app-utils /bin/bash -c "python3 -m venv temp-venv && s docker cp src/. sumologic-app-utils:/var/task/sumo_app_utils # Zip the contents of the sumologic-app-utils directory -docker exec -it sumologic-app-utils /bin/bash -c "cd sumo_app_utils && ls -l && zip -r ../sumo_app_utils.zip ." +docker exec -it sumologic-app-utils /bin/bash -c "cd sumo_app_utils && ls -l && zip -r ../sumo-app-utils.zip ." # Copy the sumologic-app-utils.zip file from the container to the host -docker cp sumologic-app-utils://var/task/sumo_app_utils.zip ./sumo_app_utils.zip +docker cp sumologic-app-utils://var/task/sumo-app-utils.zip ./sumo-app-utils.zip # Stop and remove the container docker stop sumologic-app-utils diff --git a/sumologic-app-utils/src/awsresource.py b/sumologic-app-utils/src/awsresource.py index cf50b37..d6f9f68 100644 --- a/sumologic-app-utils/src/awsresource.py +++ b/sumologic-app-utils/src/awsresource.py @@ -4,12 +4,44 @@ import time from abc import abstractmethod +import traceback import boto3 import six from botocore.exceptions import ClientError from resourcefactory import AutoRegisterResource from retrying import retry +# https://docs.aws.amazon.com/elasticloadbalancing/latest/application/enable-access-logging.html +# Elastic Load Balancing required region-specific account IDs in IAM policies, but this has been replaced by a newer, +# simplified policy. The legacy policy is still supported for older regions, with a reference list of account IDs provided. +Region2ELBAccountId = { + "us-east-1": {"AccountId": "127311923021"}, + "us-east-2": {"AccountId": "033677994240"}, + "us-west-1": {"AccountId": "027434742980"}, + "us-west-2": {"AccountId": "797873946194"}, + "af-south-1": {"AccountId": "098369216593"}, + "ca-central-1": {"AccountId": "985666609251"}, + "eu-central-1": {"AccountId": "054676820928"}, + "eu-west-1": {"AccountId": "156460612806"}, + "eu-west-2": {"AccountId": "652711504416"}, + "eu-south-1": {"AccountId": "635631232127"}, + "eu-west-3": {"AccountId": "009996457667"}, + "eu-north-1": {"AccountId": "897822967062"}, + "ap-east-1": {"AccountId": "754344448648"}, + "ap-northeast-1": {"AccountId": "582318560864"}, + "ap-northeast-2": {"AccountId": "600734575887"}, + "ap-northeast-3": {"AccountId": "383597477331"}, + "ap-southeast-1": {"AccountId": "114774131450"}, + "ap-southeast-2": {"AccountId": "783225319266"}, + "ap-south-1": {"AccountId": "718504428378"}, + "me-south-1": {"AccountId": "076674570225"}, + "sa-east-1": {"AccountId": "507241528517"}, + "us-gov-west-1": {"AccountId": "048591011584"}, + "us-gov-east-1": {"AccountId": "190560391635"}, + "cn-north-1": {"AccountId": "638102146993"}, + "cn-northwest-1": {"AccountId": "037604701340"} +} + @six.add_metaclass(AutoRegisterResource) class AWSResource(object): @@ -42,14 +74,14 @@ def __init__(self, props, *args, **kwargs): def create(self, trail_name, params, *args, **kwargs): try: response = self.cloudtrailcli.create_trail(**params) - print("Trail created %s" % trail_name) + print(f"Trail created {trail_name}") self.cloudtrailcli.start_logging(Name=trail_name) return {"TrailArn": response["TrailARN"]}, response["TrailARN"] except ClientError as e: - print("Error in creating trail %s" % e.response['Error']) + print(f"Error in creating trail {e.response['Error']}") raise except Exception as e: - print("Error in creating trail %s" % e) + print(f"Error in creating trail {e}") raise def update(self, old_trail_name, trail_name, params, *args, **kwargs): @@ -59,14 +91,14 @@ def update(self, old_trail_name, trail_name, params, *args, **kwargs): return self.create(trail_name, params) else: response = self.cloudtrailcli.update_trail(**params) - print("Trail updated %s" % trail_name) + print(f"Trail updated {trail_name}") self.cloudtrailcli.start_logging(Name=trail_name) return {"TrailArn": response["TrailARN"]}, response["TrailARN"] except ClientError as e: - print("Error in updating trail %s" % e.response['Error']) + print(f"Error in updating trail {e.response['Error']}") raise except Exception as e: - print("Error in updating trail %s" % e) + print(f"Error in updating trail {e}") raise def delete(self, trail_name, *args, **kwargs): @@ -74,12 +106,12 @@ def delete(self, trail_name, *args, **kwargs): self.cloudtrailcli.delete_trail( Name=trail_name ) - print("Trail deleted %s" % trail_name) + print(f"Trail deleted {trail_name}") except ClientError as e: - print("Error in deleting trail %s" % e.response['Error']) + print(f"Error in deleting trail {e.response['Error']}") raise except Exception as e: - print("Error in deleting trail %s" % e) + print(f"Error in deleting trail {e}") raise def _transform_bool_values(self, k, v): @@ -111,7 +143,7 @@ def extract_params(self, event): class TagAWSResources(AWSResource): def __init__(self, props, *args, **kwargs): - print('Tagging aws resource %s' % props.get("AWSResource")) + print(f'Tagging aws resource {props.get("AWSResource")}') def _tag_aws_resources(self, region_value, aws_resource, tags, account_id, delete_flag, filter_regex): # Get the class instance based on AWS Resource @@ -132,7 +164,7 @@ def _tag_aws_resources(self, region_value, aws_resource, tags, account_id, delet tag_resource.add_tags(arns, tags) def create(self, region_value, aws_resource, tags, account_id, filter_regex, *args, **kwargs): - print("TAG AWS RESOURCES - Starting the AWS resources Tag addition with Tags %s." % tags) + print(f"TAG AWS RESOURCES - Starting the AWS resources Tag addition with Tags {tags}.") regions = [region_value] for region in regions: self._tag_aws_resources(region, aws_resource, tags, account_id, False, filter_regex) @@ -156,12 +188,12 @@ def update(self, old_properties, region_value, aws_resource, tags, account_id, f self.delete(old_properties['Region'], old_properties['AWSResource'], old_tags, account_id, old_properties['Filter'], remove_on_delete_stack=True) - print("TAG AWS RESOURCES - Starting the AWS resources Tag update with Tags %s." % tags) + print(f"TAG AWS RESOURCES - Starting the AWS resources Tag update with Tags {tags}.") regions = [region_value] for region in regions: self._tag_aws_resources(region, aws_resource, tags, account_id, False, filter_regex) - print("updated tags for aws resource %s " % aws_resource) + print(f"updated tags for aws resource {aws_resource} ") return {"TAG_UPDATE": "Successful"}, aws_resource def delete(self, region_value, aws_resource, tags, account_id, filter_regex, remove_on_delete_stack, *args, @@ -169,7 +201,7 @@ def delete(self, region_value, aws_resource, tags, account_id, filter_regex, rem tags_list = [] if tags: tags_list = list(tags.keys()) - print("TAG AWS RESOURCES - Starting the AWS resources Tag deletion with Tags %s." % tags_list) + print(f"TAG AWS RESOURCES - Starting the AWS resources Tag deletion with Tags {tags_list}.") if remove_on_delete_stack: regions = [region_value] for region in regions: @@ -202,17 +234,17 @@ def extract_params(self, event): class EnableS3LogsResources(AWSResource): def __init__(self, props, *args, **kwargs): - print('Enabling S3 for ALB/ELB-classic aws resource %s' % props.get("AWSResource")) + print(f'Enabling S3 for ALB/ELB-classic aws resource {props.get("AWSResource")}') def _s3_logs_alb_resources(self, region_value, aws_resource, bucket_name, bucket_prefix, - delete_flag, filter_regex, region_account_id, account_id): + delete_flag, filter_regex, account_id): # Get the class instance based on AWS Resource tag_resource = AWSResourcesProvider.get_provider(aws_resource, region_value, account_id) # Fetch and Filter the Resources. resources = tag_resource.fetch_resources() - if(not aws_resource == 'elb'): + if aws_resource != 'elb': filtered_resources = tag_resource.filter_resources(filter_regex, resources) else: filtered_resources = resources @@ -224,41 +256,43 @@ def _s3_logs_alb_resources(self, region_value, aws_resource, bucket_name, bucket if delete_flag: tag_resource.disable_s3_logs(arns, bucket_name) else: - tag_resource.enable_s3_logs(arns, bucket_name, bucket_prefix, region_account_id) + tag_resource.enable_s3_logs(arns, bucket_name, bucket_prefix) - def create(self, region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, region_account_id, + def create(self, region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, account_id, *args, **kwargs): - print("ENABLE S3 LOGS - Starting the AWS resources S3 addition to bucket %s." % bucket_name) + print(f"ENABLE S3 LOGS - Starting the AWS resources S3 addition to bucket {bucket_name}.") self._s3_logs_alb_resources(region_value, aws_resource, bucket_name, bucket_prefix, - False, filter_regex, region_account_id, account_id) + False, filter_regex, account_id) print("ENABLE S3 LOGS - Completed the AWS resources S3 addition to bucket.") return {"S3_ENABLE": "Successful"}, aws_resource - def update(self, old_properties, region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, - region_account_id, account_id, *args, **kwargs): + def update(self, old_properties, region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, account_id, *args, **kwargs): # First Delete Old Tags from old aws resource with old filter regex and Then add new Tags. # Check if aws resource is changed, then raise exception. - if old_properties['AWSResource'] != aws_resource: - data, aws_resource = self.create(region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, - region_account_id, account_id) - else: - # If bucket name or prefix are not same, delete the old logging. - if old_properties['BucketName'] != bucket_name or old_properties['BucketPrefix'] != bucket_prefix: - self.delete(region_value, aws_resource, old_properties['BucketName'], old_properties['BucketPrefix'], - old_properties['Filter'], True, account_id) - - print("ENABLE S3 LOGS - Starting the AWS resources S3 Update with bucket %s." % bucket_name) - self._s3_logs_alb_resources(region_value, aws_resource, bucket_name, bucket_prefix, - False, filter_regex, region_account_id, account_id) - print("ENABLE S3 LOGS - Completed the AWS resources S3 Update for bucket.") - return {"S3_ENABLE": "Successful"}, aws_resource + try: + if old_properties['AWSResource'] != aws_resource: + data, aws_resource = self.create(region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, account_id) + else: + # If bucket name or prefix are not same, delete the old logging. + if old_properties['BucketName'] != bucket_name or old_properties['BucketPrefix'] != bucket_prefix: + self.delete(region_value, aws_resource, old_properties['BucketName'], old_properties['BucketPrefix'], + old_properties['Filter'], True, account_id) + + print(f"ENABLE S3 LOGS - Starting the AWS resources S3 Update with bucket {bucket_name}.") + self._s3_logs_alb_resources(region_value, aws_resource, bucket_name, bucket_prefix, + False, filter_regex, account_id) + print("ENABLE S3 LOGS - Completed the AWS resources S3 Update for bucket.") + return {"S3_ENABLE": "Successful"}, aws_resource + except Exception as e: + traceback.print_exc() + raise def delete(self, region_value, aws_resource, bucket_name, bucket_prefix, filter_regex, remove_on_delete_stack, account_id, *args, **kwargs): if remove_on_delete_stack: self._s3_logs_alb_resources(region_value, aws_resource, bucket_name, bucket_prefix, True, - filter_regex, "", account_id) + filter_regex, account_id) print("ENABLE S3 LOGS - Completed the AWS resources S3 deletion to bucket.") else: print("ENABLE S3 LOGS - Skipping the AWS resources S3 deletion to bucket.") @@ -275,7 +309,6 @@ def extract_params(self, event): "bucket_name": props.get("BucketName"), "bucket_prefix": props.get("BucketPrefix"), "filter_regex": props.get("Filter"), - "region_account_id": props.get("RegionAccountId"), "remove_on_delete_stack": props.get("RemoveOnDeleteStack"), "account_id": props.get("AccountID"), "old_properties": old_properties, @@ -313,7 +346,7 @@ def create_delivery_channel(self, delivery_frequency, bucket_name, bucket_prefix return name def create(self, delivery_frequency, bucket_name, bucket_prefix, sns_topic_arn, *args, **kwargs): - print("DELIVERY CHANNEL - Starting the AWS config Delivery channel create with bucket %s." % bucket_name) + print(f"DELIVERY CHANNEL - Starting the AWS config Delivery channel create with bucket {bucket_name}.") name = self.create_delivery_channel(delivery_frequency, bucket_name, bucket_prefix, sns_topic_arn) @@ -322,7 +355,7 @@ def create(self, delivery_frequency, bucket_name, bucket_prefix, sns_topic_arn, return {"DELIVERY_CHANNEL": "Successful"}, name def update(self, delivery_frequency, bucket_name, bucket_prefix, sns_topic_arn, *args, **kwargs): - print("updated delivery channel to %s " % bucket_name) + print(f"updated delivery channel to {bucket_name} ") name = self.create_delivery_channel(delivery_frequency, bucket_name, bucket_prefix, sns_topic_arn) return {"DELIVERY_CHANNEL": "Successful"}, name @@ -392,7 +425,6 @@ def enable_s3_logs(event, context): bucket_prefix = os.environ.get("BucketPrefix") account_id = os.environ.get("AccountID") filter_regex = os.environ.get("Filter") - region_account_id = os.environ.get("RegionAccountId") is_elbClassic = False if "detail" in event: event_detail = event.get("detail") @@ -415,13 +447,13 @@ def enable_s3_logs(event, context): resources = alb_resource.get_arn_list_cloud_trail_event(event_detail) # Enable S3 logging - alb_resource.enable_s3_logs(resources, bucket_name, bucket_prefix, region_account_id) + alb_resource.enable_s3_logs(resources, bucket_name, bucket_prefix) else: elb_resource = AWSResourcesProvider.get_provider(event_name, region_value, account_id) event_detail = elb_resource.filter_resources(filter_regex, event_detail) if event_detail: resources = elb_resource.get_arn_list_cloud_trail_event(event_detail) - elb_resource.enable_s3_logs(resources, bucket_name, bucket_prefix, region_account_id) + elb_resource.enable_s3_logs(resources, bucket_name, bucket_prefix) print("AWS S3 ENABLE ALB :- Completed s3 logs enable") @@ -846,8 +878,108 @@ def tag_resources_cloud_trail_event(self, arns, tags): tags.extend(tags_arn) self.client.add_tags_to_resource(ResourceName=arn, Tags=tags) +class LbResources(AWSResourcesAbstract): -class AlbResources(AWSResourcesAbstract): + def add_bucket_policy(self, bucket_name, elb_region_account_id): + print("Adding policy to the bucket " + bucket_name) + s3 = boto3.client('s3') + try: + response = s3.get_bucket_policy(Bucket=bucket_name) + existing_policy = json.loads(response["Policy"]) + except ClientError as e: + if "Error" in e.response and "Code" in e.response["Error"] \ + and e.response['Error']['Code'] == "NoSuchBucketPolicy": + existing_policy = { + "Version": "2012-10-17", + "Statement": [ + ] + } + else: + raise e + + bucket_policy = [ + { + "Sid": "AWSCloudTrailAclCheck", + "Effect": "Allow", + "Principal": { + "Service": "cloudtrail.amazonaws.com" + }, + "Action": "s3:GetBucketAcl", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}" + }, + { + "Sid": "AWSCloudTrailWrite", + "Effect": "Allow", + "Principal": { + "Service": "cloudtrail.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*", + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + }, + { + "Sid": "AWSBucketExistenceCheck", + "Effect": "Allow", + "Principal": { + "Service": "cloudtrail.amazonaws.com" + }, + "Action": "s3:ListBucket", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}" + }, + { + "Sid": "AWSLogDeliveryAclCheck", + "Effect": "Allow", + "Principal": { + "Service": "delivery.logs.amazonaws.com" + }, + "Action": "s3:GetBucketAcl", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}" + }, + { + "Sid": "AWSLogDeliveryWrite", + "Effect": "Allow", + "Principal": { + "Service": "delivery.logs.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*", + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + }] + if elb_region_account_id: + elb_old_region_policy = { + "Sid": "AwsElbLogs", + "Effect": "Allow", + "Principal": { + "AWS": f"arn:{self.partition}:iam::{elb_region_account_id}:root" + }, + "Action": ["s3:PutObject"], + "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*" + } + bucket_policy.append(elb_old_region_policy) + else: + elb_new_region_policy = { + "Sid": "AwsElbLogs", + "Effect": "Allow", + "Principal": { + "Service": "logdelivery.elasticloadbalancing.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*" + } + bucket_policy.append(elb_new_region_policy) + existing_policy["Statement"].extend(bucket_policy) + + s3.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(existing_policy)) + +class AlbResources(LbResources): def fetch_resources(self): resources = [] @@ -896,7 +1028,7 @@ def get_arn_list_cloud_trail_event(self, event_detail): def tag_resources_cloud_trail_event(self, arns, tags): self.client.add_tags(ResourceArns=arns, Tags=tags) - def enable_s3_logs(self, arns, s3_bucket, s3_prefix, elb_region_account_id): + def enable_s3_logs(self, arns, s3_bucket, s3_prefix): attributes = [{'Key': 'access_logs.s3.enabled', 'Value': 'true'}, {'Key': 'access_logs.s3.bucket', 'Value': s3_bucket}, {'Key': 'access_logs.s3.prefix', 'Value': s3_prefix}] @@ -913,67 +1045,16 @@ def enable_s3_logs(self, arns, s3_bucket, s3_prefix, elb_region_account_id): except ClientError as e: if "Error" in e.response and "Message" in e.response["Error"] \ and "Access Denied for bucket" in e.response['Error']['Message']: + elb_region = Region2ELBAccountId.get(self.region_value, None) + elb_region_account_id = None + if elb_region: + elb_region_account_id = elb_region.get("AccountId") self.add_bucket_policy(s3_bucket, elb_region_account_id) time.sleep(10) self.client.modify_load_balancer_attributes(LoadBalancerArn=arn, Attributes=attributes) else: raise e - def add_bucket_policy(self, bucket_name, elb_region_account_id): - print("Adding policy to the bucket " + bucket_name) - s3 = boto3.client('s3') - try: - response = s3.get_bucket_policy(Bucket=bucket_name) - existing_policy = json.loads(response["Policy"]) - except ClientError as e: - if "Error" in e.response and "Code" in e.response["Error"] \ - and e.response['Error']['Code'] == "NoSuchBucketPolicy": - existing_policy = { - "Version": "2012-10-17", - "Statement": [ - ] - } - else: - raise e - - bucket_policy = [ - { - "Sid": "AwsAlbLogs", - "Effect": "Allow", - "Principal": { - "AWS": f"arn:{self.partition}:iam::{elb_region_account_id}:root" - }, - "Action": ["s3:PutObject"], - "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*" - }, - { - "Sid": "AWSLogDeliveryAclCheck", - "Effect": "Allow", - "Principal": { - "Service": "delivery.logs.amazonaws.com" - }, - "Action": "s3:GetBucketAcl", - "Resource": f"arn:{self.partition}:s3:::{bucket_name}" - }, - { - "Sid": "AWSLogDeliveryWrite", - "Effect": "Allow", - "Principal": { - "Service": "delivery.logs.amazonaws.com" - }, - "Action": "s3:PutObject", - "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*", - "Condition": { - "StringEquals": { - "s3:x-amz-acl": "bucket-owner-full-control" - } - } - } - ] - existing_policy["Statement"].extend(bucket_policy) - - s3.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(existing_policy)) - def disable_s3_logs(self, arns, s3_bucket): attributes = [{'Key': 'access_logs.s3.enabled', 'Value': 'false'}] @@ -985,7 +1066,6 @@ def disable_s3_logs(self, arns, s3_bucket): self.client.modify_load_balancer_attributes(LoadBalancerArn=arn, Attributes=attributes) time.sleep(1) - class S3Resource(AWSResourcesAbstract): def fetch_resources(self): @@ -1023,7 +1103,7 @@ def get_arn_list_cloud_trail_event(self, event_detail): def tag_resources_cloud_trail_event(self, *args): pass - def enable_s3_logs(self, arns, s3_bucket, s3_prefix, region_account_id): + def enable_s3_logs(self, arns, s3_bucket, s3_prefix): bucket_logging = {'LoggingEnabled': {'TargetBucket': s3_bucket, 'TargetPrefix': s3_prefix}} @@ -1104,7 +1184,7 @@ def get_arn_list_cloud_trail_event(self, event_detail): def tag_resources_cloud_trail_event(self, *args): pass - def enable_s3_logs(self, arns, s3_bucket, s3_prefix, region_account_id): + def enable_s3_logs(self, arns, s3_bucket, s3_prefix): if arns: chunk_records = self._batch_size_chunk(arns, 1000) for record in chunk_records: @@ -1184,7 +1264,8 @@ def disable_s3_logs(self, arns, s3_bucket): if flow_ids: self.client.delete_flow_logs(FlowLogIds=flow_ids) -class ElbResource(AWSResourcesAbstract): + +class ElbResource(LbResources): def fetch_resources(self): resources = [] next_token = None @@ -1232,83 +1313,32 @@ def get_arn_list_cloud_trail_event(self, event_detail): def tag_resources_cloud_trail_event(self, names, tags): self.client.add_tags(LoadBalancerNames=names, Tags=tags) - def enable_s3_logs(self, names, s3_bucket, s3_prefix, elb_region_account_id): + def enable_s3_logs(self, names, s3_bucket, s3_prefix): for name in names: print("Enable S3 logging for ALB " + name) response = self.client.describe_load_balancer_attributes(LoadBalancerName=name) if "LoadBalancerAttributes" in response: access_logs = response.get("LoadBalancerAttributes").get("AccessLog") - if(access_logs["Enabled"]==False): - access_logs["Enabled"]=True - access_logs["S3BucketName"]=s3_bucket - access_logs["S3BucketPrefix"]=s3_prefix + if not access_logs["Enabled"]: + access_logs["Enabled"] = True + access_logs["S3BucketName"] = s3_bucket + access_logs["S3BucketPrefix"] = s3_prefix try: self.client.modify_load_balancer_attributes(LoadBalancerName=name, LoadBalancerAttributes=response.get("LoadBalancerAttributes")) time.sleep(10) except ClientError as e: if "Error" in e.response and "Message" in e.response["Error"] \ and "Access Denied for bucket" in e.response['Error']['Message']: + elb_region = Region2ELBAccountId.get(self.region_value, None) + elb_region_account_id = None + if elb_region: + elb_region_account_id = elb_region.get("AccountId") self.add_bucket_policy(s3_bucket, elb_region_account_id) time.sleep(10) self.client.modify_load_balancer_attributes(LoadBalancerName=name, LoadBalancerAttributes=response) else: raise e - def add_bucket_policy(self, bucket_name, elb_region_account_id): - print("Adding policy to the bucket " + bucket_name) - s3 = boto3.client('s3') - try: - response = s3.get_bucket_policy(Bucket=bucket_name) - existing_policy = json.loads(response["Policy"]) - except ClientError as e: - if "Error" in e.response and "Code" in e.response["Error"] \ - and e.response['Error']['Code'] == "NoSuchBucketPolicy": - existing_policy = { - "Version": "2012-10-17", - "Statement": [ - ] - } - else: - raise e - - bucket_policy = [ - { - "Sid": "AwsElbLogs", - "Effect": "Allow", - "Principal": { - "AWS": f"arn:{self.partition}:iam::{elb_region_account_id}:root" - }, - "Action": ["s3:PutObject"], - "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*" - }, - { - "Sid": "AWSLogDeliveryAclCheck", - "Effect": "Allow", - "Principal": { - "Service": "delivery.logs.amazonaws.com" - }, - "Action": "s3:GetBucketAcl", - "Resource": f"arn:{self.partition}:s3:::{bucket_name}" - }, - { - "Sid": "AWSLogDeliveryWrite", - "Effect": "Allow", - "Principal": { - "Service": "delivery.logs.amazonaws.com" - }, - "Action": "s3:PutObject", - "Resource": f"arn:{self.partition}:s3:::{bucket_name}/*", - "Condition": { - "StringEquals": { - "s3:x-amz-acl": "bucket-owner-full-control" - } - } - } - ] - existing_policy["Statement"].extend(bucket_policy) - - s3.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(existing_policy)) - def disable_s3_logs(self, names, s3_bucket): attributes = [{'Key': 'access_logs.s3.enabled', 'Value': 'false'}] @@ -1316,8 +1346,8 @@ def disable_s3_logs(self, names, s3_bucket): response = self.client.describe_load_balancer_attributes(LoadBalancerName=name) if "LoadBalancerAttributes" in response: access_logs = response.get("LoadBalancerAttributes").get("AccessLog") - if(access_logs["Enabled"]==True): - access_logs["Enabled"]=False + if access_logs["Enabled"]: + access_logs["Enabled"] = False self.client.modify_load_balancer_attributes(LoadBalancerName=name, LoadBalancerAttributes=response.get("LoadBalancerAttributes")) time.sleep(1) @@ -1351,7 +1381,7 @@ def get_provider(cls, provider_name, region_value, account_id, *args, **kwargs): if provider_name in cls.provider_map: return cls.provider_map[provider_name](provider_name, region_value, account_id) else: - raise Exception("%s provider not found" % provider_name) + raise Exception(f"{provider_name} provider not found") if __name__ == '__main__': diff --git a/sumologic-app-utils/src/main.py b/sumologic-app-utils/src/main.py index 256ea3f..0aae66f 100644 --- a/sumologic-app-utils/src/main.py +++ b/sumologic-app-utils/src/main.py @@ -1,73 +1,114 @@ -from crhelper import CfnResource +import logging +import json from sumoresource import SumoResource -from awsresource import AWSResource - from resourcefactory import ResourceFactory -helper = CfnResource(json_logging=False, log_level='INFO', sleep_on_delete=30) +try: + from crhelper import CfnResource + helper = CfnResource(json_logging=False, log_level='INFO', sleep_on_delete=30) + USE_CRHELPER = True +except ImportError: + helper = None + USE_CRHELPER = False + +logger = logging.getLogger() +logger.setLevel(logging.INFO) def get_resource(event): - resource_type = event.get("ResourceType").split("::")[-1] + """Factory method to get a resource object and parameters.""" + resource_type = event.get("ResourceType", "").split("::")[-1] resource_class = ResourceFactory.get_resource(resource_type) - props = event.get("ResourceProperties") + props = event.get("ResourceProperties", {}) resource = resource_class(props) params = resource.extract_params(event) + if isinstance(resource, SumoResource): params["remove_on_delete_stack"] = props.get("RemoveOnDeleteStack") == 'true' + return resource, resource_type, params -@helper.create -def create(event, context): - # Test with failure cases should not get stuck in progress - # Optionally return an ID that will be used for the resource PhysicalResourceId, - # if None is returned an ID will be generated. If a poll_create function is defined - # return value is placed into the poll event as event['CrHelperData']['PhysicalResourceId'] - resource, resource_type, params = get_resource(event) - # Handle Exception to send a proper error to CF logs. - try: - data, resource_id = resource.create(**params) - except Exception as e: - raise e - #print(data) - print(resource_id) - helper.Data.update(data) - helper.Status = "SUCCESS" - print("Created %s" % resource_type) - return "%s/%s" % (event.get('LogicalResourceId', ''), resource_id) - - -@helper.update -def update(event, context): - resource, resource_type, params = get_resource(event) - data, resource_id = resource.update(**params) - #print(data) - print(resource_id) - helper.Data.update(data) - helper.Status = "SUCCESS" - print("Updated %s" % resource_type) - return "%s/%s" % (event.get('LogicalResourceId', ''), resource_id) - # If the update resulted in a new resource being created, return an id for the new resource. - # CloudFormation will send a delete event with the old id when stack update completes - - -@helper.delete -def delete(event, context): - if "/" not in event.get('PhysicalResourceId', ""): - print("%s resource_id not found" % event.get('PhysicalResourceId')) - return - resource, resource_type, params = get_resource(event) - resource.delete(**params) - helper.Status = "SUCCESS" - print("Deleted %s" % resource_type) - # Delete never returns anything. Should not fail if the underlying resources are already deleted. Desired state. +# -------------------------- +# CFN path (crhelper managed) +# -------------------------- +if USE_CRHELPER: + + @helper.create + def create(event, context): + resource, resource_type, params = get_resource(event) + try: + data, resource_id = resource.create(**params) + except Exception as e: + logger.error(f"Create failed for {resource_type}: {e}") + raise + helper.Data.update(data) + helper.Status = "SUCCESS" + logger.info(f"Created {resource_type} with ID {resource_id}") + return f"{event.get('LogicalResourceId', '')}/{resource_id}" + + @helper.update + def update(event, context): + resource, resource_type, params = get_resource(event) + data, resource_id = resource.update(**params) + helper.Data.update(data) + helper.Status = "SUCCESS" + logger.info(f"Updated {resource_type} with ID {resource_id}") + return f"{event.get('LogicalResourceId', '')}/{resource_id}" + + @helper.delete + def delete(event, context): + phys_id = event.get("PhysicalResourceId", "") + if "/" not in phys_id: + logger.warning(f"{phys_id} resource_id not found") + return + resource, resource_type, params = get_resource(event) + resource.delete(**params) + helper.Status = "SUCCESS" + logger.info(f"Deleted {resource_type}") def handler(event, context): - helper(event, context) + """ + Common handler (CF + TF) + """ + logger.info(f"Received event: {json.dumps(event)}") + + # CloudFormation event → delegate to crhelper + if "RequestType" in event and USE_CRHELPER: + return helper(event, context) + + # Terraform/direct invoke path + action = event.get("action") + logger.info(f"Terraform action detected: {action}") + + if action in ["create", "update", "delete"]: + resource, resource_type, params = get_resource(event) + try: + if action == "create": + data, resource_id = resource.create(**params) + elif action == "update": + data, resource_id = resource.update(**params) + elif action == "delete": + resource.delete(**params) + return {"status": "success", "deleted": True} + except Exception as e: + logger.error(f"{action} failed for {resource_type}: {e}") + return {"status": "failed", "reason": str(e)} + + return {"status": "success", "id": resource_id, "data": data} + + return {"status": "failed", "reason": f"Unknown action {action}"} if __name__ == "__main__": - event = {} - create(event, None) + # Example local test + test_event = { + "action": "create", + "ResourceType": "Custom::MyResource", + "ResourceProperties": { + "BucketName": "my-bucket", + "RemoveOnDeleteStack": "true" + } + } + print(handler(test_event, None)) diff --git a/sumologic-app-utils/src/resourcefactory.py b/sumologic-app-utils/src/resourcefactory.py index fd2543b..d22aca5 100644 --- a/sumologic-app-utils/src/resourcefactory.py +++ b/sumologic-app-utils/src/resourcefactory.py @@ -13,7 +13,7 @@ def register(cls, objname, obj): def get_resource(cls, objname): if objname in cls.resource_type: return cls.resource_type[objname] - raise Exception("%s resource type is undefined" % objname) + raise Exception(f"{objname} resource type is undefined") class AutoRegisterResource(ABCMeta): diff --git a/sumologic-app-utils/src/sumologic.py b/sumologic-app-utils/src/sumologic.py index c1b096b..9244004 100644 --- a/sumologic-app-utils/src/sumologic.py +++ b/sumologic-app-utils/src/sumologic.py @@ -50,7 +50,7 @@ def _get_endpoint(self): return endpoint def get_versioned_endpoint(self, version): - return self.endpoint + '/%s' % version + return f"{self.endpoint}/{version}" def delete(self, method, params=None, headers=None, version=DEFAULT_VERSION): endpoint = self.get_versioned_endpoint(version) @@ -203,7 +203,7 @@ def millisectimestamp(ts): def delete_folder(self, folder_id, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - return self.delete('/content/%s/delete' % folder_id, headers=headers, version='v2') + return self.delete(f'/content/{folder_id}/delete', headers=headers, version='v2') def create_folder(self, name, description, parent_folder_id, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} @@ -219,50 +219,71 @@ def get_personal_folder(self): def get_folder_by_id(self, folder_id, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - response = self.get('/content/folders/%s' % folder_id, version='v2', headers=headers) + response = self.get(f'/content/folders/{folder_id}', version='v2', headers=headers) return json.loads(response.text) def update_folder_by_id(self, folder_id, content, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - response = self.put('/content/folders/%s' % folder_id, version='v2', headers=headers, params=content) + response = self.put(f'/content/folders/{folder_id}', version='v2', headers=headers, params=content) return json.loads(response.text) def copy_folder(self, folder_id, parent_folder_id, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - return self.post('/content/%s/copy?destinationFolder=%s' % (folder_id, parent_folder_id), headers=headers, + return self.post(f'/content/{folder_id}/copy?destinationFolder={parent_folder_id}', headers=headers, params={}, version='v2') def import_content(self, folder_id, content, is_overwrite="false", is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - return self.post('/content/folders/%s/import?overwrite=%s' % (folder_id, is_overwrite), headers=headers, + return self.post(f'/content/folders/{folder_id}/import?overwrite={is_overwrite}', headers=headers, params=content, version='v2') def check_import_status(self, folder_id, job_id, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - return self.get('/content/folders/%s/import/%s/status' % (folder_id, job_id), version='v2', headers=headers) + return self.get(f'/content/folders/{folder_id}/import/{job_id}/status', version='v2', headers=headers) def check_copy_status(self, folder_id, job_id): - return self.get('/content/%s/copy/%s/status' % (folder_id, job_id), version='v2') + return self.get(f'/content/{folder_id}/copy/{job_id}/status', version='v2') def install_app(self, app_id, content, is_admin=False): headers = {'isAdminMode': 'true'} if is_admin else {} - return self.post('/apps/%s/install' % (app_id), headers=headers, params=content) + return self.post(f'/apps/{app_id}/install', headers=headers, params=content) + + def install_app_v2(self, app_id, content={}): + return self.post(f'/apps/{app_id}/install', version='v2', params=content) + + def upgrade_app_v2(self, app_id, content={}): + return self.post(f'/apps/{app_id}/upgrade', version='v2', params=content) + + def uninstall_app_v2(self, app_id, content={}): + return self.post(f'/apps/{app_id}/uninstall', version='v2', params=content) def check_app_install_status(self, job_id): - return self.get('/apps/install/%s/status' % job_id) + return self.get(f'/apps/install/{job_id}/status') + + def check_app_v2_install_status(self, job_id): + return self.get(f'/apps/install/{job_id}/status', version='v2') + + def check_app_v2_uninstall_status(self, job_id): + return self.get(f'/apps/uninstall/{job_id}/status', version='v2') + + def check_app_v2_upgrade_status(self, job_id): + return self.get(f'/apps/upgrade/{job_id}/status', version='v2') def get_apps(self): response = self.get('/apps') return json.loads(response.text) + def get_instances_app_v2(self): + return self.get(f'/apps/instances', version='v2') + def create_hierarchy(self, content): return self.post('/entities/hierarchies', params=content, version='v1') def delete_hierarchy(self, hierarchy_id): - return self.delete('/entities/hierarchies/%s' % hierarchy_id, version='v1') + return self.delete(f'/entities/hierarchies/{hierarchy_id}', version='v1') def update_hierarchy(self, hierarchy_id, content): - return self.put('/entities/hierarchies/%s' % hierarchy_id, params=content, version='v1') + return self.put(f'/entities/hierarchies/{hierarchy_id}', params=content, version='v1') def get_entity_hierarchies(self): response = self.get('/entities/hierarchies', version='v1') @@ -272,13 +293,13 @@ def create_metric_rule(self, content): return self.post('/metricsRules', params=content) def delete_metric_rule(self, metric_rule_name): - return self.delete('/metricsRules/%s' % metric_rule_name) + return self.delete(f'/metricsRules/{metric_rule_name}') def create_field_extraction_rule(self, content): return self.post('/extractionRules', params=content) def delete_field_extraction_rule(self, fer_name): - return self.delete('/extractionRules/%s' % fer_name) + return self.delete(f'/extractionRules/{fer_name}') def get_all_field_extraction_rules(self, limit=None, token=None, ): params = {'limit': limit, 'token': token} @@ -286,10 +307,10 @@ def get_all_field_extraction_rules(self, limit=None, token=None, ): return json.loads(r.text) def update_field_extraction_rules(self, fer_id, fer_details): - return self.put('/extractionRules/%s' % fer_id, fer_details) + return self.put(f'/extractionRules/{fer_id}', fer_details) def get_fer_by_id(self, fer_id): - response = self.get('/extractionRules/%s' % fer_id) + response = self.get(f'/extractionRules/{fer_id}') return json.loads(response.text) def fetch_metric_data_points(self, content): @@ -304,14 +325,14 @@ def get_all_fields(self): return json.loads(response.text)['data'] def get_existing_field(self, field_id): - response = self.get('/fields/%s' % field_id) + response = self.get(f'/fields/{field_id}') return json.loads(response.text) def delete_existing_field(self, field_id): - return self.delete('/fields/%s' % field_id) + return self.delete(f'/fields/{field_id}') def import_monitors(self, folder_id, content): - response = self.post('/monitors/%s/import' % folder_id, params=content) + response = self.post(f'/monitors/{folder_id}/import', params=content) return json.loads(response.text) def set_monitors_permissions(self, content): @@ -319,7 +340,7 @@ def set_monitors_permissions(self, content): return json.loads(response.text) def export_monitors(self, folder_id): - response = self.get('/monitors/%s/export' % folder_id) + response = self.get(f'/monitors/{folder_id}/export') return json.loads(response.text) def get_root_folder(self): @@ -327,4 +348,4 @@ def get_root_folder(self): return json.loads(response.text) def delete_monitor_folder(self, folder_id): - return self.delete('/monitors/%s' % folder_id) + return self.delete(f'/monitors/{folder_id}') diff --git a/sumologic-app-utils/src/sumoresource.py b/sumologic-app-utils/src/sumoresource.py index f887f36..a9c358e 100644 --- a/sumologic-app-utils/src/sumoresource.py +++ b/sumologic-app-utils/src/sumoresource.py @@ -45,9 +45,9 @@ def api_endpoint(self): if self.deployment == "us1": return "https://api.sumologic.com/api" elif self.deployment in ["ca", "au", "de", "eu", "esc", "jp", "us2", "fed", "kr", "ch"]: - return "https://api.%s.sumologic.com/api" % self.deployment + return f"https://api.{self.deployment}.sumologic.com/api" else: - return 'https://%s-api.sumologic.net/api' % self.deployment + return f'https://{self.deployment}-api.sumologic.net/api' def is_enterprise_or_trial_account(self): to_time = int(time.time()) * 1000 @@ -61,9 +61,9 @@ def is_enterprise_or_trial_account(self): | toint(sev) as sev | benchmark percentage as global_percent from guardduty on threatpurpose=threatPurpose, threatname=threatName, severity=sev, resource=targetresource''' response = self.sumologic_cli.search_job(search_query, fromTime=from_time, toTime=to_time) - print("schedule job status: %s" % response) + print(f"schedule job status: {response}") response = self.sumologic_cli.search_job_status(response) - print("job status: %s" % response) + print(f"job status: {response}") if len(response.get("pendingErrors", [])) > 0: return False else: @@ -94,7 +94,7 @@ def _get_collector_by_name(self, collector_name, collector_type): offset += page_limit all_collectors = self.sumologic_cli.collectors(limit=page_limit, filter_type=collector_type, offset=offset) - raise Exception("Collector with name %s not found" % collector_name) + raise Exception(f"Collector with name {collector_name} not found") def create(self, collector_type, collector_name, source_category=None, description='', *args, **kwargs): collector_id = None @@ -109,13 +109,13 @@ def create(self, collector_type, collector_name, source_category=None, descripti try: resp = self.sumologic_cli.create_collector(collector, headers=None) collector_id = json.loads(resp.text)['collector']['id'] - print("created collector %s" % collector_id) + print(f"created collector {collector_id}") except Exception as e: if hasattr(e, 'response') and "code" in e.response.json() and e.response.json()[ "code"] == 'collectors.validation.name.duplicate': collector = self._get_collector_by_name(collector_name, collector_type.lower()) collector_id = collector['id'] - print("fetched existing collector %s" % collector_id) + print(f"fetched existing collector {collector_id}") else: raise @@ -129,7 +129,7 @@ def update(self, collector_id, collector_type, collector_name, source_category=N cv['collector']['description'] = description resp = self.sumologic_cli.update_collector(cv, etag) collector_id = json.loads(resp.text)['collector']['id'] - print("updated collector %s" % collector_id) + print(f"updated collector {collector_id}") return {"COLLECTOR_ID": collector_id}, collector_id def delete(self, collector_id, remove_on_delete_stack, *args, **kwargs): @@ -140,7 +140,7 @@ def delete(self, collector_id, remove_on_delete_stack, *args, **kwargs): sources = self.sumologic_cli.sources(collector_id, limit=10) if len(sources) == 0: response = self.sumologic_cli.delete_collector({"collector": {"id": collector_id}}) - print("deleted collector %s : %s" % (collector_id, response.text)) + print(f"deleted collector {collector_id} : {response.text}") else: print("skipping collector deletion") @@ -192,7 +192,7 @@ def create(self, type, name, description, url, username, password, region, servi try: resp = self.sumologic_cli.create_connection(connection, headers=None) connection_id = json.loads(resp.text)['id'] - print("created connectionId %s" % connection_id) + print(f"created connectionId {connection_id}") except Exception as e: if hasattr(e, 'response'): print(e.response.json()) @@ -215,13 +215,13 @@ def update(self, connection_id, type, url, description, username, password, *arg cv['password'] = password resp = self.sumologic_cli.update_collector(cv, etag) connection_id = json.loads(resp.text)['connections']['id'] - print("updated connections %s" % connection_id) + print(f"updated connections {connection_id}") return {"CONNECTION_ID": connection_id}, connection_id def delete(self, connection_id, remove_on_delete_stack, *args, **kwargs): if remove_on_delete_stack: response = self.sumologic_cli.delete_connection(connection_id, 'WebhookConnection') - print("deleted connection %s %s" % (connection_id, response.text)) + print(f"deleted connection {connection_id} {response.text}") else: print("skipping connection deletion") @@ -267,7 +267,7 @@ def build_common_source_params(self, props, source_json=None): source_json.update({ "category": props.get("SourceCategory"), "name": props.get("SourceName"), - "description": "This %s source is created by AWS SAM Application" % (props.get("SourceType", "HTTP")) + "description": f'This {props.get("SourceType", "HTTP")} source is created by AWS SAM Application' }) # timestamp processing if props.get("DateFormat"): @@ -388,7 +388,7 @@ def create(self, collector_id, source_name, props, *args, **kwargs): data = resp.json()['source'] source_id = data["id"] endpoint = data["url"] - print("created source %s" % source_id) + print(f"created source {source_id}") except Exception as e: # Todo 100 sources in a collector is good. Same error code for duplicates in case of Collector and source. if hasattr(e, 'response') and "code" in e.response.json() and e.response.json()[ @@ -396,7 +396,7 @@ def create(self, collector_id, source_name, props, *args, **kwargs): for source in self.sumologic_cli.sources(collector_id, limit=300): if source["name"] == source_name: source_id = source["id"] - print("fetched existing source %s" % source_id) + print(f"fetched existing source {source_id}") endpoint = source["url"] else: print(e, source_json) @@ -410,7 +410,7 @@ def update(self, collector_id, source_id, source_name, props, *args, try: resp = self.sumologic_cli.update_source(collector_id, source_json, etag) data = resp.json()['source'] - print("updated source %s" % data["id"]) + print(f"updated source {data['id']}") return {"SUMO_ENDPOINT": data["url"]}, data["id"] except Exception as e: print(e, source_json) @@ -419,7 +419,7 @@ def update(self, collector_id, source_id, source_name, props, *args, def delete(self, collector_id, source_id, remove_on_delete_stack, props, *args, **kwargs): if remove_on_delete_stack: response = self.sumologic_cli.delete_source(collector_id, {"source": {"id": source_id}}) - print("deleted source %s : %s" % (source_id, response.text)) + print(f"deleted source {source_id} : {response.text}") else: print("skipping source deletion") @@ -495,7 +495,7 @@ def create(self, collector_id, source_name, props, *args, **kwargs): data = resp.json()['source'] source_id = data["id"] endpoint = data["url"] - print("created source %s" % source_id) + print(f"created source {source_id}") except Exception as e: # Todo 100 sources in a collector is good if hasattr(e, 'response') and "code" in e.response.json() and e.response.json()[ @@ -503,7 +503,7 @@ def create(self, collector_id, source_name, props, *args, **kwargs): for source in self.sumologic_cli.sources(collector_id, limit=300): if source["name"] == source_name: source_id = source["id"] - print("fetched existing source %s" % source_id) + print(f"fetched existing source {source_id}") endpoint = source["url"] else: raise @@ -516,13 +516,13 @@ def update(self, collector_id, source_id, source_name, props, *args, resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] - print("updated source %s" % data["id"]) + print(f"updated source {data['id']}") return {"SUMO_ENDPOINT": data["url"]}, data["id"] def delete(self, collector_id, source_id, remove_on_delete_stack, *args, **kwargs): if remove_on_delete_stack: response = self.sumologic_cli.delete_source(collector_id, {"source": {"id": source_id}}) - print("deleted source %s : %s" % (source_id, response.text)) + print(f"deleted source {source_id} : {response.text}") else: print("skipping source deletion") @@ -546,14 +546,14 @@ class App(SumoResource): def _convert_to_hour(self, timeoffset): hour = timeoffset / 60 * 60 * 1000 - return "%sh" % (hour) + return f"{hour}h" def _replace_source_category(self, appjson_filepath, sourceDict): with open(appjson_filepath, 'r') as old_file: text = old_file.read() if sourceDict: for k, v in sourceDict.items(): - text = text.replace("$$%s" % k, v) + text = text.replace(f"$${k}", v) appjson = json.loads(text) return appjson @@ -592,8 +592,8 @@ def _get_app_content(self, appname, source_params, s3url=None): # Based on S3 URL provided download the data. if not s3url: key_name = "ApiExported-" + re.sub(r"\s+", "-", appname) + ".json" - s3url = "https://app-json-store.s3.amazonaws.com/%s" % key_name - print("Fetching appjson %s" % s3url) + s3url = f"https://app-json-store.s3.amazonaws.com/{key_name}" + print(f"Fetching appjson {s3url}") with requests.get(s3url, stream=True) as r: r.raise_for_status() with tempfile.NamedTemporaryFile() as fp: @@ -608,38 +608,38 @@ def _get_app_content(self, appname, source_params, s3url=None): return appjson def _wait_for_folder_creation(self, folder_id, job_id, is_admin): - print("waiting for folder creation folder_id %s job_id %s" % (folder_id, job_id)) + print(f"waiting for folder creation folder_id {folder_id} job_id {job_id}") waiting = True while waiting: response = self.sumologic_cli.check_import_status(folder_id, job_id, is_admin) waiting = response.json()['status'] == "InProgress" time.sleep(2) - print("job status: %s" % response.text) + print(f"job status: {response.text}") def _wait_for_folder_copy(self, folder_id, job_id): - print("waiting for folder copy folder_id %s job_id %s" % (folder_id, job_id)) + print(f"waiting for folder copy folder_id {folder_id} job_id {job_id}") waiting = True while waiting: response = self.sumologic_cli.check_copy_status(folder_id, job_id) waiting = response.json()['status'] == "InProgress" time.sleep(2) - print("job status: %s" % response.text) - matched = re.search('id:\s*(.*?)\"', response.text) + print(f"job status: {response.text}") + matched = re.search('id:\\s*(.*?)\"', response.text) copied_folder_id = None if matched: copied_folder_id = matched[1] return copied_folder_id def _wait_for_app_install(self, job_id): - print("waiting for app installation job_id %s" % job_id) + print(f"waiting for app installation job_id {job_id}") waiting = True while waiting: response = self.sumologic_cli.check_app_install_status(job_id) waiting = response.json()['status'] == "InProgress" time.sleep(2) - print("job status: %s" % response.text) + print(f"job status: {response.text}") return response def _create_backup_folder(self, new_app_folder_id, old_app_folder_id, is_admin): @@ -711,7 +711,6 @@ def share_content_with_org(self, is_share, content_id, org_id, is_admin): else: raise Exception(f"Unable to share {content_id} in org: {org_id}") - def share_app_by_id(self, is_share, app_folder_id, org_id, is_admin): """ This shares an app identified by its Id under the Admin Recommended folder """ response = self.share_content_with_org(is_share, app_folder_id, org_id, is_admin) @@ -750,7 +749,7 @@ def _create_or_fetch_apps_parent_folder(self, folder_prefix, org_id, is_share=Fa def create_by_import_api(self, appname, source_params, folder_name, s3url, org_id, location, is_share, *args, **kwargs): # Add retry if folder sync fails if appname in self.ENTERPRISE_ONLY_APPS and not self.is_enterprise_or_trial_account(): - raise Exception("%s is available to Enterprise or Trial Account Type only." % appname) + raise Exception(f"{appname} is available to Enterprise or Trial Account Type only.") content = self._get_app_content(appname, source_params, s3url) is_admin = False @@ -769,15 +768,14 @@ def create_by_import_api(self, appname, source_params, folder_name, s3url, org_i time.sleep(3) response = self.sumologic_cli.import_content(folder_id, content, is_overwrite="true") job_id = response.json()["id"] - print("Imported app %s: appFolderId: %s FolderId: %s jobId: %s" % ( - appname, app_folder_id, folder_id, job_id)) + print(f"Imported app {appname}: appFolderId: {app_folder_id} FolderId: {folder_id} jobId: {job_id}") self._wait_for_folder_creation(folder_id, job_id, is_admin) return {"APP_FOLDER_NAME": content["name"]}, app_folder_id def create_by_install_api(self, appid, appname, source_params, folder_name, org_id, location, is_share, *args, **kwargs): if appname in self.ENTERPRISE_ONLY_APPS and not self.is_enterprise_or_trial_account(): - raise Exception("%s is available to Enterprise or Trial Account Type only." % appname) + raise Exception(f"{appname} is available to Enterprise or Trial Account Type only.") if folder_name: folder_id = self._create_or_fetch_apps_parent_folder(folder_name, org_id, is_share, location) @@ -798,11 +796,10 @@ def create_by_install_api(self, appid, appname, source_params, folder_name, org_ json_resp = json.loads(response.content) if json_resp['status'] == 'Success': app_folder_id = json_resp['statusMessage'].split(":")[1] - print("installed app %s: appFolderId: %s parent_folder_id: %s jobId: %s" % ( - appname, app_folder_id, folder_id, job_id)) + print(f"installed app {appname}: appFolderId: {app_folder_id} parent_folder_id: {folder_id} jobId: {job_id}") return {"APP_FOLDER_NAME": content["name"]}, app_folder_id else: - print("%s installation failed." % appname) + print(f"{appname} installation failed.") raise Exception(response.text) def create(self, appname, source_params, org_id, is_share=True, location=None, appid=None, folder_name=None, s3url=None, @@ -826,7 +823,7 @@ def update(self, app_folder_id, appname, source_params, org_id, is_share=True, l data, new_app_folder_id = self.create(appname=appname, source_params=source_params, appid=appid, folder_name=folder_name, s3url=s3url, org_id=org_id, is_share=is_share, location=location) - print("updated app appFolderId: %s " % new_app_folder_id) + print(f"updated app appFolderId: {new_app_folder_id} ") if retain_old_app: try: backup_folder_id = self._create_backup_folder(new_app_folder_id, app_folder_id, is_admin) @@ -834,7 +831,7 @@ def update(self, app_folder_id, appname, source_params, org_id, is_share=True, l # Starting Folder Copy response = self.sumologic_cli.copy_folder(app_folder_id, backup_folder_id, is_admin) job_id = response.json()["id"] - print("Copy Completed parentFolderId: %s jobId: %s" % (backup_folder_id, job_id)) + print(f"Copy Completed parentFolderId: {backup_folder_id} jobId: {job_id}") copied_folder_id = self._wait_for_folder_copy(app_folder_id, job_id) # Updating copied folder name with suffix BackUp. copied_folder_details = self.sumologic_cli.get_folder_by_id(copied_folder_id, is_admin) @@ -843,9 +840,9 @@ def update(self, app_folder_id, appname, source_params, org_id, is_share=True, l "%H:%M:%S")), "description": copied_folder_details["description"][:255]} self.sumologic_cli.update_folder_by_id(copied_folder_id, copied_folder_details, is_admin) - print("Back Up done for the APP: %s." % backup_folder_id) + print(f"Back Up done for the APP: {backup_folder_id}.") except Exception as e: - print("App - Exception while taking backup of App folder ID %s, error: %s " % (app_folder_id, e)) + print(f"App - Exception while taking backup of App folder ID {app_folder_id}, error: {e} ") return data, new_app_folder_id @@ -856,9 +853,9 @@ def delete(self, app_folder_id, remove_on_delete_stack, location=None, *args, ** if remove_on_delete_stack: try: response = self.sumologic_cli.delete_folder(app_folder_id, is_admin) - print("deleting app folder %s : %s" % (app_folder_id, response.text)) + print(f"deleting app folder {app_folder_id} : {response.text}") except Exception as e: - print("App - Exception while deleting the App folder ID %s, error: %s " % (app_folder_id, e)) + print(f"App - Exception while deleting the App folder ID {app_folder_id}, error: {e} ") else: print("skipping app folder deletion") @@ -881,6 +878,110 @@ def extract_params(self, event): } +class AppV2(SumoResource): + + def _wait_for_job(self, job_id, status_fn): + print(f"Waiting for job_id: {job_id}") + while True: + response = status_fn(job_id) + json_resp = response.json() + if json_resp['status'] != "InProgress": + print(f"Job status: {json_resp}") + return response + time.sleep(2) + + @staticmethod + def is_latest(app_instance): + current_version = app_instance.get("version") + latest_version = app_instance.get("latestVersion") + appname = app_instance.get("name") + print(f"App: {appname}") + print(f"Current Version: {current_version}") + print(f"Latest Version: {latest_version}") + return current_version == latest_version + + def _handle_job_response(self, appid, response, job_id, appname, action="installed"): + json_resp = response.json() + print("json_resp", json_resp) + if json_resp['status'] == 'Success': + app_folder_id = json_resp.get('folderId') + app_path = json_resp.get('path') + print(f"jobId:{job_id} -> {action} app '{appname}', folderId: {app_folder_id}, path: {app_path}") + return {"APP_FOLDER_NAME": appname}, appid + raise Exception(f"App '{appname}' {action} failed: {json_resp}") + + def get_installed_apps(self): + response = self.sumologic_cli.get_instances_app_v2() + return (response.json() or {}).get("data", []) + + def check_app_installed(self, app_id): + return next((app for app in self.get_installed_apps() if app["uuid"] == app_id), None) + + def install_app(self, appid, appname, version, *args, **kwargs): + content = {'name': appname, 'version': version} + response = self.sumologic_cli.install_app_v2(appid, content) + job_id = response.json()["jobId"] + response = self._wait_for_job(job_id, self.sumologic_cli.check_app_v2_install_status) + return self._handle_job_response(appid, response, job_id, appname, action="installed") + + def create(self, appid, appname, version, *args, **kwargs): + if not appid: + return None + app_instance = self.check_app_installed(appid) + if app_instance: + if not self.is_latest(app_instance): + print(f"App {appname} is not latest, upgrading") + return self.upgrade(appid, appname) + return {"APP_FOLDER_NAME": appname}, appid + print(f"App {appname} is installing") + return self.install_app(appid, appname, version, *args, **kwargs) + + def upgrade(self, appid, appname): + response = self.sumologic_cli.upgrade_app_v2(appid, {}) + job_id = response.json()["jobId"] + response = self._wait_for_job(job_id, self.sumologic_cli.check_app_v2_upgrade_status) + return self._handle_job_response(appid, response, job_id, appname, action="upgraded") + + def update(self, appid, appname, version, *args, **kwargs): + if not appid: + return None + app_instance = self.check_app_installed(appid) + if not app_instance: + print(f"App {appname} is not present") + return self.install_app(appid, appname, version, *args, **kwargs) + # Extract version information + if self.is_latest(app_instance): + print(f"App {appname} is already updated") + return {"APP_FOLDER_NAME": appname}, app_instance["uuid"] + print(f"App {appname} is updating") + return self.upgrade(appid, appname) + + def delete(self, appid, appname, remove_on_delete_stack, *args, **kwargs): + if not remove_on_delete_stack or not appid: + print("Skipping app uninstallation") + return None + app_instance = self.check_app_installed(appid) + if not app_instance: + print("App is already uninstalled") + return None + response = self.sumologic_cli.uninstall_app_v2(appid) + job_id = response.json()["jobId"] + response = self._wait_for_job(job_id, self.sumologic_cli.check_app_v2_uninstall_status) + if response.json()['status'] == 'Success': + print(f"jobId:{job_id} -> uninstalled app '{appname}'") + return None + + def extract_params(self, event): + print("extract_params", event) + props = event.get("ResourceProperties", {}) + + return { + "appid": props.get("AppId"), + "appname": props.get("AppName"), + "version": props.get("Version", "latest"), + } + + class SumoLogicAWSExplorer(SumoResource): def get_explorer_id(self, hierarchy_name): @@ -889,7 +990,7 @@ def get_explorer_id(self, hierarchy_name): for hierarchy in hierarchies["data"]: if hierarchy_name == hierarchy["name"]: return hierarchy["id"] - raise Exception("Hierarchy with name %s not found" % hierarchy_name) + raise Exception(f"Hierarchy with name {hierarchy_name} not found") def create_hierarchy(self, hierarchy_name, level, hierarchy_filter): content = { @@ -900,19 +1001,19 @@ def create_hierarchy(self, hierarchy_name, level, hierarchy_filter): try: response = self.sumologic_cli.create_hierarchy(content) hierarchy_id = response.json()["id"] - print("Hierarchy - creation successful with ID %s" % hierarchy_id) + print(f"Hierarchy - creation successful with ID {hierarchy_id}") return {"Hierarchy_Name": response.json()["name"]}, hierarchy_id except Exception as e: if hasattr(e, 'response') and "errors" in e.response.json() and e.response.json()["errors"]: errors = e.response.json()["errors"] for error in errors: if error.get('code') == 'hierarchy:duplicate': - print("Hierarchy - Duplicate Exists for Name %s" % hierarchy_name) + print(f"Hierarchy - Duplicate Exists for Name {hierarchy_name}") # Get the hierarchy ID from all explorer. hierarchy_id = self.get_explorer_id(hierarchy_name) response = self.sumologic_cli.update_hierarchy(hierarchy_id, content) hierarchy_id = response.json()["id"] - print("Hierarchy - update successful with ID %s" % hierarchy_id) + print(f"Hierarchy - update successful with ID {hierarchy_id}") return {"Hierarchy_Name": hierarchy_name}, hierarchy_id raise @@ -922,7 +1023,7 @@ def create(self, hierarchy_name, level, hierarchy_filter, *args, **kwargs): # Use the new update API. def update(self, hierarchy_id, hierarchy_name, level, hierarchy_filter, *args, **kwargs): data, hierarchy_id = self.create(hierarchy_name, level, hierarchy_filter) - print("Hierarchy - update successful with ID %s" % hierarchy_id) + print(f"Hierarchy - update successful with ID {hierarchy_id}") return data, hierarchy_id # handling exception during delete, as update can fail if the previous explorer, metric rule or field has @@ -935,8 +1036,7 @@ def delete(self, hierarchy_id, hierarchy_name, remove_on_delete_stack, *args, ** if hierarchy_id == "Duplicate": hierarchy_id = self.get_explorer_id(hierarchy_name) response = self.sumologic_cli.delete_hierarchy(hierarchy_id) - print("Hierarchy - Completed the Hierarchy deletion for Name %s, response - %s" - % (hierarchy_name, response.text)) + print(f"Hierarchy - Completed the Hierarchy deletion for Name {hierarchy_name}, response - {response.text}") else: print("Hierarchy - Skipping the Hierarchy deletion.") @@ -970,7 +1070,7 @@ def create_metric_rule(self, metric_rule_name, match_expression, variables, dele try: response = self.sumologic_cli.create_metric_rule(content) job_name = response.json()["name"] - print("METRIC RULES - creation successful with Name %s" % job_name) + print(f"METRIC RULES - creation successful with Name {job_name}") return {"METRIC_RULES": response.json()["name"]}, job_name except Exception as e: if hasattr(e, 'response') and "errors" in e.response.json() and e.response.json()["errors"]: @@ -978,7 +1078,7 @@ def create_metric_rule(self, metric_rule_name, match_expression, variables, dele for error in errors: if error.get('code') == 'metrics:rule_name_already_exists' \ or error.get('code') == 'metrics:rule_already_exists': - print("METRIC RULES - Duplicate Exists for Name %s" % metric_rule_name) + print(f"METRIC RULES - Duplicate Exists for Name {metric_rule_name}") if delete: self.delete(metric_rule_name, metric_rule_name, True) # providing sleep for 10 seconds after delete. @@ -995,7 +1095,7 @@ def update(self, old_metric_rule_name, job_name, metric_rule_name, match_express # Need to add it because CF calls delete method if identifies change in metric rule name. self.delete(job_name, old_metric_rule_name, True) data, job_name = self.create_metric_rule(metric_rule_name, match_expression, variables) - print("METRIC RULES - Update successful with Name %s" % job_name) + print(f"METRIC RULES - Update successful with Name {job_name}") return data, job_name # handling exception during delete, as update can fail if the previous explorer, metric rule or field has @@ -1006,10 +1106,9 @@ def delete(self, job_name, metric_rule_name, remove_on_delete_stack, *args, **kw try: response = self.sumologic_cli.delete_metric_rule(metric_rule_name) print( - "METRIC RULES - Completed the Metric Rule deletion for Name %s, response - %s" % (metric_rule_name, - response.text)) + f"METRIC RULES - Completed the Metric Rule deletion for Name {metric_rule_name}, response - {response.text}") except Exception as e: - print("AWS EXPLORER - Exception while deleting the Metric Rules %s," % e) + print(f"AWS EXPLORER - Exception while deleting the Metric Rules {e},") else: print("METRIC RULES - Skipping the Metric Rule deletion") @@ -1059,7 +1158,7 @@ def add_fields_to_collector(self, collector_id, source_id, fields): resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] - print("Added Fields in Source %s" % data["id"]) + print(f"Added Fields in Source {data['id']}") return {"source_name": data["name"]}, str(source_id) return {"source_name": "Not updated"}, "No_Source_Id" @@ -1087,7 +1186,7 @@ def update(self, collector_id, source_id, fields, old_resource_properties, *args sv['source']['fields'] = existing_source_fields resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] - print("updated Fields in Source %s" % data["id"]) + print(f"updated Fields in Source {data['id']}") return {"source_name": data["name"]}, source_id def delete(self, collector_id, source_id, fields, remove_on_delete_stack, *args, **kwargs): @@ -1103,7 +1202,7 @@ def delete(self, collector_id, source_id, fields, remove_on_delete_stack, *args, resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] - print("reverted Fields in Source %s" % data["id"]) + print(f"reverted Fields in Source {data['id']}") else: print("UPDATE FIELDS - Skipping the Metric Rule deletion") @@ -1142,7 +1241,7 @@ def _get_fer_by_name(self, fer_name): else: response = None - raise Exception("FER with name %s not found" % fer_name) + raise Exception(f"FER with name {fer_name} not found") def create(self, fer_name, fer_scope, fer_expression, fer_enabled, *args, **kwargs): content = { @@ -1154,14 +1253,14 @@ def create(self, fer_name, fer_scope, fer_expression, fer_enabled, *args, **kwar try: response = self.sumologic_cli.create_field_extraction_rule(content) job_id = response.json()["id"] - print("FER RULES - creation successful with ID %s" % job_id) + print(f"FER RULES - creation successful with ID {job_id}") return {"FER_RULES": response.json()["name"]}, job_id except Exception as e: if hasattr(e, 'response') and "errors" in e.response.json() and e.response.json()["errors"]: errors = e.response.json()["errors"] for error in errors: if error.get('code') == 'fer:invalid_extraction_rule': - print("FER RULES - Duplicate Exists for Name %s" % fer_name) + print(f"FER RULES - Duplicate Exists for Name {fer_name}") # check if there is difference in scope, if yes then merge the scopes. fer_details = self._get_fer_by_name(fer_name) change_in_fer = False @@ -1199,7 +1298,7 @@ def update(self, fer_id, fer_name, fer_scope, fer_expression, fer_enabled, *args response = self.sumologic_cli.update_field_extraction_rules(fer_id, content) job_id = response.json()["id"] - print("FER RULES - update successful with ID %s" % job_id) + print(f"FER RULES - update successful with ID {job_id}") return {"FER_RULES": response.json()["name"]}, job_id except Exception as e: raise @@ -1207,8 +1306,7 @@ def update(self, fer_id, fer_name, fer_scope, fer_expression, fer_enabled, *args def delete(self, fer_id, remove_on_delete_stack, *args, **kwargs): if remove_on_delete_stack: response = self.sumologic_cli.delete_field_extraction_rule(fer_id) - print("FER RULES - Completed the Metric Rule deletion for ID %s, response - %s" % ( - fer_id, response.text)) + print(f"FER RULES - Completed the Metric Rule deletion for ID {fer_id}, response - {response.text}") else: print("FER RULES - Skipping the Metric Rule deletion") @@ -1242,12 +1340,12 @@ def batch_size_chunking(self, iterable, size=1): def get_source_and_collector_id(self, instances): ids = [] for instance in instances: - ids.append("InstanceId=%s" % instance["InstanceId"]) + ids.append(f'InstanceId={instance["InstanceId"]}') query = " or ".join(ids) content = { "query": [ { - "query": "_contentType=HostMetrics (%s) | count by _sourceId, _collectorId" % query, + "query": f"_contentType=HostMetrics ({query}) | count by _sourceId, _collectorId", "rowId": "A" } ], @@ -1292,7 +1390,7 @@ def add_remove_fields(self, region_value, account_id, new_fields, old_fields=Non sv['source']['fields'] = existing_source_fields resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] - print("updated Fields in Source %s" % data["id"]) + print(f"updated Fields in Source {data['id']}") def create(self, region_value, account_id, fields, add_fields, *args, **kwargs): if add_fields: @@ -1356,7 +1454,7 @@ def get_field_id(self, field_name): for field in all_fields: if field_name == field["fieldName"]: return field["fieldId"] - raise Exception("Field Name with name %s not found" % field_name) + raise Exception(f"Field Name with name {field_name} not found") def add_field(self, field_name): content = { @@ -1365,14 +1463,14 @@ def add_field(self, field_name): try: response = self.sumologic_cli.create_new_field(content) field_id = response["fieldId"] - print("FIELD NAME - creation successful with Field Id %s" % field_id) + print(f"FIELD NAME - creation successful with Field Id {field_id}") return {"FIELD_NAME": response["fieldName"]}, field_id except Exception as e: if hasattr(e, 'response') and "errors" in e.response.json() and e.response.json()["errors"]: errors = e.response.json()["errors"] for error in errors: if error.get('code') == 'field:already_exists': - print("FIELD NAME - Duplicate Exists for Name %s" % field_name) + print(f"FIELD NAME - Duplicate Exists for Name {field_name}") # Get the Field ID from the existing fields. field_id = self.get_field_id(field_name) return {"FIELD_NAME": field_name}, field_id @@ -1399,9 +1497,9 @@ def delete(self, field_id, field_name, remove_on_delete_stack, *args, **kwargs): if field_id == "Duplicate": field_id = self.get_field_id(field_name) response = self.sumologic_cli.delete_existing_field(field_id) - print("FIELD NAME - Completed the Field deletion for ID %s, response - %s" % (field_id, response.text)) + print(f"FIELD NAME - Completed the Field deletion for ID {field_id}, response - {response.text}") except Exception as e: - print("AWS EXPLORER - Exception while deleting the Field %s," % e) + print(f"AWS EXPLORER - Exception while deleting the Field {e},") else: print("FIELD NAME - Skipping the Field deletion") @@ -1511,7 +1609,7 @@ def _replace_variables(self, appjson_filepath, variables): text = old_file.read() if variables: for k, v in variables.items(): - text = text.replace("${%s}" % k, v) + text = text.replace(f"${{{k}}}", v) appjson = json.loads(text) return appjson @@ -1557,7 +1655,7 @@ def import_monitor(self, folder_name, org_id, monitors3url, variables, suffix_da # monitor_permission_payload = {"permissionStatementDefinitions": [{"permissions": ["Create","Read","Update","Delete","Manage"],"subjectType": "org","subjectId": org_id,"targetId": import_id}]} # self.sumologic_cli.set_monitors_permissions(monitor_permission_payload) # End Uncomment above when FGP feature for monitors is live - print("ALERTS MONITORS - creation successful with ID %s and Name %s." % (import_id, folder_name)) + print(f"ALERTS MONITORS - creation successful with ID {import_id} and Name {folder_name}.") except: time.sleep(10) retry_counter -= 1 @@ -1586,7 +1684,7 @@ def update(self, folder_id, folder_name, org_id, monitors3url, variables, suffix print("Error while taking backup of Monitors folder") print(e) - print("ALERTS MONITORS - Update successful with ID %s." % new_folder_id) + print(f"ALERTS MONITORS - Update successful with ID {new_folder_id}.") return data, new_folder_id def delete(self, folder_id, remove_on_delete_stack, *args, **kwargs): @@ -1595,7 +1693,7 @@ def delete(self, folder_id, remove_on_delete_stack, *args, **kwargs): self.sumologic_cli.delete_monitor_folder(folder_id) print("ALERTS MONITORS - Completed the Deletion for Monitors Folder with ID " + str(folder_id)) except Exception as e: - print("ALERTS MONITORS - Exception while deleting the Monitors Folder %s," % e) + print(f"ALERTS MONITORS - Exception while deleting the Monitors Folder {e},") else: print("ALERTS MONITORS - Skipping the Monitor Folder deletion") @@ -1623,15 +1721,15 @@ def extract_params(self, event): "SumoAccessKey": "", "SumoDeployment": "", } - app_prefix = "ALB" - # app_prefix = "GuardDuty" - collector_id = None - collector_type = "Hosted" - collector_name = "%sCollector" % app_prefix - source_name = "%sEvents" % app_prefix - source_category = "Labs/AWS/%s" % app_prefix - appname = "AWS Application LB" - appid = "ceb7fac5-1137-4a04-a5b8-2e49190be3d4" + # app_prefix = "ALB" + # # app_prefix = "GuardDuty" + # collector_id = None + # collector_type = "Hosted" + # collector_name = f"{app_prefix}Collector" + # source_name = f"{app_prefix}Events" + # source_category = f"Labs/AWS/{app_prefix}" + # appname = "AWS Application LB" + # appid = "ceb7fac5-1137-4a04-a5b8-2e49190be3d4" # appid = "570bdc0d-f824-4fcb-96b2-3230d4497180" s3url = "" # appid = "ceb7fac5-1137-4a04-a5b8-2e49190be3d4" @@ -1639,15 +1737,30 @@ def extract_params(self, event): # source_params = { # "logsrc": "_sourceCategory=%s" % source_category # } - source_params = { - "cloudtraillogsource": "_sourceCategory=%s" % source_category, - "indexname": '%rnd%', - "incrementalindex": "%rnd%" - } + # source_params = { + # "cloudtraillogsource": f"_sourceCategory={source_category}", + # "indexname": '%rnd%', + # "incrementalindex": "%rnd%" + # } # col = Collector(**params) # src = HTTPSource(**params) # app = App(props) + #appname = "AWS Application Load Balancer" + #appid = "27a17946-e475-4d56-8a8f-bc3fbc0400ca" + # appname = "Amazon Bedrock" + # appid = "8f4fd1aa-3b83-4d2e-b2ef-e8baec880afa" + appname = "AWS EC2" + appid = "3dcaacb4-a5de-4e57-a477-fccd04f9e40f" + app = AppV2(props) + #app.get_install_apps() + #id = "CD6CC478B5018A8D" + version = None + #print(app.create(appid, appname, version)) + print(app.get_installed_apps()) + + #app.install_app(appid, appname, version="latest", location="user", is_share=False) + # create # _, collector_id = col.create(collector_type, collector_name, source_category) # _, source_id = src.create(collector_id, source_name, source_category) @@ -1655,8 +1768,8 @@ def extract_params(self, event): # _, app_folder_id = app.update(app_folder_id='0000000001A70848', appname=appname, source_params=source_params,folder_name="abcd" ,s3url=s3url,orgID="0000000000BC5DF9",share=True,location='admin',retain_old_app=True) #import # app.delete(app_folder_id, True, location='admin') - monitor = AlertsMonitor(props) - monitors3 = "https://sumologic-appdev-aws-sam-apps.s3.amazonaws.com/aws-observability-versions/v2.8.0/appjson/Alerts-App.json" + # monitor = AlertsMonitor(props) + # monitors3 = "https://sumologic-appdev-aws-sam-apps.s3.amazonaws.com/aws-observability-versions/v2.8.0/appjson/Alerts-App.json" # _, app_folder_id = monitor.create('abc','0000000000BD3DDD',monitors3,"",retain_old_alerts=False) # _, app_folder_id = monitor.update('000000000002796B','abc1','0000000000285A74',monitors3,"",retain_old_alerts=True) diff --git a/sumologic-app-utils/sumo-app-utils.zip b/sumologic-app-utils/sumo-app-utils.zip new file mode 100644 index 0000000..b726744 Binary files /dev/null and b/sumologic-app-utils/sumo-app-utils.zip differ diff --git a/sumologic-app-utils/sumo_app_utils.zip b/sumologic-app-utils/sumo_app_utils.zip deleted file mode 100644 index 6ec5cd1..0000000 Binary files a/sumologic-app-utils/sumo_app_utils.zip and /dev/null differ diff --git a/upload_artifacts.py b/upload_artifacts.py index 449b458..cde84e2 100644 --- a/upload_artifacts.py +++ b/upload_artifacts.py @@ -3,18 +3,18 @@ from argparse import ArgumentParser regions = [ - "us-east-2", "us-east-1", + "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-4", + "ap-southeast-6", "ap-northeast-1", "ca-central-1", - # "cn-north-1", - "ap-northeast-3", "eu-central-1", "eu-west-1", "eu-west-2", @@ -22,39 +22,44 @@ "eu-north-1", "sa-east-1", "ap-east-1", - "me-south-1", - "eu-south-1", "af-south-1", + "eu-south-1", + "me-south-1", "me-central-1", "eu-central-2", - "ap-southeast-3" + "ap-northeast-3", + "ap-southeast-3", + "il-central-1" ] region_map = { - "us-east-1" : "appdevzipfiles-us-east-1", - "us-east-2" : "appdevzipfiles-us-east-2", - "us-west-1" : "appdevzipfiles-us-west-1", - "us-west-2" : "appdevzipfiles-us-west-2", + "us-east-1": "appdevzipfiles-us-east-1", + "us-east-2": "appdevzipfiles-us-east-2", + "us-west-1": "appdevzipfiles-us-west-1", + "us-west-2": "appdevzipfiles-us-west-2", "ap-south-1": "appdevzipfiles-ap-south-1", - "ap-northeast-2":"appdevzipfiles-ap-northeast-2", - "ap-southeast-1":"appdevzipfiles-ap-southeast-1", - "ap-southeast-2":"appdevzipfiles-ap-southeast-2", - "ap-northeast-1":"appdevzipfiles-ap-northeast-1", + "ap-northeast-2": "appdevzipfiles-ap-northeast-2", + "ap-southeast-1": "appdevzipfiles-ap-southeast-1", + "ap-southeast-2": "appdevzipfiles-ap-southeast-2", + "ap-southeast-4": "appdevzipfiles-ap-southeast-4s", + "ap-southeast-6": "appdevzipfiles-ap-southeast-6ss", + "ap-northeast-1": "appdevzipfiles-ap-northeast-1", "ca-central-1": "appdevzipfiles-ca-central-1", - "eu-central-1":"appdevzipfiles-eu-central-1", - "eu-west-1":"appdevzipfiles-eu-west-1", - "eu-west-2":"appdevzipfiles-eu-west-2", - "eu-west-3":"appdevzipfiles-eu-west-3", - "eu-north-1":"appdevzipfiles-eu-north-1s", - "sa-east-1":"appdevzipfiles-sa-east-1", - "ap-east-1":"appdevzipfiles-ap-east-1s", - "af-south-1":"appdevzipfiles-af-south-1s", - "eu-south-1":"appdevzipfiles-eu-south-1", - "me-south-1":"appdevzipfiles-me-south-1s", + "eu-central-1": "appdevzipfiles-eu-central-1", + "eu-west-1": "appdevzipfiles-eu-west-1", + "eu-west-2": "appdevzipfiles-eu-west-2", + "eu-west-3": "appdevzipfiles-eu-west-3", + "eu-north-1": "appdevzipfiles-eu-north-1s", + "sa-east-1": "appdevzipfiles-sa-east-1", + "ap-east-1": "appdevzipfiles-ap-east-1s", + "af-south-1": "appdevzipfiles-af-south-1s", + "eu-south-1": "appdevzipfiles-eu-south-1", + "me-south-1": "appdevzipfiles-me-south-1s", "me-central-1": "appdevzipfiles-me-central-1", - "eu-central-2":"appdevzipfiles-eu-central-2ss", - "ap-northeast-3" :"appdevzipfiles-ap-northeast-3s", - "ap-southeast-3": "appdevzipfiles-ap-southeast-3" + "eu-central-2": "appdevzipfiles-eu-central-2ss", + "ap-northeast-3": "appdevzipfiles-ap-northeast-3s", + "ap-southeast-3": "appdevzipfiles-ap-southeast-3", + "il-central-1": "appdevzipfiles-il-central-1" } @@ -62,10 +67,10 @@ def get_bucket_name(region): return region_map[region] -def upload_code_in_multiple_regions(filepath, bucket_prefix): +def upload_code_in_multiple_regions(filepath, bucket_prefix, s3_key_prefix=""): for region in regions: - upload_code_in_S3(filepath, get_bucket_name(region), region) + upload_code_in_S3(filepath, get_bucket_name(region), region, s3_key_prefix) def create_buckets(bucket_prefix): @@ -87,11 +92,12 @@ def create_buckets(bucket_prefix): -def upload_code_in_S3(filepath, bucket_name, region): +def upload_code_in_S3(filepath, bucket_name, region, s3_key_prefix=""): print("Uploading zip file in S3", region) s3 = boto3.client('s3', region) filename = os.path.basename(filepath) - s3.upload_file(filepath, bucket_name, filename, + s3_key = s3_key_prefix + filename if s3_key_prefix else filename + s3.upload_file(filepath, bucket_name, s3_key, ExtraArgs={'ACL': 'public-read'}) @@ -115,6 +121,9 @@ def upload_cftemplate(templatepath, bucket_name, region='us-east-1'): parser.add_argument("-d", "--deployment", dest="deployment", default="dev", help="aws account type") + parser.add_argument("-p", "--s3prefix", dest="s3prefix", default="", + help="S3 key prefix path for the zip file (e.g. sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/)") + args = parser.parse_args() if args.deployment == "prod": zip_bucket_prefix = "appdevzipfiles" @@ -135,6 +144,6 @@ def upload_cftemplate(templatepath, bucket_name, region='us-east-1'): if not os.path.isfile(args.zipfile): raise Exception("zipfile does not exists") else: - upload_code_in_multiple_regions(args.zipfile, zip_bucket_prefix) + upload_code_in_multiple_regions(args.zipfile, zip_bucket_prefix, args.s3prefix) print("Deployment Successfull: ALL files copied to %s" % args.deployment)