Merge branch 'feature/CG-996' into 'master'

tyler-dunkel · tyler-dunkel · commit e1dc9d66bc82 · 2022-03-24T15:02:26.000Z
feat(services): add new service rds cluster snapshot

Closes CG-996

See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-aws!226
diff --git a/README.md b/README.md
@@ -127,14 +127,15 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi
 | iot                         |                                                                                                                                                                                            |
 | kinesisFirehose             | kinesisStream, s3                                                                                                                                                                          |
 | kinesisStream               | kinesisFirehose                                                                                                                                                                            |
-| kms                         | cloudtrail, cloudwatchLog, codebuild, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, sns, sageMakerNotebookInstance, dmsReplicationInstance, redshiftCluster                                                             |
+| kms                         | cloudtrail, cloudwatchLog, codebuild, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, rdsClusterSnapshot, sns, sageMakerNotebookInstance, dmsReplicationInstance, redshiftCluster                                                             |
 | lambda                      | appSync, cognitoUserPool, kms, securityGroup, subnet, vpc                                                                                                                                           |
 | managedAirflow                      | iamRole, securityGroups, subnet, s3                                                                                                                                          |
 | nacl                        | vpc                                                                                                                                                                                        |
 | natGateway                  | networkInterface, subnet, vpc                                                                                                                                                              |
 | networkInterface            | ec2, eip, efsMountTarget, natGateway, sageMakerNotebookInstance, subnet, vpc, flowLog                                                                                                                                 |
 | organization                |  
-| rdsCluster                  | appSync, rdsDbInstance, securityGroup                                                                                                                                                               |
+| rdsCluster                  | appSync, rdsClusterSnapshot, rdsDbInstance, securityGroup                                                                                                                                                               |
+| rdsClusterSnapshot                  | kms, rdsCluster, vpc                                                                                                                                                             |
 | rdsDbInstance               | rdsCluster, securityGroup, vpc, subnet                                                                                                                                                     |
 | redshiftCluster             | kms, vpc                                                                                                                                                                                   |
 | route53Record               | alb, apiGatewayRestApi, elb, route53HostedZone                                                                                                                                             |
@@ -154,7 +155,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi
 | systemsManagerDocument                         |                                                                                                                                                                                            |
 | transitGateway              | routeTable, transitGatewayAttachment, vpnConnection                                                                                                                                                                                           |
 | transitGatewayAttachment    | routeTable, transitGateway, vpc, vpnConnection                                                                                                                                                                            |
-| vpc                         | alb, codebuild, dmsReplicationInstance, ec2, eip, elb, ecsService, efsMountTarget, eksCluster igw, elastiCacheCluster, elasticSearchDomain, lambda, nacl, natGateway, networkInterface, rdsDbInstance, redshiftCluster, route53HostedZone, routeTable, subnet, flowLog, vpnGateway, transitGatewayAttachment |
+| vpc                         | alb, codebuild, dmsReplicationInstance, ec2, eip, elb, ecsService, efsMountTarget, eksCluster igw, elastiCacheCluster, elasticSearchDomain, lambda, nacl, natGateway, networkInterface, rdsClusterSnapshot, rdsDbInstance, redshiftCluster, route53HostedZone, routeTable, subnet, flowLog, vpnGateway, transitGatewayAttachment |
 | vpnConnection               | customerGateway, transitGateway, transitGatewayAttachment, vpnGateway                                                                                                                                                                              |
 | vpnGateway                  | vpc, vpnConnection                                                                                                                                                                                        |
 | wafV2WebAcl                  |                                                                                                                                                                                        |
diff --git a/src/enums/schemasMap.ts b/src/enums/schemasMap.ts
@@ -78,6 +78,7 @@ export default {
   [services.vpnGateway]: 'awsVpnGateway',
   [services.sqs]: 'awsSqs',
   [services.rdsCluster]: 'awsRdsCluster',
+  [services.rdsClusterSnapshot]: 'awsRdsClusterSnapshot',
   [services.rdsDbInstance]: 'awsRdsDbInstance',
   [services.redshiftCluster]: 'awsRedshiftCluster',
   [services.route53HostedZone]: 'awsRoute53HostedZone',
diff --git a/src/enums/serviceAliases.ts b/src/enums/serviceAliases.ts
@@ -50,6 +50,7 @@ export default {
   [services.networkInterface]: 'networkInterfaces',
   [services.organization]: 'organizations',
   [services.rdsCluster]: 'rdsClusters',
+  [services.rdsClusterSnapshot]: 'rdsClusterSnapshots',
   [services.rdsDbInstance]: 'rdsDbInstances',
   [services.redshiftCluster]: 'redshiftClusters',
   [services.route53HostedZone]: 'route53HostedZones',
diff --git a/src/enums/serviceMap.ts b/src/enums/serviceMap.ts
@@ -93,6 +93,7 @@ import DmsReplicationInstance from '../services/dmsReplicationInstance'
 import SageMakerNotebookInstance from '../services/sageMakerNotebookInstance'
 import SystemsManagerInstance from '../services/systemsManagerInstance'
 import SystemsManagerDocument from '../services/systemsManagerDocument'
+import RdsClusterSnapshot from '../services/rdsClusterSnapshot'
 
 /**
  * serviceMap is an object that contains all currently supported services for AWS
@@ -157,6 +158,7 @@ export default {
   [services.vpc]: VPC,
   [services.sqs]: SQS,
   [services.rdsCluster]: RDSCluster,
+  [services.rdsClusterSnapshot]: RdsClusterSnapshot,
   [services.rdsDbInstance]: RDSDbInstance,
   [services.redshiftCluster]: RedshiftCluster,
   [services.route53HostedZone]: Route53HostedZone,
diff --git a/src/enums/services.ts b/src/enums/services.ts
@@ -68,6 +68,7 @@ export default {
   networkInterface: 'networkInterface',
   organization: 'organization',
   rdsCluster: 'rdsCluster',
+  rdsClusterSnapshot: 'rdsClusterSnapshot',
   rdsDbInstance: 'rdsDbInstance',
   redshiftCluster: 'redshiftCluster',
   route53HostedZone: 'route53HostedZone',
diff --git a/src/services/account/schema.graphql b/src/services/account/schema.graphql
@@ -70,6 +70,7 @@ type awsAccount @key(fields: "id") {
   networkInterfaces: [awsNetworkInterface]
   organizations: [awsOrganization]
   rdsClusters: [awsRdsCluster]
+  rdsClusterSnapshots: [awsRdsClusterSnapshot]
   rdsDbInstances: [awsRdsDbInstance]
   redshiftClusters: [awsRedshiftCluster]
   route53HostedZones: [awsRoute53HostedZone]
diff --git a/src/services/kms/schema.graphql b/src/services/kms/schema.graphql
@@ -29,4 +29,5 @@ type awsKms @key(fields: "id"){
   elasticSearchDomains: [awsElasticSearchDomain] @hasInverse(field: kms)
   dmsReplicationInstances: [awsDmsReplicationInstance] @hasInverse(field: kms)
   sageMakerNotebookInstances: [awsSageMakerNotebookInstance] @hasInverse(field: kms)
+  rdsClusterSnapshots: [awsRdsClusterSnapshot] @hasInverse(field: kms)
 }
diff --git a/src/services/rdsCluster/connections.ts b/src/services/rdsCluster/connections.ts
@@ -4,6 +4,7 @@ import { SecurityGroup } from 'aws-sdk/clients/ec2'
 import { DBInstance, DBCluster } from 'aws-sdk/clients/rds'
 
 import services from '../../enums/services'
+import { RawAwsRdsClusterSnapshot } from '../rdsClusterSnapshot/data'
 
 export default ({
   service,
@@ -27,7 +28,7 @@ export default ({
   )
 
   /**
-   * Find instances
+   * Find rds db instances
    */
   const instances: {
     name: string
@@ -51,6 +52,30 @@ export default ({
     }
   }
 
+  /**
+   * Find cluster snapshots
+   */
+   const snapshots: {
+    name: string
+    data: { [property: string]: RawAwsRdsClusterSnapshot[] }
+  } = data.find(({ name }) => name === services.rdsClusterSnapshot)
+
+  if (snapshots?.data?.[region]) {
+    const dataInRegion: RawAwsRdsClusterSnapshot[] = snapshots.data[region].filter(
+      ({ DBClusterIdentifier }: RawAwsRdsClusterSnapshot) => DBClusterIdentifier === clusterId
+    )
+    if (!isEmpty(dataInRegion)) {
+      for (const snapshot of dataInRegion) {
+        connections.push({
+          id: snapshot.DBClusterSnapshotIdentifier,
+          resourceType: services.rdsClusterSnapshot,
+          relation: 'child',
+          field: 'snapshots',
+        })
+      }
+    }
+  }
+
   /**
    * Find SecurityGroups
    */
diff --git a/src/services/rdsCluster/schema.graphql b/src/services/rdsCluster/schema.graphql
@@ -34,6 +34,7 @@ type awsRdsCluster @key(fields: "arn") {
   globalWriteForwardingRequested: Boolean @search
   tags: [awsRawTag]
   instances: [awsRdsDbInstance] @hasInverse(field: cluster)
+  snapshots: [awsRdsClusterSnapshot] @hasInverse(field: cluster)
   securityGroups: [awsSecurityGroup] @hasInverse(field: rdsCluster)
   appSync: [awsAppSync] @hasInverse(field: rdsCluster)
 }
diff --git a/src/services/rdsClusterSnapshot/connections.ts b/src/services/rdsClusterSnapshot/connections.ts
@@ -0,0 +1,43 @@
+import { ServiceConnection } from '@cloudgraph/sdk'
+import { isEmpty } from 'lodash'
+import { RawAwsRdsClusterSnapshot } from './data'
+import services from '../../enums/services'
+
+export default ({
+  service,
+  data,
+  region,
+}: {
+  account: string
+  service: RawAwsRdsClusterSnapshot
+  data: { name: string; data: { [property: string]: any[] } }[]
+  region: string
+}): {
+  [property: string]: ServiceConnection[]
+} => {
+  const { DBClusterSnapshotIdentifier, KmsKeyId } = service
+  const connections: ServiceConnection[] = []
+
+  /**
+   * Find KMS
+   */
+  const kmsKeys = data.find(({ name }) => name === services.kms)
+  if (kmsKeys?.data?.[region]) {
+    const kmsKeyInRegion = kmsKeys.data[region].filter(kmsKey => kmsKey.Arn === KmsKeyId)
+    if (!isEmpty(kmsKeyInRegion)) {
+      for (const kms of kmsKeyInRegion) {
+        connections.push({
+          id: kms.KeyId,
+          resourceType: services.kms,
+          relation: 'child',
+          field: 'kms',
+        })
+      }
+    }
+  }
+
+  const result = {
+    [DBClusterSnapshotIdentifier]: connections,
+  }
+  return result
+}
diff --git a/src/services/rdsClusterSnapshot/data.ts b/src/services/rdsClusterSnapshot/data.ts
@@ -0,0 +1,71 @@
+import { Config } from 'aws-sdk/lib/config'
+import { groupBy, isEmpty } from 'lodash'
+import RDS from 'aws-sdk/clients/rds'
+import { convertToPromise, fetchAllPaginatedData } from '../../utils/fetchUtils'
+import { initTestEndpoint } from '../../utils'
+import ErrorLog from '../../utils/errorLog'
+import { convertAwsTagsToTagMap } from '../../utils/format'
+import { TagMap, AwsTag } from '../../types'
+
+const serviceName = 'rdsClusterSnapshot'
+const endpoint = initTestEndpoint(serviceName)
+const errorLog = new ErrorLog(serviceName)
+
+export interface RawAwsRdsClusterSnapshot extends RDS.DBClusterSnapshot {
+  region: string
+  attributes: RDS.DBClusterSnapshotAttributeList
+  Tags: TagMap
+}
+
+/**
+ * RdsClusterSnapshot
+ */
+
+export default async ({
+  regions,
+  config,
+}: {
+  regions: string
+  config: Config
+}): Promise<{ [region: string]: RawAwsRdsClusterSnapshot[] }> => {
+  const result: RawAwsRdsClusterSnapshot[] = []
+
+  const activeRegions = regions.split(',')
+
+  for (const region of activeRegions) {
+    const client = new RDS({ ...config, region, endpoint })
+    let rdsClusterSnapshotData: RDS.DBClusterSnapshot[]
+    try {
+      rdsClusterSnapshotData = await fetchAllPaginatedData({
+        getResourcesFn: convertToPromise({
+          sdkContext: client,
+          fnName: 'describeDBClusterSnapshots',
+        }),
+        accessor: '',
+      })
+    } catch (err) {
+      errorLog.generateAwsErrorLog({ functionName: 'describeDBClusterSnapshots', err })
+    }
+
+    if (!isEmpty(rdsClusterSnapshotData)) {
+      for (const snapshot of rdsClusterSnapshotData) {
+        let snapshotAttributes: RDS.DBClusterSnapshotAttributeList
+        try {
+          const attributeResponse = await client
+            .describeDBClusterSnapshotAttributes({
+              DBClusterSnapshotIdentifier: snapshot.DBClusterSnapshotIdentifier,
+            })
+            .promise()
+          snapshotAttributes =
+            attributeResponse?.DBClusterSnapshotAttributesResult?.DBClusterSnapshotAttributes
+        } catch (err) {
+          errorLog.generateAwsErrorLog({ functionName: 'describeDBClusterSnapshotAttributes', err })
+        }
+        result.push({ ...snapshot, Tags: convertAwsTagsToTagMap(snapshot?.TagList as AwsTag[]), attributes: snapshotAttributes, region })
+      }
+    }
+  }
+
+  errorLog.reset()
+  return groupBy(result, 'region')
+}
diff --git a/src/services/rdsClusterSnapshot/format.ts b/src/services/rdsClusterSnapshot/format.ts
@@ -0,0 +1,78 @@
+import cuid from 'cuid'
+import { formatTagsFromMap } from '../../utils/format'
+import { AwsRdsClusterSnapshot } from '../../types/generated'
+import { RawAwsRdsClusterSnapshot } from './data'
+
+/**
+ * RdsClusterSnapshot
+ */
+
+export default ({
+  account,
+  service: rawData,
+  region,
+}: {
+  account: string
+  service: RawAwsRdsClusterSnapshot
+  region: string
+}): AwsRdsClusterSnapshot => {
+  const {
+    AvailabilityZones: availabilityZones,
+    DBClusterSnapshotIdentifier: dbClusterSnapshotIdentifier,
+    DBClusterIdentifier: dbClusterIdentifier,
+    SnapshotCreateTime: snapshotCreateTime,
+    Engine: engine,
+    EngineMode: engineMode,
+    AllocatedStorage: allocatedStorage,
+    Status: status,
+    Port: port,
+    VpcId: vpcId,
+    ClusterCreateTime: clusterCreateTime,
+    MasterUsername: masterUsername,
+    EngineVersion: engineVersion,
+    LicenseModel: licenseModel,
+    SnapshotType: snapshotType,
+    PercentProgress: percentProgress,
+    StorageEncrypted: storageEncrypted,
+    KmsKeyId: kmsKeyId,
+    DBClusterSnapshotArn: dbClusterSnapshotArn,
+    SourceDBClusterSnapshotArn: sourceDBClusterSnapshotArn,
+    IAMDatabaseAuthenticationEnabled: iamDatabaseAuthenticationEnabled,
+    Tags,
+    attributes,
+  } = rawData
+
+  const mappedAttributes = attributes?.map(({ AttributeName, AttributeValues }) => ({
+    id: cuid(),
+    name: AttributeName,
+    values: AttributeValues
+  }))
+
+  return {
+    id: dbClusterSnapshotIdentifier,
+    arn: dbClusterSnapshotArn,
+    region,
+    accountId: account,
+    availabilityZones,
+    dbClusterIdentifier,
+    snapshotCreateTime: snapshotCreateTime?.toISOString(),
+    engine,
+    engineMode,
+    allocatedStorage,
+    status,
+    port,
+    vpcId,
+    clusterCreateTime: clusterCreateTime?.toISOString(),
+    masterUsername,
+    engineVersion,
+    licenseModel,
+    snapshotType,
+    percentProgress,
+    storageEncrypted,
+    kmsKeyId,
+    iamDatabaseAuthenticationEnabled,
+    sourceDBClusterSnapshotArn,
+    tags: formatTagsFromMap(Tags ?? {}),
+    attributes: mappedAttributes
+  }
+}
diff --git a/src/services/rdsClusterSnapshot/index.ts b/src/services/rdsClusterSnapshot/index.ts
@@ -0,0 +1,17 @@
+import { Service } from '@cloudgraph/sdk'
+  import BaseService from '../base'
+  import format from './format'
+  import getData from './data'
+  import getConnections from './connections'
+  import mutation from './mutation'
+      
+  export default class RdsClusterSnapshot extends BaseService implements Service {
+    format = format.bind(this)
+      
+    getData = getData.bind(this)
+
+    getConnections = getConnections.bind(this)
+      
+    mutation = mutation
+  }
+  
diff --git a/src/services/rdsClusterSnapshot/mutation.ts b/src/services/rdsClusterSnapshot/mutation.ts
@@ -0,0 +1,5 @@
+export default `mutation($input: [AddawsRdsClusterSnapshotInput!]!) {
+  addawsRdsClusterSnapshot(input: $input, upsert: true) {
+    numUids
+  }
+}`;
diff --git a/src/services/rdsClusterSnapshot/schema.graphql b/src/services/rdsClusterSnapshot/schema.graphql
diff --git a/src/services/tag/connections.ts b/src/services/tag/connections.ts
diff --git a/src/services/tag/schema.graphql b/src/services/tag/schema.graphql
diff --git a/src/services/vpc/connections.ts b/src/services/vpc/connections.ts
diff --git a/src/services/vpc/schema.graphql b/src/services/vpc/schema.graphql
diff --git a/src/types/generated.ts b/src/types/generated.ts

Original file line number	Diff line number	Diff line change
`@@ -29,4 +29,5 @@ type awsKms @key(fields: "id"){`
`29`	`29`	`elasticSearchDomains: [awsElasticSearchDomain] @hasInverse(field: kms)`
`30`	`30`	`dmsReplicationInstances: [awsDmsReplicationInstance] @hasInverse(field: kms)`
`31`	`31`	`sageMakerNotebookInstances: [awsSageMakerNotebookInstance] @hasInverse(field: kms)`
	`32`	`+ rdsClusterSnapshots: [awsRdsClusterSnapshot] @hasInverse(field: kms)`
`32`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,7 @@ type awsRdsCluster @key(fields: "arn") {`
`34`	`34`	`globalWriteForwardingRequested: Boolean @search`
`35`	`35`	`tags: [awsRawTag]`
`36`	`36`	`instances: [awsRdsDbInstance] @hasInverse(field: cluster)`
	`37`	`+ snapshots: [awsRdsClusterSnapshot] @hasInverse(field: cluster)`
`37`	`38`	`securityGroups: [awsSecurityGroup] @hasInverse(field: rdsCluster)`
`38`	`39`	`appSync: [awsAppSync] @hasInverse(field: rdsCluster)`
`39`	`40`	`}`